From 5f97adedf657ecc6ef0c93f2c9e3dc9403eec9ac Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sat, 2 May 2020 15:55:30 +0200 Subject: [PATCH 01/22] GPU: Shuffle some TPC SliceData functions around for later GPU adaption --- GPU/GPUTracking/Base/GPUReconstruction.h | 1 + .../SliceTracker/GPUTPCSliceData.cxx | 129 ++++++++++-------- .../SliceTracker/GPUTPCSliceData.h | 20 +-- .../SliceTracker/GPUTPCTracker.cxx | 2 +- GPU/GPUTracking/SliceTracker/GPUTPCTracker.h | 45 +++--- .../Standalone/display/GPUDisplay.cxx | 4 +- 6 files changed, 102 insertions(+), 99 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstruction.h b/GPU/GPUTracking/Base/GPUReconstruction.h index 1a81754b81372..745f5c28813ef 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.h +++ b/GPU/GPUTracking/Base/GPUReconstruction.h @@ -193,6 +193,7 @@ class GPUReconstruction DeviceType GetDeviceType() const { return (DeviceType)mProcessingSettings.deviceType; } bool IsGPU() const { return GetDeviceType() != DeviceType::INVALID_DEVICE && GetDeviceType() != DeviceType::CPU; } const GPUParam& GetParam() const { return mHostConstantMem->param; } + const GPUConstantMem& GetConstantMem() const { return *mHostConstantMem; } const GPUSettingsEvent& GetEventSettings() const { return mEventSettings; } const GPUSettingsProcessing& GetProcessingSettings() { return mProcessingSettings; } const GPUSettingsDeviceProcessing& GetDeviceProcessingSettings() const { return mDeviceProcessingSettings; } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index 37352e19ff41e..383cf251de71b 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -19,12 +19,82 @@ #include "GPUProcessor.h" #include "GPUO2DataTypes.h" #include "GPUTPCConvertImpl.h" +#include "GPUCommonMath.h" #include #include #include "utils/vecpod.h" using namespace GPUCA_NAMESPACE::gpu; +#ifndef GPUCA_GPUCODE + +void GPUTPCSliceData::InitializeRows(const MEM_CONSTANT(GPUParam) & p) +{ + // initialisation of rows + for (int i = 0; i < GPUCA_ROW_COUNT + 1; ++i) { + new (&mRows[i]) GPUTPCRow; + } + for (int i = 0; i < GPUCA_ROW_COUNT; ++i) { + mRows[i].mX = p.tpcGeometry.Row2X(i); + mRows[i].mMaxY = CAMath::Tan(p.DAlpha / 2.) * mRows[i].mX; + } +} + +void GPUTPCSliceData::SetClusterData(const GPUTPCClusterData* data, int nClusters, int clusterIdOffset) +{ + mClusterData = data; + mNumberOfHits = nClusters; + mClusterIdOffset = clusterIdOffset; +} + +void GPUTPCSliceData::SetMaxData() +{ + int hitMemCount = GPUCA_ROW_COUNT * GPUCA_ROWALIGNMENT + mNumberOfHits; + const unsigned int kVectorAlignment = 256; + mNumberOfHitsPlusAlign = GPUProcessor::nextMultipleOf<(kVectorAlignment > GPUCA_ROWALIGNMENT ? kVectorAlignment : GPUCA_ROWALIGNMENT) / sizeof(int)>(hitMemCount); +} + +void* GPUTPCSliceData::SetPointersInput(void* mem, bool idsOnGPU) +{ + const int firstHitInBinSize = (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; + GPUProcessor::computePointerWithAlignment(mem, mHitData, mNumberOfHitsPlusAlign); + GPUProcessor::computePointerWithAlignment(mem, mFirstHitInBin, firstHitInBinSize); + if (idsOnGPU) { + mem = SetPointersScratchHost(mem, false); // Hijack the allocation from SetPointersScratchHost + } + return mem; +} + +void* GPUTPCSliceData::SetPointersScratch(const GPUConstantMem& cm, void* mem) +{ + GPUProcessor::computePointerWithAlignment(mem, mLinkUpData, mNumberOfHitsPlusAlign); + GPUProcessor::computePointerWithAlignment(mem, mLinkDownData, mNumberOfHitsPlusAlign); + GPUProcessor::computePointerWithAlignment(mem, mHitWeights, mNumberOfHitsPlusAlign + 16 / sizeof(*mHitWeights)); + size_t tmpMemMaxSize = CAMath::nextMultipleOf<4>(mNumberOfHits + 256) * (sizeof(float2) + sizeof(int) + sizeof(GPUTPCHit)); + int maxbiny, maxbinz; + GetMaxNBins(&cm, maxbiny, maxbinz); + size_t binMemoryMaxSize = (2 * (maxbiny * maxbinz + 3) + mNumberOfHits) * sizeof(calink) + GPUCA_ROW_COUNT * GPUCA_ROWALIGNMENT; + tmpMemMaxSize += binMemoryMaxSize; + GPUProcessor::computePointerWithAlignment(mem, mTmpMem, (tmpMemMaxSize + sizeof(*mTmpMem) - 1) / sizeof(*mTmpMem)); + return mem; +} + +void* GPUTPCSliceData::SetPointersScratchHost(void* mem, bool idsOnGPU) +{ + if (!idsOnGPU) { + GPUProcessor::computePointerWithAlignment(mem, mClusterDataIndex, mNumberOfHitsPlusAlign); + } + return mem; +} + +void* GPUTPCSliceData::SetPointersRows(void* mem) +{ + GPUProcessor::computePointerWithAlignment(mem, mRows, GPUCA_ROW_COUNT + 1); + return mem; +} + +#endif + // calculates an approximation for 1/sqrt(x) // Google for 0x5f3759df :) static inline float fastInvSqrt(float _x) @@ -116,65 +186,6 @@ inline int GPUTPCSliceData::PackHitData(GPUTPCRow* const GPUrestrict() row, cons return 0; } -void GPUTPCSliceData::InitializeRows(const GPUParam& p) -{ - // initialisation of rows - for (int i = 0; i < GPUCA_ROW_COUNT + 1; ++i) { - new (&mRows[i]) GPUTPCRow; - } - for (int i = 0; i < GPUCA_ROW_COUNT; ++i) { - mRows[i].mX = p.tpcGeometry.Row2X(i); - mRows[i].mMaxY = CAMath::Tan(p.DAlpha / 2.) * mRows[i].mX; - } -} - -void GPUTPCSliceData::SetClusterData(const GPUTPCClusterData* data, int nClusters, int clusterIdOffset) -{ - mClusterData = data; - mNumberOfHits = nClusters; - mClusterIdOffset = clusterIdOffset; -} - -void GPUTPCSliceData::SetMaxData() -{ - int hitMemCount = GPUCA_ROW_COUNT * GPUCA_ROWALIGNMENT + mNumberOfHits; - const unsigned int kVectorAlignment = 256; - mNumberOfHitsPlusAlign = GPUProcessor::nextMultipleOf<(kVectorAlignment > GPUCA_ROWALIGNMENT ? kVectorAlignment : GPUCA_ROWALIGNMENT) / sizeof(int)>(hitMemCount); -} - -void* GPUTPCSliceData::SetPointersInput(void* mem, bool idsOnGPU) -{ - const int firstHitInBinSize = (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; - GPUProcessor::computePointerWithAlignment(mem, mHitData, mNumberOfHitsPlusAlign); - GPUProcessor::computePointerWithAlignment(mem, mFirstHitInBin, firstHitInBinSize); - if (idsOnGPU) { - mem = SetPointersScratchHost(mem, false); // Hijack the allocation from SetPointersScratchHost - } - return mem; -} - -void* GPUTPCSliceData::SetPointersScratch(void* mem) -{ - GPUProcessor::computePointerWithAlignment(mem, mLinkUpData, mNumberOfHitsPlusAlign); - GPUProcessor::computePointerWithAlignment(mem, mLinkDownData, mNumberOfHitsPlusAlign); - GPUProcessor::computePointerWithAlignment(mem, mHitWeights, mNumberOfHitsPlusAlign + 16 / sizeof(*mHitWeights)); - return mem; -} - -void* GPUTPCSliceData::SetPointersScratchHost(void* mem, bool idsOnGPU) -{ - if (!idsOnGPU) { - GPUProcessor::computePointerWithAlignment(mem, mClusterDataIndex, mNumberOfHitsPlusAlign); - } - return mem; -} - -void* GPUTPCSliceData::SetPointersRows(void* mem) -{ - GPUProcessor::computePointerWithAlignment(mem, mRows, GPUCA_ROW_COUNT + 1); - return mem; -} - int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) { //////////////////////////////////// diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index 66734b77d31c3..95ed01d4efc2f 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -33,24 +33,16 @@ class GPUTPCSliceData public: GPUTPCSliceData() : mNumberOfHits(0), mNumberOfHitsPlusAlign(0), mClusterIdOffset(0), mMaxZ(0.f), mGPUTextureBase(nullptr), mRows(nullptr), mLinkUpData(nullptr), mLinkDownData(nullptr), mClusterData(nullptr) {} -#ifndef GPUCA_GPUCODE +#ifndef GPUCA_GPUCODE_DEVICE ~GPUTPCSliceData() CON_DEFAULT; -#endif //! GPUCA_GPUCODE - - MEM_CLASS_PRE2() - void InitializeRows(const MEM_LG2(GPUParam) & parameters); - - /** - * (Re)Create the data that is tuned for optimal performance of the algorithm from the cluster - * data. - */ - + void InitializeRows(const MEM_CONSTANT(GPUParam) & p); void SetMaxData(); void SetClusterData(const GPUTPCClusterData* data, int nClusters, int clusterIdOffset); void* SetPointersInput(void* mem, bool idsOnGPU); - void* SetPointersScratch(void* mem); + void* SetPointersScratch(GPUconstantref() const MEM_CONSTANT(GPUConstantMem)& cm, void* mem); void* SetPointersScratchHost(void* mem, bool idsOnGPU); void* SetPointersRows(void* mem); +#endif int InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int iSlice); @@ -165,7 +157,9 @@ class GPUTPCSliceData GPUglobalref() calink* mLinkDownData; // hit index in the row below which is linked to the given (global) hit index GPUglobalref() cahit2* mHitData; // packed y,z coordinate of the given (global) hit index GPUglobalref() int* mClusterDataIndex; // see ClusterDataIndex() - + + uint4* mTmpMem; + /* * The size of the array is row.Grid.N + row.Grid.Ny + 3. The row.Grid.Ny + 3 is an optimization * to remove the need for bounds checking. The last values are the same as the entry at [N - 1]. diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx index 2fa0d4b655ea0..1d13826350598 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx @@ -66,7 +66,7 @@ void GPUTPCTracker::InitializeProcessor() void* GPUTPCTracker::SetPointersDataInput(void* mem) { return mData.SetPointersInput(mem, mRec->GetRecoStepsGPU() & GPUReconstruction::RecoStep::TPCMerging); } -void* GPUTPCTracker::SetPointersDataScratch(void* mem) { return mData.SetPointersScratch(mem); } +void* GPUTPCTracker::SetPointersDataScratch(void* mem) { return mData.SetPointersScratch(mRec->GetConstantMem(), mem); } void* GPUTPCTracker::SetPointersDataRows(void* mem) { return mData.SetPointersRows(mem); } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h index 3dfa8fc04f992..86cc77675ffaf 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h @@ -50,6 +50,27 @@ class GPUTPCTracker : public GPUProcessor ~GPUTPCTracker(); GPUTPCTracker(const GPUTPCTracker&) CON_DELETE; GPUTPCTracker& operator=(const GPUTPCTracker&) CON_DELETE; + + MEM_CLASS_PRE2() + void SetSlice(int iSlice); + MEM_CLASS_PRE2() + void InitializeProcessor(); + MEM_CLASS_PRE2() + void InitializeRows(const MEM_CONSTANT(GPUParam) * param) { mData.InitializeRows(*param); } + + int ReadEvent(); + int CheckEmptySlice(); + void WriteOutputPrepare(); + void WriteOutput(); + + // Debugging Stuff + void DumpSliceData(std::ostream& out); // Dump Input Slice Data + void DumpLinks(std::ostream& out); // Dump all links to file (for comparison after NeighboursFinder/Cleaner) + void DumpStartHits(std::ostream& out); // Same for Start Hits + void DumpHitWeights(std::ostream& out); //.... + void DumpTrackHits(std::ostream& out); // Same for Track Hits + void DumpTrackletHits(std::ostream& out); // Same for Track Hits + void DumpOutput(std::ostream& out); // Similar for output #endif struct StructGPUParameters { @@ -74,30 +95,6 @@ class GPUTPCTracker : public GPUProcessor StructGPUParameters gpuParameters; // GPU parameters }; - MEM_CLASS_PRE2() - void SetSlice(int iSlice); - MEM_CLASS_PRE2() - void InitializeProcessor(); - MEM_CLASS_PRE2() - void InitializeRows(const MEM_CONSTANT(GPUParam) * param) { mData.InitializeRows(*param); } - - int CheckEmptySlice(); - void WriteOutputPrepare(); - void WriteOutput(); - -// GPU Tracker Interface -#if !defined(GPUCA_GPUCODE_DEVICE) - // Debugging Stuff - void DumpSliceData(std::ostream& out); // Dump Input Slice Data - void DumpLinks(std::ostream& out); // Dump all links to file (for comparison after NeighboursFinder/Cleaner) - void DumpStartHits(std::ostream& out); // Same for Start Hits - void DumpHitWeights(std::ostream& out); //.... - void DumpTrackHits(std::ostream& out); // Same for Track Hits - void DumpTrackletHits(std::ostream& out); // Same for Track Hits - void DumpOutput(std::ostream& out); // Similar for output - - int ReadEvent(); -#endif #if !defined(__OPENCL__) || defined(__OPENCLCPP__) GPUhdi() GPUglobalref() const GPUTPCClusterData* ClusterData() const diff --git a/GPU/GPUTracking/Standalone/display/GPUDisplay.cxx b/GPU/GPUTracking/Standalone/display/GPUDisplay.cxx index 53893f0df19e2..ddfd45df231e0 100644 --- a/GPU/GPUTracking/Standalone/display/GPUDisplay.cxx +++ b/GPU/GPUTracking/Standalone/display/GPUDisplay.cxx @@ -1551,9 +1551,9 @@ int GPUDisplay::DrawGLScene_internal(bool mixAnimation, float mAnimateTime) #endif for (int iSlice = 0; iSlice < NSLICES; iSlice++) { GPUTPCTracker& tracker = (GPUTPCTracker&)sliceTracker(iSlice); - tracker.Data().SetPointersScratch(tracker.LinkTmpMemory()); + tracker.Data().SetPointersScratch(*tracker.GetConstantMem(), tracker.LinkTmpMemory()); mGlDLLines[iSlice][tINITLINK] = DrawLinks(tracker, tINITLINK, true); - tracker.Data().SetPointersScratch(mChain->rec()->Res(tracker.MemoryResLinksScratch()).Ptr()); + tracker.Data().SetPointersScratch(*tracker.GetConstantMem(), mChain->rec()->Res(tracker.MemoryResLinksScratch()).Ptr()); } GPUTPCGMPropagator prop; const float kRho = 1.025e-3; // 0.9e-3; From d1b80eae0a945e92b8530058d02f93fe30f4b587 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sat, 2 May 2020 15:57:49 +0200 Subject: [PATCH 02/22] GPU: Add global constants for max/min TPC FastSearch Grid bin size --- GPU/Common/GPUDefConstantsAndSettings.h | 3 ++ GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx | 9 ++-- GPU/GPUTracking/SliceTracker/GPUTPCGrid.h | 2 +- .../SliceTracker/GPUTPCSliceData.cxx | 50 +++++++++++++------ .../SliceTracker/GPUTPCSliceData.h | 7 ++- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/GPU/Common/GPUDefConstantsAndSettings.h b/GPU/Common/GPUDefConstantsAndSettings.h index aa9967f6cc7ae..fc6833b402364 100644 --- a/GPU/Common/GPUDefConstantsAndSettings.h +++ b/GPU/Common/GPUDefConstantsAndSettings.h @@ -55,6 +55,9 @@ #define GPUCA_MAX_SIN_PHI_LOW 0.99f // Limits for maximum sin phi during fit #define GPUCA_MAX_SIN_PHI 0.999f // Must be preprocessor define because c++ pre 11 cannot use static constexpr for initializes +#define GPUCA_MIN_BIN_SIZE 2.f // Minimum bin size in TPC fast access grid +#define GPUCA_MAX_BIN_SIZE 1000.f // Maximum bin size in TPC fast access grid + #define GPUCA_TPC_COMP_CHUNK_SIZE 1024 // Chunk size of sorted unattached TPC cluster in compression #if defined(HAVE_O2HEADERS) && (!defined(__OPENCL__) || defined(__OPENCLCPP__)) && !(defined(ROOT_VERSION_CODE) && ROOT_VERSION_CODE < 393216) diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx index f0503a9aa5ae2..3a97c7ab72d86 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx @@ -37,17 +37,20 @@ void MEM_LG(GPUTPCGrid)::CreateEmpty() } MEM_CLASS_PRE() -GPUd() void MEM_LG(GPUTPCGrid)::Create(float yMin, float yMax, float zMin, float zMax, float sy, float sz) +GPUd() void MEM_LG(GPUTPCGrid)::Create(float yMin, float yMax, float zMin, float zMax, int ny, int nz) { //* Create the grid mYMin = yMin; mZMin = zMin; + float sy = CAMath::Max((yMax + 0.1f - yMin) / ny, GPUCA_MIN_BIN_SIZE); + float sz = CAMath::Max((zMax + 0.1f - zMin) / nz, GPUCA_MIN_BIN_SIZE); + mStepYInv = 1.f / sy; mStepZInv = 1.f / sz; - mNy = static_cast((yMax - mYMin) * mStepYInv + 1.f); - mNz = static_cast((zMax - mZMin) * mStepZInv + 1.f); + mNy = ny; + mNz = nz; mN = mNy * mNz; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h index 7d115948950c2..9268fefb40dda 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h @@ -33,7 +33,7 @@ class GPUTPCGrid { public: void CreateEmpty(); - GPUd() void Create(float yMin, float yMax, float zMin, float zMax, float sy, float sz); + GPUd() void Create(float yMin, float yMax, float zMin, float zMax, int ny, int nz); GPUd() int GetBin(float Y, float Z) const; /** diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index 383cf251de71b..45253fdc9e74f 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -54,9 +54,14 @@ void GPUTPCSliceData::SetMaxData() mNumberOfHitsPlusAlign = GPUProcessor::nextMultipleOf<(kVectorAlignment > GPUCA_ROWALIGNMENT ? kVectorAlignment : GPUCA_ROWALIGNMENT) / sizeof(int)>(hitMemCount); } +unsigned int GPUTPCSliceData::GetGridSize() +{ + return (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; +} + void* GPUTPCSliceData::SetPointersInput(void* mem, bool idsOnGPU) { - const int firstHitInBinSize = (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; + const int firstHitInBinSize = GetGridSize(); GPUProcessor::computePointerWithAlignment(mem, mHitData, mNumberOfHitsPlusAlign); GPUProcessor::computePointerWithAlignment(mem, mFirstHitInBin, firstHitInBinSize); if (idsOnGPU) { @@ -110,7 +115,14 @@ static inline float fastInvSqrt(float _x) return x.f; } -inline void GPUTPCSliceData::CreateGrid(GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset) +void GPUTPCSliceData::GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int& maxY, int& maxZ) +{ + maxY = mRows[GPUCA_ROW_COUNT - 1].mMaxY * 2.f / GPUCA_MIN_BIN_SIZE + 1; + maxZ = mem->param.continuousMaxTimeBin > 0 ? mem->calibObjects.fastTransform->convTimeToZinTimeFrame(0, 0, mem->param.continuousMaxTimeBin) + 50 : 300; + maxZ = maxZ / GPUCA_MIN_BIN_SIZE + 1; +} + +inline void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset) { // grid creation if (row->NHits() <= 0) { // no hits or invalid data @@ -147,7 +159,13 @@ inline void GPUTPCSliceData::CreateGrid(GPUTPCRow* GPUrestrict() row, const floa dz = 250.; } const float norm = fastInvSqrt(row->mNHits / tfFactor); - row->mGrid.Create(yMin, yMax, zMin, zMax, CAMath::Max((yMax - yMin) * norm, 2.f), CAMath::Max(dz * norm, 2.f)); + float sy = CAMath::Min(CAMath::Max((yMax - yMin) * norm, GPUCA_MIN_BIN_SIZE), GPUCA_MAX_BIN_SIZE); + float sz = CAMath::Min(CAMath::Max(dz * norm, GPUCA_MIN_BIN_SIZE), GPUCA_MAX_BIN_SIZE); + int maxy, maxz; + GetMaxNBins(mem, maxy, maxz); + int ny = CAMath::Max(1, CAMath::Min(maxy, (yMax - yMin) / sy + 1)); + int nz = CAMath::Max(1, CAMath::Min(maxz, (zMax - zMin) / sz + 1)); + row->mGrid.Create(yMin, yMax, zMin, zMax, ny, nz); } inline int GPUTPCSliceData::PackHitData(GPUTPCRow* const GPUrestrict() row, const GPUTPCHit* GPUrestrict() binSortedHits) @@ -217,11 +235,11 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU for (int i = 0; i < GPUCA_ROW_COUNT; i++) { if ((long long int)NumberOfClustersInRow[i] >= ((long long int)1 << (sizeof(calink) * 8))) { GPUError("Too many clusters in row %d for row indexing (%d >= %lld), indexing insufficient", i, NumberOfClustersInRow[i], ((long long int)1 << (sizeof(calink) * 8))); - return (1); + return 1; } if (NumberOfClustersInRow[i] >= (1 << 24)) { GPUError("Too many clusters in row %d for hit id indexing (%d >= %d), indexing insufficient", i, NumberOfClustersInRow[i], 1 << 24); - return (1); + return 1; } RowOffset[i] = tmpOffset; tmpOffset += NumberOfClustersInRow[i]; @@ -261,10 +279,10 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU vecpod binSortedHits(mNumberOfHits + GPUCA_ROWALIGNMENT); - int gridContentOffset = 0; - int hitOffset = 0; + unsigned int gridContentOffset = 0; + unsigned int hitOffset = 0; - int binCreationMemorySize = 103 * 2 + mNumberOfHits; + unsigned int binCreationMemorySize = 103 * 2 + mNumberOfHits; vecpod binCreationMemory(binCreationMemorySize); for (int rowIndex = 0; rowIndex < GPUCA_ROW_COUNT; ++rowIndex) { @@ -285,16 +303,21 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU } row.mNHits = NumberOfClustersInRow[rowIndex]; row.mHitNumberOffset = hitOffset; - hitOffset += GPUProcessor::nextMultipleOf(NumberOfClustersInRow[rowIndex]); + hitOffset += GPUProcessor::nextMultipleOf(NumberOfClustersInRow[rowIndex]); row.mFirstHitInBinOffset = gridContentOffset; - CreateGrid(&row, YZData, RowOffset[rowIndex]); + CreateGrid(mem, &row, YZData, RowOffset[rowIndex]); const GPUTPCGrid& grid = row.mGrid; const int numberOfBins = grid.N(); if ((long long int)numberOfBins >= ((long long int)1 << (sizeof(calink) * 8))) { GPUError("Too many bins in row %d for grid (%d >= %lld), indexing insufficient", rowIndex, numberOfBins, ((long long int)1 << (sizeof(calink) * 8))); - return (1); + return 1; + } + const int nn = numberOfBins + grid.Ny() + 3; + if (gridContentOffset + nn >= GetGridSize()) { + GPUError("firstHitInBin overflow"); + return 1; } int binCreationMemorySizeNew = numberOfBins * 2 + 6 + row.mNHits + GPUCA_ROWALIGNMENT / sizeof(unsigned short) * (GPUCA_ROW_COUNT + 1) + 1; @@ -338,7 +361,7 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU } if (PackHitData(&row, binSortedHits.data())) { - return (1); + return 1; } for (int i = 0; i < numberOfBins; ++i) { @@ -346,7 +369,6 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU } const calink a = c[numberOfBins]; // grid.N is <= row.mNHits - const int nn = numberOfBins + grid.Ny() + 3; for (int i = numberOfBins; i < nn; ++i) { mFirstHitInBin[row.mFirstHitInBinOffset + i] = a; } @@ -358,5 +380,5 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU gridContentOffset = GPUProcessor::nextMultipleOf(gridContentOffset); } - return (0); + return 0; } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index 95ed01d4efc2f..2a62b951e5789 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -37,6 +37,7 @@ class GPUTPCSliceData ~GPUTPCSliceData() CON_DEFAULT; void InitializeRows(const MEM_CONSTANT(GPUParam) & p); void SetMaxData(); + unsigned int GetGridSize(); void SetClusterData(const GPUTPCClusterData* data, int nClusters, int clusterIdOffset); void* SetPointersInput(void* mem, bool idsOnGPU); void* SetPointersScratch(GPUconstantref() const MEM_CONSTANT(GPUConstantMem)& cm, void* mem); @@ -137,9 +138,11 @@ class GPUTPCSliceData #ifndef GPUCA_GPUCODE GPUTPCSliceData& operator=(const GPUTPCSliceData&) CON_DELETE; // ROOT 5 tries to use this if it is not private GPUTPCSliceData(const GPUTPCSliceData&) CON_DELETE; // - void CreateGrid(GPUTPCRow* row, const float2* data, int ClusterDataHitNumberOffset); - int PackHitData(GPUTPCRow* row, const GPUTPCHit* binSortedHits); #endif + void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow)* row, const float2* data, int ClusterDataHitNumberOffset); + void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int& maxY, int& maxZ); + int PackHitData(MEM_GLOBAL(GPUTPCRow)* row, const GPUTPCHit* binSortedHits); + friend class GPUTPCNeighboursFinder; friend class GPUTPCStartHitsFinder; From a01f443bf3b1f36ee1ef460c5e6312a5cb71ef7f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 3 May 2020 17:48:47 +0200 Subject: [PATCH 03/22] GPU: Create TPC FastSearch Grid bins in-place --- GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx | 2 +- GPU/GPUTracking/SliceTracker/GPUTPCRow.cxx | 2 +- GPU/GPUTracking/SliceTracker/GPUTPCRow.h | 2 - .../SliceTracker/GPUTPCSliceData.cxx | 119 ++++++------------ .../SliceTracker/GPUTPCSliceData.h | 11 +- 5 files changed, 46 insertions(+), 90 deletions(-) diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx index 3a97c7ab72d86..463035e467c40 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx @@ -45,7 +45,7 @@ GPUd() void MEM_LG(GPUTPCGrid)::Create(float yMin, float yMax, float zMin, float float sy = CAMath::Max((yMax + 0.1f - yMin) / ny, GPUCA_MIN_BIN_SIZE); float sz = CAMath::Max((zMax + 0.1f - zMin) / nz, GPUCA_MIN_BIN_SIZE); - + mStepYInv = 1.f / sy; mStepZInv = 1.f / sz; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCRow.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCRow.cxx index 17bfc5b42a670..71ae58aa3fd62 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCRow.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCRow.cxx @@ -15,7 +15,7 @@ using namespace GPUCA_NAMESPACE::gpu; #if !defined(GPUCA_GPUCODE) -GPUTPCRow::GPUTPCRow() : mNHits(0), mX(0), mMaxY(0), mGrid(), mHy0(0), mHz0(0), mHstepY(0), mHstepZ(0), mHstepYi(0), mHstepZi(0), mFullSize(0), mHitNumberOffset(0), mFirstHitInBinOffset(0) +GPUTPCRow::GPUTPCRow() : mNHits(0), mX(0), mMaxY(0), mGrid(), mHy0(0), mHz0(0), mHstepY(0), mHstepZ(0), mHstepYi(0), mHstepZi(0), mHitNumberOffset(0), mFirstHitInBinOffset(0) { // dummy constructor } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCRow.h b/GPU/GPUTracking/SliceTracker/GPUTPCRow.h index 6a8fd499a6530..8303248168ffd 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCRow.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCRow.h @@ -53,7 +53,6 @@ class GPUTPCRow GPUhd() float HstepZ() const { return mHstepZ; } GPUhd() float HstepYi() const { return mHstepYi; } GPUhd() float HstepZi() const { return mHstepZi; } - GPUhd() int FullSize() const { return mFullSize; } GPUhd() int HitNumberOffset() const { return mHitNumberOffset; } GPUhd() unsigned int FirstHitInBinOffset() const { return mFirstHitInBinOffset; } @@ -75,7 +74,6 @@ class GPUTPCRow float mHstepYi; // inverse step size float mHstepZi; // inverse step size - int mFullSize; // size of this row in Tracker::mRowData int mHitNumberOffset; // index of the first hit in the hit array, used as // offset in GPUTPCSliceData::LinkUp/DownData/HitDataY/... unsigned int mFirstHitInBinOffset; // offset in Tracker::mRowData to find the FirstHitInBin diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index 45253fdc9e74f..f584c994e408d 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -76,10 +76,6 @@ void* GPUTPCSliceData::SetPointersScratch(const GPUConstantMem& cm, void* mem) GPUProcessor::computePointerWithAlignment(mem, mLinkDownData, mNumberOfHitsPlusAlign); GPUProcessor::computePointerWithAlignment(mem, mHitWeights, mNumberOfHitsPlusAlign + 16 / sizeof(*mHitWeights)); size_t tmpMemMaxSize = CAMath::nextMultipleOf<4>(mNumberOfHits + 256) * (sizeof(float2) + sizeof(int) + sizeof(GPUTPCHit)); - int maxbiny, maxbinz; - GetMaxNBins(&cm, maxbiny, maxbinz); - size_t binMemoryMaxSize = (2 * (maxbiny * maxbinz + 3) + mNumberOfHits) * sizeof(calink) + GPUCA_ROW_COUNT * GPUCA_ROWALIGNMENT; - tmpMemMaxSize += binMemoryMaxSize; GPUProcessor::computePointerWithAlignment(mem, mTmpMem, (tmpMemMaxSize + sizeof(*mTmpMem) - 1) / sizeof(*mTmpMem)); return mem; } @@ -115,9 +111,9 @@ static inline float fastInvSqrt(float _x) return x.f; } -void GPUTPCSliceData::GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int& maxY, int& maxZ) +void GPUTPCSliceData::GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, int& maxY, int& maxZ) { - maxY = mRows[GPUCA_ROW_COUNT - 1].mMaxY * 2.f / GPUCA_MIN_BIN_SIZE + 1; + maxY = row->mMaxY * 2.f / GPUCA_MIN_BIN_SIZE + 1; maxZ = mem->param.continuousMaxTimeBin > 0 ? mem->calibObjects.fastTransform->convTimeToZinTimeFrame(0, 0, mem->param.continuousMaxTimeBin) + 50 : 300; maxZ = maxZ / GPUCA_MIN_BIN_SIZE + 1; } @@ -162,48 +158,12 @@ inline void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUC float sy = CAMath::Min(CAMath::Max((yMax - yMin) * norm, GPUCA_MIN_BIN_SIZE), GPUCA_MAX_BIN_SIZE); float sz = CAMath::Min(CAMath::Max(dz * norm, GPUCA_MIN_BIN_SIZE), GPUCA_MAX_BIN_SIZE); int maxy, maxz; - GetMaxNBins(mem, maxy, maxz); + GetMaxNBins(mem, row, maxy, maxz); int ny = CAMath::Max(1, CAMath::Min(maxy, (yMax - yMin) / sy + 1)); int nz = CAMath::Max(1, CAMath::Min(maxz, (zMax - zMin) / sz + 1)); row->mGrid.Create(yMin, yMax, zMin, zMax, ny, nz); } -inline int GPUTPCSliceData::PackHitData(GPUTPCRow* const GPUrestrict() row, const GPUTPCHit* GPUrestrict() binSortedHits) -{ - // hit data packing - static const float maxVal = (((long long int)1 << CAMath::Min((size_t)24, sizeof(cahit) * 8)) - 1); // Stay within float precision in any case! - static const float packingConstant = 1.f / (maxVal - 2.); - const float y0 = row->mGrid.YMin(); - const float z0 = row->mGrid.ZMin(); - const float stepY = (row->mGrid.YMax() - y0) * packingConstant; - const float stepZ = (row->mGrid.ZMax() - z0) * packingConstant; - const float stepYi = 1.f / stepY; - const float stepZi = 1.f / stepZ; - - row->mHy0 = y0; - row->mHz0 = z0; - row->mHstepY = stepY; - row->mHstepZ = stepZ; - row->mHstepYi = stepYi; - row->mHstepZi = stepZi; - - for (int hitIndex = 0; hitIndex < row->mNHits; ++hitIndex) { - // bin sorted index! - const int globalHitIndex = row->mHitNumberOffset + hitIndex; - const GPUTPCHit& hh = binSortedHits[hitIndex]; - const float xx = ((hh.Y() - y0) * stepYi) + .5; - const float yy = ((hh.Z() - z0) * stepZi) + .5; - if (xx < 0 || yy < 0 || xx > maxVal || yy > maxVal) { - std::cout << "!!!! hit packing error!!! " << xx << " " << yy << " (" << maxVal << ")" << std::endl; - return 1; - } - // HitData is bin sorted - mHitData[globalHitIndex].x = (cahit)xx; - mHitData[globalHitIndex].y = (cahit)yy; - } - return 0; -} - int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) { //////////////////////////////////// @@ -277,20 +237,14 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU // 2. fill HitData and FirstHitInBin //////////////////////////////////// - vecpod binSortedHits(mNumberOfHits + GPUCA_ROWALIGNMENT); - unsigned int gridContentOffset = 0; unsigned int hitOffset = 0; - unsigned int binCreationMemorySize = 103 * 2 + mNumberOfHits; - vecpod binCreationMemory(binCreationMemorySize); - for (int rowIndex = 0; rowIndex < GPUCA_ROW_COUNT; ++rowIndex) { GPUTPCRow& row = mRows[rowIndex]; if (NumberOfClustersInRow[rowIndex] == 0) { row.mGrid.CreateEmpty(); row.mNHits = 0; - row.mFullSize = 0; row.mHitNumberOffset = 0; row.mFirstHitInBinOffset = 0; row.mHy0 = 0.f; @@ -320,60 +274,65 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU return 1; } - int binCreationMemorySizeNew = numberOfBins * 2 + 6 + row.mNHits + GPUCA_ROWALIGNMENT / sizeof(unsigned short) * (GPUCA_ROW_COUNT + 1) + 1; - if (binCreationMemorySizeNew > binCreationMemorySize) { - binCreationMemorySize = binCreationMemorySizeNew; - binCreationMemory.resize(binCreationMemorySize); - } - - calink* c = binCreationMemory.data(); // number of hits in all previous bins - calink* bins = c + numberOfBins + 3; // cache for the bin index for every hit in this row, 3 extra empty bins at the end!!! - calink* filled = bins + row.mNHits; // counts how many hits there are per bin + calink* c = mFirstHitInBin + row.mFirstHitInBinOffset; // number of hits in all previous bins + calink* bins = mLinkUpData + RowOffset[rowIndex]; // Reuse mLinkUpData memory as temporary memory - for (unsigned int bin = 0; bin < row.mGrid.N() + 3; ++bin) { - filled[bin] = 0; // initialize filled[] to 0 + for (int bin = 0; bin < numberOfBins; ++bin) { + c[bin] = 0; // initialize filled[] to 0 } for (int hitIndex = 0; hitIndex < row.mNHits; ++hitIndex) { const int globalHitIndex = RowOffset[rowIndex] + hitIndex; const calink bin = row.mGrid.GetBin(YZData[globalHitIndex].x, YZData[globalHitIndex].y); bins[hitIndex] = bin; - ++filled[bin]; + ++c[bin]; } calink n = 0; - for (int bin = 0; bin < numberOfBins + 3; ++bin) { + for (int bin = 0; bin < numberOfBins; ++bin) { + n += c[bin]; + c[bin] = n; + } + for (int bin = numberOfBins; bin < nn; bin++) { c[bin] = n; - n += filled[bin]; } + static const float maxVal = (((long long int)1 << CAMath::Min((size_t)24, sizeof(cahit) * 8)) - 1); // Stay within float precision in any case! + static const float packingConstant = 1.f / (maxVal - 2.); + const float y0 = row.mGrid.YMin(); + const float z0 = row.mGrid.ZMin(); + const float stepY = (row.mGrid.YMax() - y0) * packingConstant; + const float stepZ = (row.mGrid.ZMax() - z0) * packingConstant; + const float stepYi = 1.f / stepY; + const float stepZi = 1.f / stepZ; + + row.mHy0 = y0; + row.mHz0 = z0; + row.mHstepY = stepY; + row.mHstepZ = stepZ; + row.mHstepYi = stepYi; + row.mHstepZi = stepZi; + for (int hitIndex = 0; hitIndex < row.mNHits; ++hitIndex) { const calink bin = bins[hitIndex]; - --filled[bin]; - const calink ind = c[bin] + filled[bin]; // generate an index for this hit that is >= c[bin] and < c[bin + 1] + const calink ind = --c[bin]; // generate an index for this hit that is >= c[bin] and < c[bin + 1] const int globalBinsortedIndex = row.mHitNumberOffset + ind; const int globalHitIndex = RowOffset[rowIndex] + hitIndex; // allows to find the global hit index / coordinates from a global bin sorted hit index mClusterDataIndex[globalBinsortedIndex] = tmpHitIndex[globalHitIndex]; - binSortedHits[ind].SetY(YZData[globalHitIndex].x); - binSortedHits[ind].SetZ(YZData[globalHitIndex].y); - } - if (PackHitData(&row, binSortedHits.data())) { - return 1; - } - - for (int i = 0; i < numberOfBins; ++i) { - mFirstHitInBin[row.mFirstHitInBinOffset + i] = c[i]; // global bin-sorted hit index - } - const calink a = c[numberOfBins]; - // grid.N is <= row.mNHits - for (int i = numberOfBins; i < nn; ++i) { - mFirstHitInBin[row.mFirstHitInBinOffset + i] = a; + const float xx = ((YZData[globalHitIndex].x - y0) * stepYi) + .5; + const float yy = ((YZData[globalHitIndex].y - z0) * stepZi) + .5; + if (xx < 0 || yy < 0 || xx > maxVal || yy > maxVal) { + std::cout << "!!!! hit packing error!!! " << xx << " " << yy << " (" << maxVal << ")" << std::endl; + return 1; + } + // HitData is bin sorted + mHitData[globalBinsortedIndex].x = (cahit)xx; + mHitData[globalBinsortedIndex].y = (cahit)yy; } - row.mFullSize = nn; gridContentOffset += nn; // Make pointer aligned diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index 2a62b951e5789..1072f89b67969 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -40,7 +40,7 @@ class GPUTPCSliceData unsigned int GetGridSize(); void SetClusterData(const GPUTPCClusterData* data, int nClusters, int clusterIdOffset); void* SetPointersInput(void* mem, bool idsOnGPU); - void* SetPointersScratch(GPUconstantref() const MEM_CONSTANT(GPUConstantMem)& cm, void* mem); + void* SetPointersScratch(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) & cm, void* mem); void* SetPointersScratchHost(void* mem, bool idsOnGPU); void* SetPointersRows(void* mem); #endif @@ -139,9 +139,8 @@ class GPUTPCSliceData GPUTPCSliceData& operator=(const GPUTPCSliceData&) CON_DELETE; // ROOT 5 tries to use this if it is not private GPUTPCSliceData(const GPUTPCSliceData&) CON_DELETE; // #endif - void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow)* row, const float2* data, int ClusterDataHitNumberOffset); - void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int& maxY, int& maxZ); - int PackHitData(MEM_GLOBAL(GPUTPCRow)* row, const GPUTPCHit* binSortedHits); + void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * row, const float2* data, int ClusterDataHitNumberOffset); + static void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, int& maxY, int& maxZ); friend class GPUTPCNeighboursFinder; friend class GPUTPCStartHitsFinder; @@ -160,9 +159,9 @@ class GPUTPCSliceData GPUglobalref() calink* mLinkDownData; // hit index in the row below which is linked to the given (global) hit index GPUglobalref() cahit2* mHitData; // packed y,z coordinate of the given (global) hit index GPUglobalref() int* mClusterDataIndex; // see ClusterDataIndex() - + uint4* mTmpMem; - + /* * The size of the array is row.Grid.N + row.Grid.Ny + 3. The row.Grid.Ny + 3 is an optimization * to remove the need for bounds checking. The last values are the same as the entry at [N - 1]. From 0a21b3a914536d0630c16aceb7bdc0bdb1da0e8a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 3 May 2020 23:11:48 +0200 Subject: [PATCH 04/22] GPU: Make TPC Grid creation compile (for now) on GPU, reuse available temp memory instead of allocating std::vectors, reshuffle to create row by row for later parallelization --- GPU/Common/GPUCommonMath.h | 14 ++ .../Base/GPUReconstructionIncludesDevice.h | 1 + GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx | 2 +- GPU/GPUTracking/SliceTracker/GPUTPCGrid.h | 2 +- .../SliceTracker/GPUTPCSliceData.cxx | 183 +++++++++--------- .../SliceTracker/GPUTPCSliceData.h | 10 +- 6 files changed, 116 insertions(+), 96 deletions(-) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index efc083aecf757..f31fc638f603b 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -46,6 +46,7 @@ class GPUCommonMath template GPUhd() static T MaxWithRef(T x, T y, S refX, S refY, S& r); GPUhdni() static float Sqrt(float x); + GPUhdni() static float FastInvSqrt(float x); template GPUhd() static T Abs(T x); GPUhdni() static float ASin(float x); @@ -240,6 +241,19 @@ GPUhdi() T GPUCommonMath::MaxWithRef(T x, T y, S refX, S refY, S& r) GPUhdi() float GPUCommonMath::Sqrt(float x) { return CHOICE(sqrtf(x), sqrtf(x), sqrt(x)); } +GPUhdi() float GPUCommonMath::FastInvSqrt(float _x) +{ + // the function calculates fast inverse sqrt + union { + float f; + int i; + } x = {_x}; + const float xhalf = 0.5f * x.f; + x.i = 0x5f3759df - (x.i >> 1); + x.f = x.f * (1.5f - xhalf * x.f * x.f); + return x.f; +} + template <> GPUhdi() float GPUCommonMath::Abs(float x) { diff --git a/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h b/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h index e571a7c40b10c..805cdab388567 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h +++ b/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h @@ -49,6 +49,7 @@ using namespace GPUCA_NAMESPACE::gpu; #include "GPUTPCGMTrackParam.cxx" #include "GPUTPCGMPhysicalTrackModel.cxx" #include "GPUTPCGMPropagator.cxx" +#include "GPUTPCSliceData.cxx" #if defined(HAVE_O2HEADERS) // Files for propagation with material diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx index 463035e467c40..5b3262825749f 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.cxx @@ -20,7 +20,7 @@ using namespace GPUCA_NAMESPACE::gpu; #endif MEM_CLASS_PRE() -void MEM_LG(GPUTPCGrid)::CreateEmpty() +GPUd() void MEM_LG(GPUTPCGrid)::CreateEmpty() { // Create an empty grid mYMin = 0.f; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h index 9268fefb40dda..fe23185a39bf6 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCGrid.h @@ -32,7 +32,7 @@ MEM_CLASS_PRE() class GPUTPCGrid { public: - void CreateEmpty(); + GPUd() void CreateEmpty(); GPUd() void Create(float yMin, float yMax, float zMin, float zMax, int ny, int nz); GPUd() int GetBin(float Y, float Z) const; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index f584c994e408d..7248615fa7c6e 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -20,9 +20,12 @@ #include "GPUO2DataTypes.h" #include "GPUTPCConvertImpl.h" #include "GPUCommonMath.h" + +#ifndef __OPENCL__ +#include "utils/vecpod.h" #include #include -#include "utils/vecpod.h" +#endif using namespace GPUCA_NAMESPACE::gpu; @@ -54,11 +57,6 @@ void GPUTPCSliceData::SetMaxData() mNumberOfHitsPlusAlign = GPUProcessor::nextMultipleOf<(kVectorAlignment > GPUCA_ROWALIGNMENT ? kVectorAlignment : GPUCA_ROWALIGNMENT) / sizeof(int)>(hitMemCount); } -unsigned int GPUTPCSliceData::GetGridSize() -{ - return (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; -} - void* GPUTPCSliceData::SetPointersInput(void* mem, bool idsOnGPU) { const int firstHitInBinSize = GetGridSize(); @@ -75,8 +73,6 @@ void* GPUTPCSliceData::SetPointersScratch(const GPUConstantMem& cm, void* mem) GPUProcessor::computePointerWithAlignment(mem, mLinkUpData, mNumberOfHitsPlusAlign); GPUProcessor::computePointerWithAlignment(mem, mLinkDownData, mNumberOfHitsPlusAlign); GPUProcessor::computePointerWithAlignment(mem, mHitWeights, mNumberOfHitsPlusAlign + 16 / sizeof(*mHitWeights)); - size_t tmpMemMaxSize = CAMath::nextMultipleOf<4>(mNumberOfHits + 256) * (sizeof(float2) + sizeof(int) + sizeof(GPUTPCHit)); - GPUProcessor::computePointerWithAlignment(mem, mTmpMem, (tmpMemMaxSize + sizeof(*mTmpMem) - 1) / sizeof(*mTmpMem)); return mem; } @@ -96,29 +92,19 @@ void* GPUTPCSliceData::SetPointersRows(void* mem) #endif -// calculates an approximation for 1/sqrt(x) -// Google for 0x5f3759df :) -static inline float fastInvSqrt(float _x) -{ - // the function calculates fast inverse sqrt - union { - float f; - int i; - } x = {_x}; - const float xhalf = 0.5f * x.f; - x.i = 0x5f3759df - (x.i >> 1); - x.f = x.f * (1.5f - xhalf * x.f * x.f); - return x.f; -} - -void GPUTPCSliceData::GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, int& maxY, int& maxZ) +GPUd() void GPUTPCSliceData::GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, int& maxY, int& maxZ) { maxY = row->mMaxY * 2.f / GPUCA_MIN_BIN_SIZE + 1; maxZ = mem->param.continuousMaxTimeBin > 0 ? mem->calibObjects.fastTransform->convTimeToZinTimeFrame(0, 0, mem->param.continuousMaxTimeBin) + 50 : 300; maxZ = maxZ / GPUCA_MIN_BIN_SIZE + 1; } -inline void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset) +GPUd() unsigned int GPUTPCSliceData::GetGridSize() +{ + return (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; +} + +GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset) { // grid creation if (row->NHits() <= 0) { // no hits or invalid data @@ -154,7 +140,7 @@ inline void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUC tfFactor = dz / 250.; dz = 250.; } - const float norm = fastInvSqrt(row->mNHits / tfFactor); + const float norm = CAMath::FastInvSqrt(row->mNHits / tfFactor); float sy = CAMath::Min(CAMath::Max((yMax - yMin) * norm, GPUCA_MIN_BIN_SIZE), GPUCA_MAX_BIN_SIZE); float sz = CAMath::Min(CAMath::Max(dz * norm, GPUCA_MIN_BIN_SIZE), GPUCA_MAX_BIN_SIZE); int maxy, maxz; @@ -164,85 +150,80 @@ inline void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUC row->mGrid.Create(yMin, yMax, zMin, zMax, ny, nz); } -int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) +GPUd() int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) { - //////////////////////////////////// - // 0. sort rows - //////////////////////////////////// - +#ifdef GPUCA_GPUCODE + constexpr bool EarlyTransformWithoutClusterNative = false; + if (mem->ioPtrs.clustersNative == nullptr) { + GPUError("Cluster Native Access Structure missing"); + return 1; + } +#else + bool EarlyTransformWithoutClusterNative = mem->param.earlyTpcTransform && mem->ioPtrs.clustersNative == nullptr; +#endif mMaxZ = 0.f; + int* tmpHitIndex = nullptr; + const unsigned int* NumberOfClustersInRow = nullptr; + const unsigned int* RowOffsets = nullptr; + +#ifndef GPUCA_GPUCODE std::unique_ptr YZData_p(new float2[mNumberOfHits]); - std::unique_ptr tmpHitIndex_p(new int[mNumberOfHits]); float2* YZData = YZData_p.get(); - int* tmpHitIndex = tmpHitIndex_p.get(); - - int RowOffset[GPUCA_ROW_COUNT]; - const unsigned int* NumberOfClustersInRow; + unsigned int RowOffsetsA[GPUCA_ROW_COUNT]; unsigned int NumberOfClustersInRowA[GPUCA_ROW_COUNT]; - if (mem->param.earlyTpcTransform) { + std::unique_ptr tmpHitIndex_p(new int[mNumberOfHits]); + if (EarlyTransformWithoutClusterNative) { // Implies mem->param.earlyTpcTransform but no ClusterNative present + NumberOfClustersInRow = NumberOfClustersInRowA; + RowOffsets = RowOffsetsA; + tmpHitIndex = tmpHitIndex_p.get(); + memset(NumberOfClustersInRowA, 0, GPUCA_ROW_COUNT * sizeof(NumberOfClustersInRowA[0])); for (int i = 0; i < mNumberOfHits; i++) { const int tmpRow = mClusterData[i].row; NumberOfClustersInRowA[tmpRow]++; } - NumberOfClustersInRow = NumberOfClustersInRowA; - } else { - NumberOfClustersInRow = &mem->ioPtrs.clustersNative->nClusters[iSlice][0]; - } - - int tmpOffset = 0; - for (int i = 0; i < GPUCA_ROW_COUNT; i++) { - if ((long long int)NumberOfClustersInRow[i] >= ((long long int)1 << (sizeof(calink) * 8))) { - GPUError("Too many clusters in row %d for row indexing (%d >= %lld), indexing insufficient", i, NumberOfClustersInRow[i], ((long long int)1 << (sizeof(calink) * 8))); - return 1; - } - if (NumberOfClustersInRow[i] >= (1 << 24)) { - GPUError("Too many clusters in row %d for hit id indexing (%d >= %d), indexing insufficient", i, NumberOfClustersInRow[i], 1 << 24); - return 1; + int tmpOffset = 0; + for (int i = 0; i < GPUCA_ROW_COUNT; i++) { + RowOffsetsA[i] = tmpOffset; + tmpOffset += NumberOfClustersInRow[i]; } - RowOffset[i] = tmpOffset; - tmpOffset += NumberOfClustersInRow[i]; - } - - if (mem->param.earlyTpcTransform) { int RowsFilled[GPUCA_ROW_COUNT]; memset(RowsFilled, 0, GPUCA_ROW_COUNT * sizeof(int)); for (int i = 0; i < mNumberOfHits; i++) { float2 tmp; tmp.x = mClusterData[i].y; tmp.y = mClusterData[i].z; - if (fabsf(tmp.y) > mMaxZ) { - mMaxZ = fabsf(tmp.y); + if (CAMath::Abs((float)tmp.y) > mMaxZ) { + mMaxZ = CAMath::Abs((float)tmp.y); } int tmpRow = mClusterData[i].row; - int newIndex = RowOffset[tmpRow] + (RowsFilled[tmpRow])++; + int newIndex = RowOffsetsA[tmpRow] + (RowsFilled[tmpRow])++; YZData[newIndex] = tmp; tmpHitIndex[newIndex] = i; } - } else { - size_t k = 0; - for (int i = 0; i < GPUCA_ROW_COUNT; i++) { - for (unsigned int j = 0; j < NumberOfClustersInRow[i]; j++) { - float2 tmp; - float x; - GPUTPCConvertImpl::convert(*mem, iSlice, i, mem->ioPtrs.clustersNative->clusters[iSlice][i][j].getPad(), mem->ioPtrs.clustersNative->clusters[iSlice][i][j].getTime(), x, tmp.x, tmp.y); - tmpHitIndex[k] = k; - YZData[k++] = tmp; - } - } - } - - //////////////////////////////////// - // 2. fill HitData and FirstHitInBin - //////////////////////////////////// - + } // Other cases below in loop over rows +#else + float2* YZData = (float2*)mLinkUpData; + static_assert(sizeof(*YZData) <= (sizeof(*mLinkUpData) + sizeof(*mLinkDownData)), "Cannot reuse memory"); +#endif unsigned int gridContentOffset = 0; unsigned int hitOffset = 0; for (int rowIndex = 0; rowIndex < GPUCA_ROW_COUNT; ++rowIndex) { + const unsigned int NumberOfClusters = EarlyTransformWithoutClusterNative ? NumberOfClustersInRow[rowIndex] : mem->ioPtrs.clustersNative->nClusters[iSlice][rowIndex]; + const unsigned int RowOffset = EarlyTransformWithoutClusterNative ? RowOffsets[rowIndex] : (mem->ioPtrs.clustersNative->clusterOffset[iSlice][rowIndex] - mem->ioPtrs.clustersNative->clusterOffset[iSlice][0]); + if ((long long int)NumberOfClusters >= ((long long int)1 << (sizeof(calink) * 8))) { + GPUError("Too many clusters in row %d for row indexing (%d >= %lld), indexing insufficient", rowIndex, NumberOfClusters, ((long long int)1 << (sizeof(calink) * 8))); + return 1; + } + if (NumberOfClusters >= (1 << 24)) { + GPUError("Too many clusters in row %d for hit id indexing (%d >= %d), indexing insufficient", rowIndex, NumberOfClusters, 1 << 24); + return 1; + } + GPUTPCRow& row = mRows[rowIndex]; - if (NumberOfClustersInRow[rowIndex] == 0) { + if (NumberOfClusters == 0) { row.mGrid.CreateEmpty(); row.mNHits = 0; row.mHitNumberOffset = 0; @@ -255,13 +236,36 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU row.mHstepZi = 1.f; continue; } - row.mNHits = NumberOfClustersInRow[rowIndex]; - row.mHitNumberOffset = hitOffset; - hitOffset += GPUProcessor::nextMultipleOf(NumberOfClustersInRow[rowIndex]); + if (!EarlyTransformWithoutClusterNative) { + if (mem->param.earlyTpcTransform) { // Early transform case with ClusterNative present + for (unsigned int i = 0; i < NumberOfClusters; i++) { + float2 tmp; + tmp.x = mClusterData[RowOffset + i].y; + tmp.y = mClusterData[RowOffset + i].z; + if (CAMath::Abs((float)tmp.y) > mMaxZ) { + mMaxZ = CAMath::Abs((float)tmp.y); + } + YZData[RowOffset + i] = tmp; + } + } else { + for (unsigned int i = 0; i < NumberOfClusters; i++) { + float x, y, z; + GPUTPCConvertImpl::convert(*mem, iSlice, rowIndex, mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getPad(), mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getTime(), x, y, z); + if (CAMath::Abs(z) > mMaxZ) { + mMaxZ = CAMath::Abs(z); + } + YZData[RowOffset + i] = CAMath::MakeFloat2(y, z); + } + } + } + + row.mNHits = NumberOfClusters; + row.mHitNumberOffset = hitOffset; row.mFirstHitInBinOffset = gridContentOffset; + hitOffset += CAMath::nextMultipleOf(NumberOfClusters); - CreateGrid(mem, &row, YZData, RowOffset[rowIndex]); + CreateGrid(mem, &row, YZData, RowOffset); const GPUTPCGrid& grid = row.mGrid; const int numberOfBins = grid.N(); if ((long long int)numberOfBins >= ((long long int)1 << (sizeof(calink) * 8))) { @@ -275,13 +279,14 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU } calink* c = mFirstHitInBin + row.mFirstHitInBinOffset; // number of hits in all previous bins - calink* bins = mLinkUpData + RowOffset[rowIndex]; // Reuse mLinkUpData memory as temporary memory + calink* bins = (calink*)mHitWeights + RowOffset; // Reuse mLinkUpData memory as temporary memory + static_assert(sizeof(*bins) <= sizeof(*mHitWeights), "Cannot reuse memory"); for (int bin = 0; bin < numberOfBins; ++bin) { c[bin] = 0; // initialize filled[] to 0 } for (int hitIndex = 0; hitIndex < row.mNHits; ++hitIndex) { - const int globalHitIndex = RowOffset[rowIndex] + hitIndex; + const int globalHitIndex = RowOffset + hitIndex; const calink bin = row.mGrid.GetBin(YZData[globalHitIndex].x, YZData[globalHitIndex].y); bins[hitIndex] = bin; @@ -297,8 +302,8 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU c[bin] = n; } - static const float maxVal = (((long long int)1 << CAMath::Min((size_t)24, sizeof(cahit) * 8)) - 1); // Stay within float precision in any case! - static const float packingConstant = 1.f / (maxVal - 2.); + constexpr float maxVal = (((long long int)1 << (sizeof(cahit) < 3 ? sizeof(cahit) * 8 : 24)) - 1); // Stay within float precision in any case! + constexpr float packingConstant = 1.f / (maxVal - 2.); const float y0 = row.mGrid.YMin(); const float z0 = row.mGrid.ZMin(); const float stepY = (row.mGrid.YMax() - y0) * packingConstant; @@ -317,17 +322,19 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU const calink bin = bins[hitIndex]; const calink ind = --c[bin]; // generate an index for this hit that is >= c[bin] and < c[bin + 1] const int globalBinsortedIndex = row.mHitNumberOffset + ind; - const int globalHitIndex = RowOffset[rowIndex] + hitIndex; + const int globalHitIndex = RowOffset + hitIndex; // allows to find the global hit index / coordinates from a global bin sorted hit index - mClusterDataIndex[globalBinsortedIndex] = tmpHitIndex[globalHitIndex]; + mClusterDataIndex[globalBinsortedIndex] = EarlyTransformWithoutClusterNative ? tmpHitIndex[globalHitIndex] : (RowOffset + hitIndex); const float xx = ((YZData[globalHitIndex].x - y0) * stepYi) + .5; const float yy = ((YZData[globalHitIndex].y - z0) * stepZi) + .5; +#if !defined(GPUCA_GPUCODE) && !defined(NDEBUG) if (xx < 0 || yy < 0 || xx > maxVal || yy > maxVal) { std::cout << "!!!! hit packing error!!! " << xx << " " << yy << " (" << maxVal << ")" << std::endl; return 1; } +#endif // HitData is bin sorted mHitData[globalBinsortedIndex].x = (cahit)xx; mHitData[globalBinsortedIndex].y = (cahit)yy; @@ -336,7 +343,7 @@ int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPU gridContentOffset += nn; // Make pointer aligned - gridContentOffset = GPUProcessor::nextMultipleOf(gridContentOffset); + gridContentOffset = CAMath::nextMultipleOf(gridContentOffset); } return 0; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index 1072f89b67969..4396b08955722 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -37,7 +37,6 @@ class GPUTPCSliceData ~GPUTPCSliceData() CON_DEFAULT; void InitializeRows(const MEM_CONSTANT(GPUParam) & p); void SetMaxData(); - unsigned int GetGridSize(); void SetClusterData(const GPUTPCClusterData* data, int nClusters, int clusterIdOffset); void* SetPointersInput(void* mem, bool idsOnGPU); void* SetPointersScratch(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) & cm, void* mem); @@ -45,7 +44,7 @@ class GPUTPCSliceData void* SetPointersRows(void* mem); #endif - int InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int iSlice); + GPUd() int InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int iSlice); /** * Return the number of hits in this slice. @@ -139,8 +138,9 @@ class GPUTPCSliceData GPUTPCSliceData& operator=(const GPUTPCSliceData&) CON_DELETE; // ROOT 5 tries to use this if it is not private GPUTPCSliceData(const GPUTPCSliceData&) CON_DELETE; // #endif - void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * row, const float2* data, int ClusterDataHitNumberOffset); - static void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, int& maxY, int& maxZ); + GPUd() void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, const float2* data, int ClusterDataHitNumberOffset); + GPUd() static void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, int& maxY, int& maxZ); + GPUd() unsigned int GetGridSize(); friend class GPUTPCNeighboursFinder; friend class GPUTPCStartHitsFinder; @@ -160,8 +160,6 @@ class GPUTPCSliceData GPUglobalref() cahit2* mHitData; // packed y,z coordinate of the given (global) hit index GPUglobalref() int* mClusterDataIndex; // see ClusterDataIndex() - uint4* mTmpMem; - /* * The size of the array is row.Grid.N + row.Grid.Ny + 3. The row.Grid.Ny + 3 is an optimization * to remove the need for bounds checking. The last values are the same as the entry at [N - 1]. From 07ec32815d04016682b80ed8f7d785f6376c585e Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 4 May 2020 11:04:19 +0200 Subject: [PATCH 05/22] GPU: Run TPC Grid creation as a kernel --- GPU/Common/GPUDefGPUParameters.h | 5 +++ GPU/GPUTracking/Base/GPUReconstructionCPU.h | 1 + .../Base/GPUReconstructionKernels.h | 1 + GPU/GPUTracking/CMakeLists.txt | 1 + GPU/GPUTracking/Global/GPUChainTracking.cxx | 9 +--- GPU/GPUTracking/Global/GPUChainTracking.h | 2 +- .../SliceTracker/GPUTPCCreateSliceData.cxx | 27 ++++++++++++ .../SliceTracker/GPUTPCCreateSliceData.h | 44 +++++++++++++++++++ .../SliceTracker/GPUTPCSliceData.cxx | 21 +++++---- .../SliceTracker/GPUTPCSliceData.h | 7 +-- .../SliceTracker/GPUTPCTracker.cxx | 14 ------ GPU/GPUTracking/SliceTracker/GPUTPCTracker.h | 1 - 12 files changed, 97 insertions(+), 36 deletions(-) create mode 100644 GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx create mode 100644 GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h diff --git a/GPU/Common/GPUDefGPUParameters.h b/GPU/Common/GPUDefGPUParameters.h index 4c1fbbd018848..c37af7b75ed2b 100644 --- a/GPU/Common/GPUDefGPUParameters.h +++ b/GPU/Common/GPUDefGPUParameters.h @@ -30,6 +30,7 @@ #if defined(GPUCA_GPUTYPE_VEGA) #define GPUCA_WARP_SIZE 64 #define GPUCA_THREAD_COUNT 256 + #define GPUCA_LB_GPUTPCCreateSliceData 256 #define GPUCA_LB_GPUTPCStartHitsSorter 1024, 2 #define GPUCA_LB_GPUTPCStartHitsFinder 1024 #define GPUCA_LB_GPUTPCTrackletConstructor 512, 1 @@ -73,6 +74,7 @@ #elif defined(GPUCA_GPUTYPE_TURING) #define GPUCA_WARP_SIZE 32 #define GPUCA_THREAD_COUNT 512 + #define GPUCA_LB_GPUTPCCreateSliceData 256 #define GPUCA_LB_GPUTPCStartHitsSorter 512, 1 #define GPUCA_LB_GPUTPCStartHitsFinder 512 #define GPUCA_LB_GPUTPCTrackletConstructor 384, 1 @@ -125,6 +127,9 @@ #ifndef GPUCA_THREAD_COUNT #define GPUCA_THREAD_COUNT 256 #endif + #ifndef GPUCA_LB_GPUTPCCreateSliceData + #define GPUCA_LB_GPUTPCCreateSliceData 256 + #endif #ifndef GPUCA_LB_GPUTPCTrackletConstructor #define GPUCA_LB_GPUTPCTrackletConstructor 256 #endif diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.h b/GPU/GPUTracking/Base/GPUReconstructionCPU.h index fa761ba29836c..d16bb26360e3e 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.h +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.h @@ -22,6 +22,7 @@ #include #include "GPUGeneralKernels.h" +#include "GPUTPCCreateSliceData.h" #include "GPUTPCNeighboursFinder.h" #include "GPUTPCNeighboursCleaner.h" #include "GPUTPCStartHitsFinder.h" diff --git a/GPU/GPUTracking/Base/GPUReconstructionKernels.h b/GPU/GPUTracking/Base/GPUReconstructionKernels.h index c404a3211128c..a47dd121e8245 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionKernels.h +++ b/GPU/GPUTracking/Base/GPUReconstructionKernels.h @@ -25,6 +25,7 @@ GPUCA_KRNL((GPUTPCTrackletSelector ), (both, REG, (GPUCA_L GPUCA_KRNL((GPUMemClean16 ), (simple, REG, (GPUCA_THREAD_COUNT, 1)), (, GPUPtr1(void*, ptr), unsigned long size), (, GPUPtr2(void*, ptr), size)) GPUCA_KRNL((GPUTPCGlobalTrackingCopyNumbers ), (single), (, int n), (, n)) #if !defined(GPUCA_OPENCL1) && (!defined(GPUCA_ALIROOT_LIB) || !defined(GPUCA_GPUCODE)) +GPUCA_KRNL((GPUTPCCreateSliceData ), (single, REG, (GPUCA_LB_GPUTPCCreateSliceData)), (), ()) GPUCA_KRNL((GPUTPCGlobalTracking ), (single, REG, (GPUCA_LB_GPUTPCGlobalTracking)), (), ()) GPUCA_KRNL((GPUTPCGMMergerTrackFit ), (simple, REG, (GPUCA_LB_GPUTPCGMMergerTrackFit)), (, int mode), (, mode)) GPUCA_KRNL((GPUTPCGMMergerFollowLoopers ), (simple, REG, (GPUCA_LB_GPUTPCGMMergerFollowLoopers)), (), ()) diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt index 8eca2fa514a08..eb25ba06f6632 100644 --- a/GPU/GPUTracking/CMakeLists.txt +++ b/GPU/GPUTracking/CMakeLists.txt @@ -45,6 +45,7 @@ set(SRCS SliceTracker/GPUTPCTrackletSelector.cxx SliceTracker/GPUTPCRow.cxx SliceTracker/GPUTPCGlobalTracking.cxx + SliceTracker/GPUTPCCreateSliceData.cxx Merger/GPUTPCGMMerger.cxx Merger/GPUTPCGMSliceTrack.cxx Merger/GPUTPCGMTrackParam.cxx diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index bd0bdf7e85d9a..7a261e334b7d4 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -713,17 +713,12 @@ void GPUChainTracking::SetTRDGeometry(std::unique_ptr& processors()->calibObjects.trdGeometry = mTRDGeometryU.get(); } -int GPUChainTracking::ReadEvent(int iSlice, int threadId) +int GPUChainTracking::ReadEvent(unsigned int iSlice, int threadId) { if (GetDeviceProcessingSettings().debugLevel >= 5) { GPUInfo("Running ReadEvent for slice %d on thread %d\n", iSlice, threadId); } - HighResTimer& timer = getTimer("ReadEvent", threadId); - timer.Start(); - if (processors()->tpcTrackers[iSlice].ReadEvent()) { - return (1); - } - timer.Stop(); + runKernel({GetGridBlk(1, 0, GPUReconstruction::krnlDeviceType::CPU)}, {iSlice}); if (GetDeviceProcessingSettings().debugLevel >= 5) { GPUInfo("Finished ReadEvent for slice %d on thread %d\n", iSlice, threadId); } diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 9c69fd007ea16..565f131c677e5 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -177,7 +177,7 @@ class GPUChainTracking : public GPUChain, GPUReconstructionHelpers::helperDelega GPUChainTracking(GPUReconstruction* rec, unsigned int maxTPCHits = GPUCA_MAX_CLUSTERS, unsigned int maxTRDTracklets = GPUCA_MAX_TRD_TRACKLETS); - int ReadEvent(int iSlice, int threadId); + int ReadEvent(unsigned int iSlice, int threadId); void WriteOutput(int iSlice, int threadId); int GlobalTracking(unsigned int iSlice, int threadId); void PrepareEventFromNative(); diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx new file mode 100644 index 0000000000000..a0314e1f5ff51 --- /dev/null +++ b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx @@ -0,0 +1,27 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUTPCCreateSliceData.cxx +/// \author David Rohr + +#include "GPUTPCCreateSliceData.h" +#include "GPUTPCTracker.h" +#include "GPUCommonMath.h" + +using namespace GPUCA_NAMESPACE::gpu; + +template <> +GPUdii() void GPUTPCCreateSliceData::Thread<0>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() MEM_LOCAL(GPUSharedMemory) & GPUrestrict() s, processorType& GPUrestrict() tracker) +{ + if (iThread || iBlock) { + return; + } + tracker.Data().InitFromClusterData(nBlocks, nThreads, iBlock, iThread, tracker.GetConstantMem(), tracker.ISlice()); +} diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h new file mode 100644 index 0000000000000..dc7f97fb4775c --- /dev/null +++ b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h @@ -0,0 +1,44 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUTPCCreateSliceData.h +/// \author David Rohr + +#ifndef GPUTPCCREATESLICEDATA_H +#define GPUTPCCREATESLICEDATA_H + +#include "GPUTPCDef.h" +#include "GPUTPCHitId.h" +#include "GPUGeneralKernels.h" +#include "GPUConstantMem.h" + +namespace GPUCA_NAMESPACE +{ +namespace gpu +{ +class GPUTPCTracker; + +class GPUTPCCreateSliceData : public GPUKernelTemplate +{ + public: + typedef GPUconstantref() MEM_GLOBAL(GPUTPCTracker) processorType; + GPUhdi() CONSTEXPRRET static GPUDataTypes::RecoStep GetRecoStep() { return GPUCA_RECO_STEP::TPCSliceTracking; } + MEM_TEMPLATE() + GPUhdi() static processorType* Processor(MEM_TYPE(GPUConstantMem) & processors) + { + return processors.tpcTrackers; + } + template + GPUd() static void Thread(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() MEM_LOCAL(GPUSharedMemory) & smem, processorType& tracker); +}; +} // namespace gpu +} // namespace GPUCA_NAMESPACE + +#endif // GPUTPCCREATESLICEDATA_H diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index 7248615fa7c6e..ac098d1e023a3 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -150,7 +150,7 @@ GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPU row->mGrid.Create(yMin, yMax, zMin, zMax, ny, nz); } -GPUd() int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) +GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int iBlock, int iThread, GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) { #ifdef GPUCA_GPUCODE constexpr bool EarlyTransformWithoutClusterNative = false; @@ -161,7 +161,7 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONST #else bool EarlyTransformWithoutClusterNative = mem->param.earlyTpcTransform && mem->ioPtrs.clustersNative == nullptr; #endif - mMaxZ = 0.f; + float maxAbsZ = 0.f; int* tmpHitIndex = nullptr; const unsigned int* NumberOfClustersInRow = nullptr; @@ -194,8 +194,8 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONST float2 tmp; tmp.x = mClusterData[i].y; tmp.y = mClusterData[i].z; - if (CAMath::Abs((float)tmp.y) > mMaxZ) { - mMaxZ = CAMath::Abs((float)tmp.y); + if (CAMath::Abs((float)tmp.y) > maxAbsZ) { + maxAbsZ = CAMath::Abs((float)tmp.y); } int tmpRow = mClusterData[i].row; int newIndex = RowOffsetsA[tmpRow] + (RowsFilled[tmpRow])++; @@ -243,8 +243,8 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONST float2 tmp; tmp.x = mClusterData[RowOffset + i].y; tmp.y = mClusterData[RowOffset + i].z; - if (CAMath::Abs((float)tmp.y) > mMaxZ) { - mMaxZ = CAMath::Abs((float)tmp.y); + if (CAMath::Abs((float)tmp.y) > maxAbsZ) { + maxAbsZ = CAMath::Abs((float)tmp.y); } YZData[RowOffset + i] = tmp; } @@ -252,8 +252,8 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONST for (unsigned int i = 0; i < NumberOfClusters; i++) { float x, y, z; GPUTPCConvertImpl::convert(*mem, iSlice, rowIndex, mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getPad(), mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getTime(), x, y, z); - if (CAMath::Abs(z) > mMaxZ) { - mMaxZ = CAMath::Abs(z); + if (CAMath::Abs(z) > maxAbsZ) { + maxAbsZ = CAMath::Abs(z); } YZData[RowOffset + i] = CAMath::MakeFloat2(y, z); } @@ -346,5 +346,10 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(GPUconstantref() const MEM_CONST gridContentOffset = CAMath::nextMultipleOf(gridContentOffset); } + if (maxAbsZ > 300 && mem->param.ContinuousTracking) { + GPUError("Need to set continuous tracking mode for data outside of the TPC volume!"); // TODO: Set GPU error code + return 1; + } + return 0; } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index 4396b08955722..32600d5fc98f1 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -31,7 +31,7 @@ MEM_CLASS_PRE() class GPUTPCSliceData { public: - GPUTPCSliceData() : mNumberOfHits(0), mNumberOfHitsPlusAlign(0), mClusterIdOffset(0), mMaxZ(0.f), mGPUTextureBase(nullptr), mRows(nullptr), mLinkUpData(nullptr), mLinkDownData(nullptr), mClusterData(nullptr) {} + GPUTPCSliceData() : mNumberOfHits(0), mNumberOfHitsPlusAlign(0), mClusterIdOffset(0), mGPUTextureBase(nullptr), mRows(nullptr), mLinkUpData(nullptr), mLinkDownData(nullptr), mClusterData(nullptr) {} #ifndef GPUCA_GPUCODE_DEVICE ~GPUTPCSliceData() CON_DEFAULT; @@ -44,7 +44,7 @@ class GPUTPCSliceData void* SetPointersRows(void* mem); #endif - GPUd() int InitFromClusterData(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int iSlice); + GPUd() int InitFromClusterData(int nBlocks, int nThreads, int iBlock, int iThread, GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int iSlice); /** * Return the number of hits in this slice. @@ -131,7 +131,6 @@ class GPUTPCSliceData GPUhdi() char* GPUTextureBaseConst() const { return ((char*)mGPUTextureBase); } GPUhdi() GPUglobalref() const GPUTPCClusterData* ClusterData() const { return mClusterData; } - float MaxZ() const { return mMaxZ; } private: #ifndef GPUCA_GPUCODE @@ -149,8 +148,6 @@ class GPUTPCSliceData int mNumberOfHitsPlusAlign; int mClusterIdOffset; - float mMaxZ; - GPUglobalref() const void* mGPUTextureBase; // pointer to start of GPU texture GPUglobalref() MEM_GLOBAL(GPUTPCRow) * mRows; // The row objects needed for most accessor functions diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx index 1d13826350598..fb3762af41895 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.cxx @@ -158,20 +158,6 @@ void GPUTPCTracker::UpdateMaxData() void GPUTPCTracker::SetupCommonMemory() { new (mCommonMem) commonMemoryStruct; } -int GPUTPCTracker::ReadEvent() -{ - //* Convert input hits, create grids, etc. - if (mData.InitFromClusterData(mConstantMem, mISlice)) { - GPUError("Error initializing from cluster data"); - return 1; - } - if (mData.MaxZ() > 300 && !Param().ContinuousTracking) { - GPUError("Need to set continuous tracking mode for data outside of the TPC volume!"); - return 1; - } - return 0; -} - GPUh() int GPUTPCTracker::CheckEmptySlice() { // Check if the Slice is empty, if so set the output apropriate and tell the reconstuct procesdure to terminate diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h index 86cc77675ffaf..66dfaed24b31d 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTracker.h @@ -58,7 +58,6 @@ class GPUTPCTracker : public GPUProcessor MEM_CLASS_PRE2() void InitializeRows(const MEM_CONSTANT(GPUParam) * param) { mData.InitializeRows(*param); } - int ReadEvent(); int CheckEmptySlice(); void WriteOutputPrepare(); void WriteOutput(); From 0d84f076d252cba7aba4ee45546e57a5963c9d13 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 4 May 2020 12:13:04 +0200 Subject: [PATCH 06/22] GPU: Avoid additional pass over data to determine TPC grid dimension --- .../SliceTracker/GPUTPCSliceData.cxx | 83 +++++++++---------- .../SliceTracker/GPUTPCSliceData.h | 2 +- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index ac098d1e023a3..dca13119720ac 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -104,36 +104,8 @@ GPUd() unsigned int GPUTPCSliceData::GetGridSize() return (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; } -GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset) +GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset, float yMin, float yMax, float zMin, float zMax) { - // grid creation - if (row->NHits() <= 0) { // no hits or invalid data - // grid coordinates don't matter, since there are no hits - row->mGrid.CreateEmpty(); - return; - } - - float yMin = 1.e6f; - float yMax = -1.e6f; - float zMin = 1.e6f; - float zMax = -1.e6f; - for (int i = ClusterDataHitNumberOffset; i < ClusterDataHitNumberOffset + row->mNHits; ++i) { - const float y = data[i].x; - const float z = data[i].y; - if (yMax < y) { - yMax = y; - } - if (yMin > y) { - yMin = y; - } - if (zMax < z) { - zMax = z; - } - if (zMin > z) { - zMin = z; - } - } - float dz = zMax - zMin; float tfFactor = 1.; if (dz > 270.) { @@ -150,6 +122,22 @@ GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPU row->mGrid.Create(yMin, yMax, zMin, zMax, ny, nz); } +GPUdi() static void UpdateMinMaxYZ(float& yMin, float& yMax, float& zMin, float& zMax, float y, float z) +{ + if (yMax < y) { + yMax = y; + } + if (yMin > y) { + yMin = y; + } + if (zMax < z) { + zMax = z; + } + if (zMin > z) { + zMin = z; + } +} + GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int iBlock, int iThread, GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) { #ifdef GPUCA_GPUCODE @@ -161,8 +149,6 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i #else bool EarlyTransformWithoutClusterNative = mem->param.earlyTpcTransform && mem->ioPtrs.clustersNative == nullptr; #endif - float maxAbsZ = 0.f; - int* tmpHitIndex = nullptr; const unsigned int* NumberOfClustersInRow = nullptr; const unsigned int* RowOffsets = nullptr; @@ -172,6 +158,7 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i float2* YZData = YZData_p.get(); unsigned int RowOffsetsA[GPUCA_ROW_COUNT]; unsigned int NumberOfClustersInRowA[GPUCA_ROW_COUNT]; + std::unique_ptr tmpHitIndex_p(new int[mNumberOfHits]); if (EarlyTransformWithoutClusterNative) { // Implies mem->param.earlyTpcTransform but no ClusterNative present NumberOfClustersInRow = NumberOfClustersInRowA; @@ -194,9 +181,6 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i float2 tmp; tmp.x = mClusterData[i].y; tmp.y = mClusterData[i].z; - if (CAMath::Abs((float)tmp.y) > maxAbsZ) { - maxAbsZ = CAMath::Abs((float)tmp.y); - } int tmpRow = mClusterData[i].row; int newIndex = RowOffsetsA[tmpRow] + (RowsFilled[tmpRow])++; YZData[newIndex] = tmp; @@ -211,6 +195,12 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i unsigned int hitOffset = 0; for (int rowIndex = 0; rowIndex < GPUCA_ROW_COUNT; ++rowIndex) { + + float yMin = 1.e6f; + float yMax = -1.e6f; + float zMin = 1.e6f; + float zMax = -1.e6f; + const unsigned int NumberOfClusters = EarlyTransformWithoutClusterNative ? NumberOfClustersInRow[rowIndex] : mem->ioPtrs.clustersNative->nClusters[iSlice][rowIndex]; const unsigned int RowOffset = EarlyTransformWithoutClusterNative ? RowOffsets[rowIndex] : (mem->ioPtrs.clustersNative->clusterOffset[iSlice][rowIndex] - mem->ioPtrs.clustersNative->clusterOffset[iSlice][0]); if ((long long int)NumberOfClusters >= ((long long int)1 << (sizeof(calink) * 8))) { @@ -237,24 +227,24 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i continue; } - if (!EarlyTransformWithoutClusterNative) { + if (EarlyTransformWithoutClusterNative) { + for (unsigned int i = 0; i < NumberOfClusters; i++) { + UpdateMinMaxYZ(yMin, yMax, zMin, zMax, YZData[RowOffset - i].x, YZData[RowOffset - i].y); + } + } else { if (mem->param.earlyTpcTransform) { // Early transform case with ClusterNative present for (unsigned int i = 0; i < NumberOfClusters; i++) { float2 tmp; tmp.x = mClusterData[RowOffset + i].y; tmp.y = mClusterData[RowOffset + i].z; - if (CAMath::Abs((float)tmp.y) > maxAbsZ) { - maxAbsZ = CAMath::Abs((float)tmp.y); - } + UpdateMinMaxYZ(yMin, yMax, zMin, zMax, tmp.x, tmp.y); YZData[RowOffset + i] = tmp; } } else { for (unsigned int i = 0; i < NumberOfClusters; i++) { float x, y, z; GPUTPCConvertImpl::convert(*mem, iSlice, rowIndex, mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getPad(), mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getTime(), x, y, z); - if (CAMath::Abs(z) > maxAbsZ) { - maxAbsZ = CAMath::Abs(z); - } + UpdateMinMaxYZ(yMin, yMax, zMin, zMax, y, z); YZData[RowOffset + i] = CAMath::MakeFloat2(y, z); } } @@ -265,7 +255,7 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i row.mFirstHitInBinOffset = gridContentOffset; hitOffset += CAMath::nextMultipleOf(NumberOfClusters); - CreateGrid(mem, &row, YZData, RowOffset); + CreateGrid(mem, &row, YZData, RowOffset, yMin, yMax, zMin, zMax); const GPUTPCGrid& grid = row.mGrid; const int numberOfBins = grid.N(); if ((long long int)numberOfBins >= ((long long int)1 << (sizeof(calink) * 8))) { @@ -344,11 +334,12 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i // Make pointer aligned gridContentOffset = CAMath::nextMultipleOf(gridContentOffset); - } - if (maxAbsZ > 300 && mem->param.ContinuousTracking) { - GPUError("Need to set continuous tracking mode for data outside of the TPC volume!"); // TODO: Set GPU error code - return 1; + const float maxAbsZ = CAMath::Max(CAMath::Abs(zMin), CAMath::Abs(zMax)); + if (maxAbsZ > 300 && !mem->param.ContinuousTracking) { + GPUError("Need to set continuous tracking mode for data outside of the TPC volume!"); // TODO: Set GPU error code + return 1; + } } return 0; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index 32600d5fc98f1..5cef738ff887c 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -137,7 +137,7 @@ class GPUTPCSliceData GPUTPCSliceData& operator=(const GPUTPCSliceData&) CON_DELETE; // ROOT 5 tries to use this if it is not private GPUTPCSliceData(const GPUTPCSliceData&) CON_DELETE; // #endif - GPUd() void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, const float2* data, int ClusterDataHitNumberOffset); + GPUd() void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, const float2* data, int ClusterDataHitNumberOffset, float yMin, float yMax, float zMin, float zMax); GPUd() static void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, int& maxY, int& maxZ); GPUd() unsigned int GetGridSize(); From 9328112e894643346e756163bf664f173d470145 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 4 May 2020 12:28:01 +0200 Subject: [PATCH 07/22] GPU: Compute all offsets absolutely during TPC grid creation --- .../SliceTracker/GPUTPCSliceData.cxx | 25 ++++++------------- .../SliceTracker/GPUTPCSliceData.h | 2 +- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index dca13119720ac..87b586bb3bc3d 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -59,7 +59,7 @@ void GPUTPCSliceData::SetMaxData() void* GPUTPCSliceData::SetPointersInput(void* mem, bool idsOnGPU) { - const int firstHitInBinSize = GetGridSize(); + const int firstHitInBinSize = GetGridSize(mNumberOfHits, GPUCA_ROW_COUNT) + GPUCA_ROW_COUNT * GPUCA_ROWALIGNMENT / sizeof(int); GPUProcessor::computePointerWithAlignment(mem, mHitData, mNumberOfHitsPlusAlign); GPUProcessor::computePointerWithAlignment(mem, mFirstHitInBin, firstHitInBinSize); if (idsOnGPU) { @@ -99,9 +99,9 @@ GPUd() void GPUTPCSliceData::GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPU maxZ = maxZ / GPUCA_MIN_BIN_SIZE + 1; } -GPUd() unsigned int GPUTPCSliceData::GetGridSize() +GPUd() unsigned int GPUTPCSliceData::GetGridSize(unsigned int nHits, unsigned int nRows) { - return (23 + GPUCA_ROWALIGNMENT / sizeof(int)) * GPUCA_ROW_COUNT + 4 * mNumberOfHits + 3; + return 26 * nRows + 4 * nHits; } GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset, float yMin, float yMax, float zMin, float zMax) @@ -191,11 +191,8 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i float2* YZData = (float2*)mLinkUpData; static_assert(sizeof(*YZData) <= (sizeof(*mLinkUpData) + sizeof(*mLinkDownData)), "Cannot reuse memory"); #endif - unsigned int gridContentOffset = 0; - unsigned int hitOffset = 0; for (int rowIndex = 0; rowIndex < GPUCA_ROW_COUNT; ++rowIndex) { - float yMin = 1.e6f; float yMax = -1.e6f; float zMin = 1.e6f; @@ -251,9 +248,8 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i } row.mNHits = NumberOfClusters; - row.mHitNumberOffset = hitOffset; - row.mFirstHitInBinOffset = gridContentOffset; - hitOffset += CAMath::nextMultipleOf(NumberOfClusters); + row.mHitNumberOffset = CAMath::nextMultipleOf(RowOffset + rowIndex * GPUCA_ROWALIGNMENT / sizeof(calink)); + row.mFirstHitInBinOffset = CAMath::nextMultipleOf(GetGridSize(RowOffset, rowIndex) + rowIndex * GPUCA_ROWALIGNMENT / sizeof(int)); CreateGrid(mem, &row, YZData, RowOffset, yMin, yMax, zMin, zMax); const GPUTPCGrid& grid = row.mGrid; @@ -262,8 +258,8 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i GPUError("Too many bins in row %d for grid (%d >= %lld), indexing insufficient", rowIndex, numberOfBins, ((long long int)1 << (sizeof(calink) * 8))); return 1; } - const int nn = numberOfBins + grid.Ny() + 3; - if (gridContentOffset + nn >= GetGridSize()) { + const unsigned int nn = numberOfBins + grid.Ny() + 3; + if (nn >= GetGridSize(NumberOfClusters, 1)) { GPUError("firstHitInBin overflow"); return 1; } @@ -288,7 +284,7 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i n += c[bin]; c[bin] = n; } - for (int bin = numberOfBins; bin < nn; bin++) { + for (unsigned int bin = numberOfBins; bin < nn; bin++) { c[bin] = n; } @@ -330,11 +326,6 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i mHitData[globalBinsortedIndex].y = (cahit)yy; } - gridContentOffset += nn; - - // Make pointer aligned - gridContentOffset = CAMath::nextMultipleOf(gridContentOffset); - const float maxAbsZ = CAMath::Max(CAMath::Abs(zMin), CAMath::Abs(zMax)); if (maxAbsZ > 300 && !mem->param.ContinuousTracking) { GPUError("Need to set continuous tracking mode for data outside of the TPC volume!"); // TODO: Set GPU error code diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index 5cef738ff887c..f22d1e59c3016 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -139,7 +139,7 @@ class GPUTPCSliceData #endif GPUd() void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, const float2* data, int ClusterDataHitNumberOffset, float yMin, float yMax, float zMin, float zMax); GPUd() static void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, int& maxY, int& maxZ); - GPUd() unsigned int GetGridSize(); + GPUd() unsigned int GetGridSize(unsigned int nHits, unsigned int nRows); friend class GPUTPCNeighboursFinder; friend class GPUTPCStartHitsFinder; From ee9c0dc0c757c7b9fc55cf17ce09092b8fb3e5e7 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 4 May 2020 13:05:35 +0200 Subject: [PATCH 08/22] GPU: Template atomic functions --- GPU/Common/GPUCommonDefAPI.h | 2 + GPU/Common/GPUCommonMath.h | 48 +++++++++++++++---- .../Base/GPUReconstructionIncludesDevice.h | 1 + .../GPUTPCCompressionKernels.cxx | 4 +- GPU/GPUTracking/ITS/GPUITSFitterKernels.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 16 +++---- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 6 +-- .../SliceTracker/GPUTPCGlobalTracking.cxx | 4 +- .../SliceTracker/GPUTPCSliceData.h | 8 ++-- .../SliceTracker/GPUTPCStartHitsFinder.cxx | 8 ++-- .../GPUTPCTrackletConstructor.cxx | 6 +-- .../SliceTracker/GPUTPCTrackletSelector.cxx | 6 +-- .../TPCClusterFinder/GPUTPCCFClusterizer.cxx | 2 +- .../TPCClusterFinder/GPUTPCCFDecodeZS.cxx | 2 +- 14 files changed, 75 insertions(+), 40 deletions(-) diff --git a/GPU/Common/GPUCommonDefAPI.h b/GPU/Common/GPUCommonDefAPI.h index edee215b0a12e..ea80086792ff1 100644 --- a/GPU/Common/GPUCommonDefAPI.h +++ b/GPU/Common/GPUCommonDefAPI.h @@ -107,6 +107,8 @@ #define GPUCA_USE_TEMPLATE_ADDRESS_SPACES // TODO: check if we can make this (partially, where it is already implemented) compatible with OpenCL CPP #define GPUsharedref() GPUshared() #define GPUglobalref() GPUglobal() + #undef GPUgeneric() + #define GPUgeneric() #endif #if (!defined(__OPENCLCPP__) || !defined(GPUCA_NO_CONSTANT_MEMORY)) #define GPUconstantref() GPUconstant() diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index f31fc638f603b..89b42311f86e4 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -66,14 +66,46 @@ class GPUCommonMath GPUhdni() static unsigned int Popcount(unsigned int val); GPUhdni() static float Log(float x); - GPUdi() static unsigned int AtomicExch(GPUglobalref() GPUAtomic(unsigned int) * addr, unsigned int val) { return GPUCommonMath::AtomicExchInt(addr, val); } - GPUdi() static unsigned int AtomicAdd(GPUglobalref() GPUAtomic(unsigned int) * addr, unsigned int val) { return GPUCommonMath::AtomicAddInt(addr, val); } - GPUdi() static void AtomicMax(GPUglobalref() GPUAtomic(unsigned int) * addr, unsigned int val) { GPUCommonMath::AtomicMaxInt(addr, val); } - GPUdi() static void AtomicMin(GPUglobalref() GPUAtomic(unsigned int) * addr, unsigned int val) { GPUCommonMath::AtomicMinInt(addr, val); } - GPUdi() static unsigned int AtomicExchShared(GPUsharedref() GPUAtomic(unsigned int) * addr, unsigned int val) { return GPUCommonMath::AtomicExchInt(addr, val); } - GPUdi() static unsigned int AtomicAddShared(GPUsharedref() GPUAtomic(unsigned int) * addr, unsigned int val) { return GPUCommonMath::AtomicAddInt(addr, val); } - GPUdi() static void AtomicMaxShared(GPUsharedref() GPUAtomic(unsigned int) * addr, unsigned int val) { GPUCommonMath::AtomicMaxInt(addr, val); } - GPUdi() static void AtomicMinShared(GPUsharedref() GPUAtomic(unsigned int) * addr, unsigned int val) { GPUCommonMath::AtomicMinInt(addr, val); } + template + GPUdi() static T AtomicExch(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + return GPUCommonMath::AtomicExchInt(addr, val); + } + template + GPUdi() static T AtomicAdd(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + return GPUCommonMath::AtomicAddInt(addr, val); + } + template + GPUdi() static void AtomicMax(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + GPUCommonMath::AtomicMaxInt(addr, val); + } + template + GPUdi() static void AtomicMin(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + GPUCommonMath::AtomicMinInt(addr, val); + } + template + GPUdi() static T AtomicExchShared(GPUsharedref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + return GPUCommonMath::AtomicExchInt(addr, val); + } + template + GPUdi() static T AtomicAddShared(GPUsharedref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + return GPUCommonMath::AtomicAddInt(addr, val); + } + template + GPUdi() static void AtomicMaxShared(GPUsharedref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + GPUCommonMath::AtomicMaxInt(addr, val); + } + template + GPUdi() static void AtomicMinShared(GPUsharedref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + GPUCommonMath::AtomicMinInt(addr, val); + } GPUd() static int Mul24(int a, int b); GPUd() static float FMulRZ(float a, float b); diff --git a/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h b/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h index 805cdab388567..c278e057fdefc 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h +++ b/GPU/GPUTracking/Base/GPUReconstructionIncludesDevice.h @@ -50,6 +50,7 @@ using namespace GPUCA_NAMESPACE::gpu; #include "GPUTPCGMPhysicalTrackModel.cxx" #include "GPUTPCGMPropagator.cxx" #include "GPUTPCSliceData.cxx" +#include "GPUTPCCreateSliceData.cxx" #if defined(HAVE_O2HEADERS) // Files for propagation with material diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index 80d0d05b439d8..263fd03627542 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -39,7 +39,7 @@ GPUdii() void GPUTPCCompressionKernels::Thread processors.param.rec.tpcRejectQPt; - int nClustersStored = 0; + unsigned int nClustersStored = 0; CompressedClustersPtrsOnly& GPUrestrict() c = compressor.mPtrs; unsigned int lastRow = 0, lastSlice = 0; // BUG: These should be unsigned char, but then CUDA breaks GPUTPCCompressionTrackModel track; @@ -86,7 +86,7 @@ GPUdii() void GPUTPCCompressionKernels::Thread 0 ? 254 : 0); track.Init(x, y, z, param.SliceParam[hit.slice].Alpha, qpt, param); // TODO: Compression track model must respect Z offset! - myTrack = CAMath::AtomicAdd(&compressor.mMemory->nStoredTracks, 1); + myTrack = CAMath::AtomicAdd(&compressor.mMemory->nStoredTracks, 1u); compressor.mAttachedClusterFirstIndex[myTrack] = trk.FirstClusterRef(); lastLeg = hit.leg; c.qPtA[myTrack] = qpt; diff --git a/GPU/GPUTracking/ITS/GPUITSFitterKernels.cxx b/GPU/GPUTracking/ITS/GPUITSFitterKernels.cxx index 2d304455dcc97..9aa51d047b13f 100644 --- a/GPU/GPUTracking/ITS/GPUITSFitterKernels.cxx +++ b/GPU/GPUTracking/ITS/GPUITSFitterKernels.cxx @@ -189,7 +189,7 @@ GPUdii() void GPUITSFitterKernel::Thread<0>(int nBlocks, int nThreads, int iBloc continue; } CA_DEBUGGER(refitCounters[nClusters - 4]++); - int trackId = CAMath::AtomicAdd(&Fitter.NumberOfTracks(), 1); + int trackId = CAMath::AtomicAdd(&Fitter.NumberOfTracks(), 1u); Fitter.tracks()[trackId] = temporaryTrack; } #ifdef CA_DEBUG diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 6b06c43208932..198fbc85ae5e4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -439,7 +439,7 @@ GPUd() void GPUTPCGMMerger::UnpackSliceGlobal(int nBlocks, int nThreads, int iBl if (localId == -1) { continue; } - unsigned int myTrack = CAMath::AtomicAdd(&mMemory->nUnpackedTracks, 1); + unsigned int myTrack = CAMath::AtomicAdd(&mMemory->nUnpackedTracks, 1u); GPUTPCGMSliceTrack& track = mSliceTrackInfos[myTrack]; SetTrackClusterZT(track, iSlice, sliceTr); track.Set(this, sliceTr, alpha, iSlice); @@ -504,7 +504,7 @@ GPUd() void GPUTPCGMMerger::RefitSliceTracks(int nBlocks, int nThreads, int iBlo track.SetPrevSegmentNeighbour(-1); track.SetGlobalTrackId(0, -1); track.SetGlobalTrackId(1, -1); - unsigned int myTrack = CAMath::AtomicAdd(&mMemory->nUnpackedTracks, 1); + unsigned int myTrack = CAMath::AtomicAdd(&mMemory->nUnpackedTracks, 1u); TrackIds[iSlice * mNMaxSingleSliceTracks + sliceTr->LocalTrackId()] = myTrack; mSliceTrackInfos[myTrack] = track; } @@ -586,7 +586,7 @@ GPUd() void GPUTPCGMMerger::MakeBorderTracks(int nBlocks, int nThreads, int iBlo if (CAMath::Abs(b.Cov()[4]) >= 0.5) { b.SetCov(4, 0.5); } - unsigned int myTrack = CAMath::AtomicAdd(&nB[iSlice], 1); + unsigned int myTrack = CAMath::AtomicAdd(&nB[iSlice], 1u); B[iSlice][myTrack] = b; } } @@ -807,7 +807,7 @@ GPUd() void GPUTPCGMMerger::MergeWithinSlicesPrepare(int nBlocks, int nThreads, CADEBUG( printf("WITHIN SLICE %d Track %d - ", iSlice, itr); for (int i = 0; i < 5; i++) { printf("%8.3f ", b.Par()[i]); } printf(" - "); for (int i = 0; i < 5; i++) { printf("%8.3f ", b.Cov()[i]); } printf("\n")); b.SetNClusters(track.NClusters()); - unsigned int myTrack = CAMath::AtomicAdd(&mTmpCounter[iSlice], 1); + unsigned int myTrack = CAMath::AtomicAdd(&mTmpCounter[iSlice], 1u); mBorder[iSlice][myTrack] = b; } } @@ -1032,7 +1032,7 @@ GPUd() void GPUTPCGMMerger::MergeCEFill(const GPUTPCGMSliceTrack* track, const G } b.SetRow(cls.row); unsigned int id = slice + attempt * NSLICES; - unsigned int myTrack = CAMath::AtomicAdd(&mTmpCounter[id], 1); + unsigned int myTrack = CAMath::AtomicAdd(&mTmpCounter[id], 1u); mBorder[id][myTrack] = b; break; } @@ -1524,7 +1524,7 @@ GPUd() void GPUTPCGMMerger::SortTracksPrepare(int nBlocks, int nThreads, int iBl for (unsigned int i = iBlock * nThreads + iThread; i < mMemory->nOutputTracks; i += nThreads * nBlocks) { const GPUTPCGMMergedTrack& trk = mOutputTracks[i]; if (trk.CCE() || trk.Legs()) { - CAMath::AtomicAdd(&mMemory->nSlowTracks, 1); + CAMath::AtomicAdd(&mMemory->nSlowTracks, 1u); } mTrackOrderProcess[i] = i; } @@ -1558,7 +1558,7 @@ GPUd() void GPUTPCGMMerger::PrepareClustersForFit1(int nBlocks, int nThreads, in } for (unsigned int i = iBlock * nThreads + iThread; i < mMemory->nOutputTrackClusters; i += nBlocks * nThreads) { mClusterAttachment[mClusters[i].num] = attachAttached | attachGood; - CAMath::AtomicAdd(&sharedCount[mClusters[i].num], 1); + CAMath::AtomicAdd(&sharedCount[mClusters[i].num], 1u); } } @@ -1599,7 +1599,7 @@ GPUd() void GPUTPCGMMerger::Finalize1(int nBlocks, int nThreads, int iBlock, int char goodLeg = mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].leg; for (unsigned int j = 0; j < trk.NClusters(); j++) { int id = mClusters[trk.FirstClusterRef() + j].num; - int weight = mTrackOrderAttach[i] | attachAttached; + unsigned int weight = mTrackOrderAttach[i] | attachAttached; unsigned char clusterState = mClusters[trk.FirstClusterRef() + j].state; if (!(clusterState & GPUTPCGMMergedTrackHit::flagReject)) { weight |= attachGood; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 978aece2ab163..f7e13ecb2227b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -545,9 +545,9 @@ GPUd() bool GPUTPCGMTrackParam::FollowCircleChk(float lrFactor, float toY, float GPUd() void GPUTPCGMTrackParam::StoreAttachMirror(const GPUTPCGMMerger* GPUrestrict() Merger, int slice, int iRow, int iTrack, bool goodLeg, float toAlpha, float toY, float toX, int toSlice, int toRow, bool inFlyDirection, float alpha) { - unsigned int nLoopData = CAMath::AtomicAdd(&Merger->Memory()->nLoopData, 1); + unsigned int nLoopData = CAMath::AtomicAdd(&Merger->Memory()->nLoopData, 1u); if (nLoopData >= Merger->NMaxTracks()) { - CAMath::AtomicExch(&Merger->Memory()->nLoopData, 0); + CAMath::AtomicExch(&Merger->Memory()->nLoopData, 0u); // TODO: GPUCA_ERROR_LOOPER_OVERFLOW return; } @@ -962,7 +962,7 @@ GPUd() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() tr Alpha = track.Alpha(); ok = t.Fit(merger, iTrk, clusters + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, 1, GPUCA_MAX_SIN_PHI, &track.OuterParam(), merger->Param().dodEdx ? &track.dEdxInfo() : nullptr); } else { - unsigned int nRefit = CAMath::AtomicAdd(&merger->Memory()->nRetryRefit, 1); + unsigned int nRefit = CAMath::AtomicAdd(&merger->Memory()->nRetryRefit, 1u); merger->RetryRefitIds()[nRefit] = iTrk; return; } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCGlobalTracking.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCGlobalTracking.cxx index 4158b01422df9..931d6e06a7de9 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCGlobalTracking.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCGlobalTracking.cxx @@ -72,13 +72,13 @@ GPUd() int GPUTPCGlobalTracking::PerformGlobalTrackingRun(GPUTPCTracker& tracker int nHits = GPUTPCTrackletConstructor::GPUTPCTrackletConstructorGlobalTracking(tracker, smem, tParam, rowIndex, direction, 0, rowHits); if (nHits >= GPUCA_GLOBAL_TRACKING_MIN_HITS) { // GPUInfo("%d hits found", nHits); - unsigned int hitId = CAMath::AtomicAdd(&tracker.CommonMemory()->nTrackHits, nHits); + unsigned int hitId = CAMath::AtomicAdd(&tracker.CommonMemory()->nTrackHits, (unsigned int)nHits); if (hitId + nHits > tracker.NMaxTrackHits()) { tracker.CommonMemory()->kernelError = GPUCA_ERROR_GLOBAL_TRACKING_TRACK_HIT_OVERFLOW; CAMath::AtomicExch(&tracker.CommonMemory()->nTrackHits, tracker.NMaxTrackHits()); return (0); } - unsigned int trackId = CAMath::AtomicAdd(&tracker.CommonMemory()->nTracks, 1); + unsigned int trackId = CAMath::AtomicAdd(&tracker.CommonMemory()->nTracks, 1u); if (direction == 1) { int i = 0; while (i < nHits) { diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index f22d1e59c3016..c247b1e611638 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -101,9 +101,9 @@ class GPUTPCSliceData * If the given weight is higher than what is currently stored replace with the new weight. */ MEM_TEMPLATE() - GPUd() void MaximizeHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, int weight); + GPUd() void MaximizeHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, unsigned int weight); MEM_TEMPLATE() - GPUd() void SetHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, int weight); + GPUd() void SetHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, unsigned int weight); /** * Return the maximal weight the given hit got from one tracklet @@ -210,14 +210,14 @@ GPUhdi() int MEM_LG(GPUTPCSliceData)::ClusterDataIndex(const MEM_TYPE(GPUTPCRow) MEM_CLASS_PRE() MEM_TEMPLATE() -GPUdi() void MEM_LG(GPUTPCSliceData)::MaximizeHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, int weight) +GPUdi() void MEM_LG(GPUTPCSliceData)::MaximizeHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, unsigned int weight) { CAMath::AtomicMax(&mHitWeights[row.mHitNumberOffset + hitIndex], weight); } MEM_CLASS_PRE() MEM_TEMPLATE() -GPUdi() void MEM_LG(GPUTPCSliceData)::SetHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, int weight) +GPUdi() void MEM_LG(GPUTPCSliceData)::SetHitWeight(const MEM_TYPE(GPUTPCRow) & row, unsigned int hitIndex, unsigned int weight) { mHitWeights[row.mHitNumberOffset + hitIndex] = weight; } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCStartHitsFinder.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCStartHitsFinder.cxx index f37b73a2ee2f3..785ff3cd5e585 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCStartHitsFinder.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCStartHitsFinder.cxx @@ -42,18 +42,18 @@ GPUdii() void GPUTPCStartHitsFinder::Thread<0>(int /*nBlocks*/, int nThreads, in if (tracker.mData.mLinkDownData[lHitNumberOffset + ih] == CALINK_INVAL && linkUpData != CALINK_INVAL && tracker.mData.mLinkUpData[rowUp.mHitNumberOffset + linkUpData] != CALINK_INVAL) { #ifdef GPUCA_SORT_STARTHITS GPUglobalref() GPUTPCHitId* const GPUrestrict() startHits = tracker.mTrackletTmpStartHits + s.mIRow * tracker.mNMaxRowStartHits; - unsigned int nextRowStartHits = CAMath::AtomicAddShared(&s.mNRowStartHits, 1); + unsigned int nextRowStartHits = CAMath::AtomicAddShared(&s.mNRowStartHits, 1u); CONSTEXPR int errCode = GPUCA_ERROR_ROWSTARTHIT_OVERFLOW; if (nextRowStartHits >= tracker.mNMaxRowStartHits) #else GPUglobalref() GPUTPCHitId* const GPUrestrict() startHits = tracker.mTrackletStartHits; - unsigned int nextRowStartHits = CAMath::AtomicAdd(&tracker.mCommonMem->nStartHits, 1); + unsigned int nextRowStartHits = CAMath::AtomicAdd(&tracker.mCommonMem->nStartHits, 1u); CONSTEXPR int errCode = GPUCA_ERROR_STARTHIT_OVERFLOW; if (nextRowStartHits >= tracker.mNMaxStartHits) #endif { trackerX.CommonMemory()->kernelError = errCode; - CAMath::AtomicExch(&tracker.mCommonMem->nStartHits, 0); + CAMath::AtomicExch(&tracker.mCommonMem->nStartHits, 0u); break; } startHits[nextRowStartHits].Set(s.mIRow, ih); @@ -67,7 +67,7 @@ GPUdii() void GPUTPCStartHitsFinder::Thread<0>(int /*nBlocks*/, int nThreads, in tracker.mRowStartHitCountOffset[s.mIRow] = s.mNRowStartHits; if (nOffset + s.mNRowStartHits > tracker.mNMaxStartHits) { trackerX.CommonMemory()->kernelError = GPUCA_ERROR_STARTHIT_OVERFLOW; - CAMath::AtomicExch(&tracker.mCommonMem->nStartHits, 0); + CAMath::AtomicExch(&tracker.mCommonMem->nStartHits, 0u); } } #endif diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx index bf1f572e054ed..8fae5bff79598 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx @@ -65,12 +65,12 @@ GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int /*nBlocks*/, int /*nThr tParam.Cov()[0], tParam.Cov()[1], tParam.Cov()[2], tParam.Cov()[3], tParam.Cov()[4], tParam.Cov()[5], tParam.Cov()[6], tParam.Cov()[7], tParam.Cov()[8], tParam.Cov()[9], tParam.Cov()[10], tParam.Cov()[11], tParam.Cov()[12], tParam.Cov()[13], tParam.Cov()[14]);*/ - unsigned int itrout = CAMath::AtomicAdd(tracker.NTracklets(), 1); + unsigned int itrout = CAMath::AtomicAdd(tracker.NTracklets(), 1u); const unsigned int nHits = r.mLastRow + 1 - r.mFirstRow; unsigned int hitout = CAMath::AtomicAdd(tracker.NRowHits(), nHits); if (itrout >= tracker.NMaxTracklets() || hitout + nHits > tracker.NMaxRowHits()) { tracker.CommonMemory()->kernelError = itrout >= tracker.NMaxTracklets() ? GPUCA_ERROR_TRACKLET_OVERFLOW : GPUCA_ERROR_TRACKLET_HIT_OVERFLOW; - CAMath::AtomicExch(tracker.NTracklets(), 0); + CAMath::AtomicExch(tracker.NTracklets(), 0u); if (hitout + nHits > tracker.NMaxRowHits()) { CAMath::AtomicExch(tracker.NRowHits(), tracker.NMaxRowHits()); } @@ -488,7 +488,7 @@ GPUd() int GPUTPCTrackletConstructor::FetchTracklet(GPUconstantref() MEM_GLOBAL( sMem.mNextStartHitFirstRun = 0; } else { if (tracker.GPUParameters()->nextStartHit < nStartHit) { - firstStartHit = CAMath::AtomicAdd(&tracker.GPUParameters()->nextStartHit, GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCTrackletConstructor)); + firstStartHit = CAMath::AtomicAdd(&tracker.GPUParameters()->nextStartHit, GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCTrackletConstructor)); } } sMem.mNextStartHitFirst = firstStartHit < (int)nStartHit ? firstStartHit : -2; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletSelector.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletSelector.cxx index 382b0b00af8f6..c5c084ff7e525 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletSelector.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletSelector.cxx @@ -84,11 +84,11 @@ GPUdii() void GPUTPCTrackletSelector::Thread<0>(int nBlocks, int nThreads, int i if (gap > kMaxRowGap || irow == lastRow) { // store if (nHits >= minHits) { - unsigned int itrout = CAMath::AtomicAdd(tracker.NTracks(), 1); - unsigned int nFirstTrackHit = CAMath::AtomicAdd(tracker.NTrackHits(), nHits); + unsigned int itrout = CAMath::AtomicAdd(tracker.NTracks(), 1u); + unsigned int nFirstTrackHit = CAMath::AtomicAdd(tracker.NTrackHits(), (unsigned int)nHits); if (itrout >= tracker.NMaxTracks() || nFirstTrackHit + nHits > tracker.NMaxTrackHits()) { tracker.CommonMemory()->kernelError = (itrout >= tracker.NMaxTracks()) ? GPUCA_ERROR_TRACK_OVERFLOW : GPUCA_ERROR_TRACK_HIT_OVERFLOW; - CAMath::AtomicExch(tracker.NTracks(), 0); + CAMath::AtomicExch(tracker.NTracks(), 0u); if (nFirstTrackHit + nHits > tracker.NMaxTrackHits()) { CAMath::AtomicExch(tracker.NTrackHits(), tracker.NMaxTrackHits()); } diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx index e201b9a94a3eb..dc05ea7764f51 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx @@ -364,7 +364,7 @@ GPUd() void GPUTPCCFClusterizer::buildClusterNaive( GPUd() uint GPUTPCCFClusterizer::sortIntoBuckets(const tpc::ClusterNative& cluster, uint row, uint maxElemsPerBucket, uint* elemsInBucket, tpc::ClusterNative* buckets) { - uint index = CAMath::AtomicAdd(&elemsInBucket[row], 1); + uint index = CAMath::AtomicAdd(&elemsInBucket[row], 1u); if (index < maxElemsPerBucket) { buckets[maxElemsPerBucket * row + index] = cluster; } diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx index ae4a06cf40f93..e9cafdcbdcf57 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx @@ -104,7 +104,7 @@ GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUShared GPUbarrier(); for (int n = iThread; n < nRowsUsed; n += nThreads) { const unsigned char* rowData = n == 0 ? pagePtr : (page + tbHdr->rowAddr1()[n - 1]); - s.RowClusterOffset[n] = CAMath::AtomicAddShared(&s.rowOffsetCounter, rowData[2 * *rowData]); + s.RowClusterOffset[n] = CAMath::AtomicAddShared(&s.rowOffsetCounter, rowData[2 * *rowData]); } /*if (iThread < GPUCA_WARP_SIZE) { // TODO: Seems to miscompile with HIP, CUDA performance doesn't really change, for now sticking to the AtomicAdd GPUSharedMemory& smem = s; From 43277025f3f9dca8fafdc0b168804bf4ab1feb2f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 4 May 2020 17:03:27 +0200 Subject: [PATCH 09/22] GPU: Add bandwidth measurements for kernels doing DMA transfer via Zero-Copy --- GPU/GPUTracking/Base/GPUReconstructionCPU.cxx | 8 +- GPU/GPUTracking/Base/GPUReconstructionCPU.h | 8 +- GPU/GPUTracking/Global/GPUChain.h | 4 +- GPU/GPUTracking/Global/GPUChainTracking.cxx | 84 ++++++++++--------- 4 files changed, 60 insertions(+), 44 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx index 79148a32dbe73..4f1661c01dd58 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx @@ -226,7 +226,11 @@ int GPUReconstructionCPU::RunChains() kernelStepTimes[stepNum] += time; } type = type == 0 ? 'K' : 'C'; - printf("Execution Time: Task (%c %8ux): %50s Time: %'10d us\n", type, count, mTimers[i]->name.c_str(), (int)(time * 1000000 / mStatNEvents)); + char bandwidth[256] = ""; + if (mTimers[i]->memSize && mStatNEvents && time != 0.) { + snprintf(bandwidth, 256, " (%6.3f GB/s - %'14lu bytes)", mTimers[i]->memSize / time * 1e-9, (unsigned long)(mTimers[i]->memSize / mStatNEvents)); + } + printf("Execution Time: Task (%c %8ux): %50s Time: %'10d us%s\n", type, count, mTimers[i]->name.c_str(), (int)(time * 1000000 / mStatNEvents), bandwidth); } for (int i = 0; i < N_RECO_STEPS; i++) { if (kernelStepTimes[i] != 0.) { @@ -295,7 +299,7 @@ GPUReconstructionCPU::timerMeta* GPUReconstructionCPU::insertTimer(unsigned int if (J >= 0) { name += std::to_string(J); } - mTimers[id].reset(new timerMeta{std::unique_ptr{new HighResTimer[num]}, name, num, type, 1u, step}); + mTimers[id].reset(new timerMeta{std::unique_ptr{new HighResTimer[num]}, name, num, type, 1u, step, (size_t)0}); } timerMeta* retVal = mTimers[id].get(); timerFlag.clear(); diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.h b/GPU/GPUTracking/Base/GPUReconstructionCPU.h index d16bb26360e3e..84ca2c986a7cd 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.h +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.h @@ -199,6 +199,7 @@ class GPUReconstructionCPU : public GPUReconstructionKernels - HighResTimer& getKernelTimer(RecoStep step, int num = 0); + HighResTimer& getKernelTimer(RecoStep step, int num = 0, size_t addMemorySize = 0); template HighResTimer& getTimer(const char* name, int num = -1); int getRecoStepNum(RecoStep step, bool validCheck = true); @@ -308,7 +309,7 @@ inline void GPUReconstructionCPU::AddGPUEvents(T*& events) } template -HighResTimer& GPUReconstructionCPU::getKernelTimer(RecoStep step, int num) +HighResTimer& GPUReconstructionCPU::getKernelTimer(RecoStep step, int num, size_t addMemorySize) { static int id = getNextTimerId(); timerMeta* timer = getTimerById(id); @@ -316,6 +317,9 @@ HighResTimer& GPUReconstructionCPU::getKernelTimer(RecoStep step, int num) int max = step == GPUCA_RECO_STEP::NoRecoStep || step == GPUCA_RECO_STEP::TPCSliceTracking || step == GPUCA_RECO_STEP::TPCClusterFinding ? NSLICES : 1; timer = insertTimer(id, GetKernelName(), J, max, 0, step); } + if (addMemorySize) { + timer->memSize += addMemorySize; + } if (num < 0 || num >= timer->num) { throw std::runtime_error("Invalid timer requested"); } diff --git a/GPU/GPUTracking/Global/GPUChain.h b/GPU/GPUTracking/Global/GPUChain.h index 20c8662937508..1c78976f32692 100644 --- a/GPU/GPUTracking/Global/GPUChain.h +++ b/GPU/GPUTracking/Global/GPUChain.h @@ -170,9 +170,9 @@ class GPUChain } template - HighResTimer& getKernelTimer(int num = 0) + HighResTimer& getKernelTimer(RecoStep step, int num = 0, size_t addMemorySize = 0) { - return mRec->getKernelTimer(num); + return mRec->getKernelTimer(step, num, addMemorySize); } template HighResTimer& getTimer(const char* name, int num = -1) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 7a261e334b7d4..75546da460766 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1686,66 +1686,74 @@ int GPUChainTracking::RunTPCCompression() runKernel(GetGridBlk(BlockCount(), 0), krnlRunRangeNone, krnlEventNone); TransferMemoryResourcesToHost(myStep, &Compressor, 0); SynchronizeGPU(); - memset((void*)Compressor.mOutput, 0, sizeof(*Compressor.mOutput)); - Compressor.mOutput->nTracks = Compressor.mMemory->nStoredTracks; - Compressor.mOutput->nAttachedClusters = Compressor.mMemory->nStoredAttachedClusters; - Compressor.mOutput->nUnattachedClusters = Compressor.mMemory->nStoredUnattachedClusters; - Compressor.mOutput->nAttachedClustersReduced = Compressor.mOutput->nAttachedClusters - Compressor.mOutput->nTracks; - Compressor.mOutput->nSliceRows = NSLICES * GPUCA_ROW_COUNT; - Compressor.mOutput->nComppressionModes = param().rec.tpcCompressionModes; + o2::tpc::CompressedClusters* O = Compressor.mOutput; + memset((void*)O, 0, sizeof(*O)); + O->nTracks = Compressor.mMemory->nStoredTracks; + O->nAttachedClusters = Compressor.mMemory->nStoredAttachedClusters; + O->nUnattachedClusters = Compressor.mMemory->nStoredUnattachedClusters; + O->nAttachedClustersReduced = O->nAttachedClusters - O->nTracks; + O->nSliceRows = NSLICES * GPUCA_ROW_COUNT; + O->nComppressionModes = param().rec.tpcCompressionModes; AllocateRegisteredMemory(Compressor.mMemoryResOutputHost, &mRec->OutputControl()); - const GPUTPCCompression* outputSrc = nullptr; + const o2::tpc::CompressedClustersPtrsOnly* P = nullptr; if (DeviceProcessingSettings().tpcCompressionGatherMode == 2) { TransferMemoryResourcesToGPU(myStep, &Compressor, 0); runKernel(GetGridBlk(BlockCount(), 0), krnlRunRangeNone, krnlEventNone); + getKernelTimer(RecoStep::TPCCompression, 0, + NSLICES * GPUCA_ROW_COUNT * sizeof(O->nSliceRowClusters[0]) + + O->nTracks * sizeof(O->nTrackClusters[0]) + + O->nUnattachedClusters * (sizeof(O->qTotU[0]) + sizeof(O->qMaxU[0]) + sizeof(O->flagsU[0]) + sizeof(O->padDiffU[0]) + sizeof(O->timeDiffU[0]) + sizeof(O->sigmaPadU[0]) + sizeof(O->sigmaTimeU[0])) + + O->nAttachedClusters * (sizeof(O->qTotA[0]) + sizeof(O->qMaxA[0]) + sizeof(O->flagsA[0]) + sizeof(O->sigmaPadA[0]) + sizeof(O->sigmaTimeA[0])) + + O->nAttachedClustersReduced * (sizeof(O->rowDiffA[0]) + sizeof(O->sliceLegDiffA[0]) + sizeof(O->padResA[0]) + sizeof(O->timeResA[0])) + + O->nTracks * (sizeof(O->qPtA[0]) + sizeof(O->rowA[0]) + sizeof(O->sliceA[0]) + sizeof(O->timeA[0]) + sizeof(O->padA[0]))); } else { char direction = 0; if (DeviceProcessingSettings().tpcCompressionGatherMode == 0) { - outputSrc = &CompressorShadow; + P = &CompressorShadow.mPtrs; } else if (DeviceProcessingSettings().tpcCompressionGatherMode == 1) { - outputSrc = &Compressor; + P = &Compressor.mPtrs; direction = -1; } - GPUMemCpyAlways(myStep, Compressor.mOutput->nSliceRowClusters, outputSrc->mPtrs.nSliceRowClusters, NSLICES * GPUCA_ROW_COUNT * sizeof(Compressor.mOutput->nSliceRowClusters[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->nTrackClusters, outputSrc->mPtrs.nTrackClusters, Compressor.mOutput->nTracks * sizeof(Compressor.mOutput->nTrackClusters[0]), 0, direction); + GPUMemCpyAlways(myStep, O->nSliceRowClusters, P->nSliceRowClusters, NSLICES * GPUCA_ROW_COUNT * sizeof(O->nSliceRowClusters[0]), 0, direction); + GPUMemCpyAlways(myStep, O->nTrackClusters, P->nTrackClusters, O->nTracks * sizeof(O->nTrackClusters[0]), 0, direction); SynchronizeGPU(); unsigned int offset = 0; for (unsigned int i = 0; i < NSLICES; i++) { for (unsigned int j = 0; j < GPUCA_ROW_COUNT; j++) { - GPUMemCpyAlways(myStep, Compressor.mOutput->qTotU + offset, outputSrc->mPtrs.qTotU + mClusterNativeAccess->clusterOffset[i][j], Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(Compressor.mOutput->qTotU[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->qMaxU + offset, outputSrc->mPtrs.qMaxU + mClusterNativeAccess->clusterOffset[i][j], Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(Compressor.mOutput->qMaxU[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->flagsU + offset, outputSrc->mPtrs.flagsU + mClusterNativeAccess->clusterOffset[i][j], Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(Compressor.mOutput->flagsU[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->padDiffU + offset, outputSrc->mPtrs.padDiffU + mClusterNativeAccess->clusterOffset[i][j], Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(Compressor.mOutput->padDiffU[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->timeDiffU + offset, outputSrc->mPtrs.timeDiffU + mClusterNativeAccess->clusterOffset[i][j], Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(Compressor.mOutput->timeDiffU[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->sigmaPadU + offset, outputSrc->mPtrs.sigmaPadU + mClusterNativeAccess->clusterOffset[i][j], Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(Compressor.mOutput->sigmaPadU[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->sigmaTimeU + offset, outputSrc->mPtrs.sigmaTimeU + mClusterNativeAccess->clusterOffset[i][j], Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(Compressor.mOutput->sigmaTimeU[0]), 0, direction); - offset += Compressor.mOutput->nSliceRowClusters[i * GPUCA_ROW_COUNT + j]; + GPUMemCpyAlways(myStep, O->qTotU + offset, P->qTotU + mClusterNativeAccess->clusterOffset[i][j], O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->qTotU[0]), 0, direction); + GPUMemCpyAlways(myStep, O->qMaxU + offset, P->qMaxU + mClusterNativeAccess->clusterOffset[i][j], O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->qMaxU[0]), 0, direction); + GPUMemCpyAlways(myStep, O->flagsU + offset, P->flagsU + mClusterNativeAccess->clusterOffset[i][j], O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->flagsU[0]), 0, direction); + GPUMemCpyAlways(myStep, O->padDiffU + offset, P->padDiffU + mClusterNativeAccess->clusterOffset[i][j], O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->padDiffU[0]), 0, direction); + GPUMemCpyAlways(myStep, O->timeDiffU + offset, P->timeDiffU + mClusterNativeAccess->clusterOffset[i][j], O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->timeDiffU[0]), 0, direction); + GPUMemCpyAlways(myStep, O->sigmaPadU + offset, P->sigmaPadU + mClusterNativeAccess->clusterOffset[i][j], O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->sigmaPadU[0]), 0, direction); + GPUMemCpyAlways(myStep, O->sigmaTimeU + offset, P->sigmaTimeU + mClusterNativeAccess->clusterOffset[i][j], O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->sigmaTimeU[0]), 0, direction); + offset += O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j]; } } offset = 0; - for (unsigned int i = 0; i < Compressor.mOutput->nTracks; i++) { - GPUMemCpyAlways(myStep, Compressor.mOutput->qTotA + offset, outputSrc->mPtrs.qTotA + Compressor.mAttachedClusterFirstIndex[i], Compressor.mOutput->nTrackClusters[i] * sizeof(Compressor.mOutput->qTotA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->qMaxA + offset, outputSrc->mPtrs.qMaxA + Compressor.mAttachedClusterFirstIndex[i], Compressor.mOutput->nTrackClusters[i] * sizeof(Compressor.mOutput->qMaxA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->flagsA + offset, outputSrc->mPtrs.flagsA + Compressor.mAttachedClusterFirstIndex[i], Compressor.mOutput->nTrackClusters[i] * sizeof(Compressor.mOutput->flagsA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->sigmaPadA + offset, outputSrc->mPtrs.sigmaPadA + Compressor.mAttachedClusterFirstIndex[i], Compressor.mOutput->nTrackClusters[i] * sizeof(Compressor.mOutput->sigmaPadA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->sigmaTimeA + offset, outputSrc->mPtrs.sigmaTimeA + Compressor.mAttachedClusterFirstIndex[i], Compressor.mOutput->nTrackClusters[i] * sizeof(Compressor.mOutput->sigmaTimeA[0]), 0, direction); + for (unsigned int i = 0; i < O->nTracks; i++) { + GPUMemCpyAlways(myStep, O->qTotA + offset, P->qTotA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->qTotA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->qMaxA + offset, P->qMaxA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->qMaxA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->flagsA + offset, P->flagsA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->flagsA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->sigmaPadA + offset, P->sigmaPadA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->sigmaPadA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->sigmaTimeA + offset, P->sigmaTimeA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->sigmaTimeA[0]), 0, direction); // First index stored with track - GPUMemCpyAlways(myStep, Compressor.mOutput->rowDiffA + offset - i, outputSrc->mPtrs.rowDiffA + Compressor.mAttachedClusterFirstIndex[i] + 1, (Compressor.mOutput->nTrackClusters[i] - 1) * sizeof(Compressor.mOutput->rowDiffA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->sliceLegDiffA + offset - i, outputSrc->mPtrs.sliceLegDiffA + Compressor.mAttachedClusterFirstIndex[i] + 1, (Compressor.mOutput->nTrackClusters[i] - 1) * sizeof(Compressor.mOutput->sliceLegDiffA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->padResA + offset - i, outputSrc->mPtrs.padResA + Compressor.mAttachedClusterFirstIndex[i] + 1, (Compressor.mOutput->nTrackClusters[i] - 1) * sizeof(Compressor.mOutput->padResA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->timeResA + offset - i, outputSrc->mPtrs.timeResA + Compressor.mAttachedClusterFirstIndex[i] + 1, (Compressor.mOutput->nTrackClusters[i] - 1) * sizeof(Compressor.mOutput->timeResA[0]), 0, direction); - offset += Compressor.mOutput->nTrackClusters[i]; - } - GPUMemCpyAlways(myStep, Compressor.mOutput->qPtA, outputSrc->mPtrs.qPtA, Compressor.mOutput->nTracks * sizeof(Compressor.mOutput->qPtA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->rowA, outputSrc->mPtrs.rowA, Compressor.mOutput->nTracks * sizeof(Compressor.mOutput->rowA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->sliceA, outputSrc->mPtrs.sliceA, Compressor.mOutput->nTracks * sizeof(Compressor.mOutput->sliceA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->timeA, outputSrc->mPtrs.timeA, Compressor.mOutput->nTracks * sizeof(Compressor.mOutput->timeA[0]), 0, direction); - GPUMemCpyAlways(myStep, Compressor.mOutput->padA, outputSrc->mPtrs.padA, Compressor.mOutput->nTracks * sizeof(Compressor.mOutput->padA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->rowDiffA + offset - i, P->rowDiffA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->rowDiffA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->sliceLegDiffA + offset - i, P->sliceLegDiffA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->sliceLegDiffA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->padResA + offset - i, P->padResA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->padResA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->timeResA + offset - i, P->timeResA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->timeResA[0]), 0, direction); + offset += O->nTrackClusters[i]; + } + GPUMemCpyAlways(myStep, O->qPtA, P->qPtA, O->nTracks * sizeof(O->qPtA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->rowA, P->rowA, O->nTracks * sizeof(O->rowA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->sliceA, P->sliceA, O->nTracks * sizeof(O->sliceA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->timeA, P->timeA, O->nTracks * sizeof(O->timeA[0]), 0, direction); + GPUMemCpyAlways(myStep, O->padA, P->padA, O->nTracks * sizeof(O->padA[0]), 0, direction); } SynchronizeGPU(); - mIOPtrs.tpcCompressedClusters = Compressor.mOutput; + mIOPtrs.tpcCompressedClusters = O; #endif return 0; } From 35985b7133a93d208919d7f6e1aea0f3547e8338 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 10:58:31 +0200 Subject: [PATCH 10/22] GPU: Reduce block-dimension to 1 if kernel execution falls back to CPU --- GPU/GPUTracking/Base/GPUReconstructionCPU.h | 10 +++++--- GPU/GPUTracking/Global/GPUChain.cxx | 4 +-- GPU/GPUTracking/Global/GPUChainTracking.cxx | 28 ++++++++++----------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.h b/GPU/GPUTracking/Base/GPUReconstructionCPU.h index 84ca2c986a7cd..a8fe8ac17d711 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.h +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.h @@ -244,13 +244,15 @@ inline int GPUReconstructionCPU::runKernel(const krnlExec& x, const krnlRunRange int cpuFallback = IsGPU() ? (x.device == krnlDeviceType::CPU ? 2 : (mRecoStepsGPU & myStep) != myStep) : 0; unsigned int nThreads = x.nThreads; unsigned int nBlocks = x.nBlocks; - const int autoThreads = (cpuFallback || x.device == krnlDeviceType::CPU) ? 1 : getKernelProperties().nThreads; - if (nThreads == (unsigned int)-1) { - nThreads = autoThreads; - } + const int autoThreads = cpuFallback ? 1 : getKernelProperties().nThreads; if (nBlocks == (unsigned int)-1) { nBlocks = (nThreads + autoThreads - 1) / autoThreads; nThreads = autoThreads; + } else if (nBlocks == (unsigned int)-2) { + nBlocks = nThreads; + nThreads = autoThreads; + } else if ((int)nThreads < 0) { + nThreads = cpuFallback ? 1 : -nThreads; } if (nThreads > GPUCA_MAX_THREADS) { throw std::runtime_error("GPUCA_MAX_THREADS exceeded"); diff --git a/GPU/GPUTracking/Global/GPUChain.cxx b/GPU/GPUTracking/Global/GPUChain.cxx index c65c9133999df..66d638dc9be0b 100644 --- a/GPU/GPUTracking/Global/GPUChain.cxx +++ b/GPU/GPUTracking/Global/GPUChain.cxx @@ -30,10 +30,10 @@ GPUChain::krnlExec GPUChain::GetGrid(unsigned int totalItems, int stream, GPURec GPUChain::krnlExec GPUChain::GetGridBlk(unsigned int nBlocks, int stream, GPUReconstruction::krnlDeviceType d, GPUCA_RECO_STEP st) { - return {nBlocks, (unsigned int)-1, stream, d, st}; + return {(unsigned int)-2, nBlocks, stream, d, st}; } GPUChain::krnlExec GPUChain::GetGridBlkStep(unsigned int nBlocks, int stream, GPUCA_RECO_STEP st) { - return {nBlocks, (unsigned int)-1, stream, GPUReconstruction::krnlDeviceType::Auto, st}; + return {(unsigned int)-2, nBlocks, stream, GPUReconstruction::krnlDeviceType::Auto, st}; } diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 75546da460766..3e6c6d26d28ad 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1274,7 +1274,7 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() if (!(doGPU || GetDeviceProcessingSettings().debugLevel >= 1) || GetDeviceProcessingSettings().trackletSelectorInPipeline) { runKernel(GetGridBlk(getKernelProperties().minBlocks * BlockCount(), useStream), {iSlice}); - runKernel({1, ThreadCount(), useStream}, {iSlice}, {}, 1); + runKernel({1, -ThreadCount(), useStream}, {iSlice}, {}, 1); TransferMemoryResourceLinkToHost(RecoStep::TPCSliceTracking, trk.MemoryResCommon(), useStream, &mEvents->selector[iSlice]); streamMap[iSlice] = useStream; if (GetDeviceProcessingSettings().debugLevel >= 3) { @@ -1333,7 +1333,7 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() GPUInfo("Running TPC Tracklet selector (Stream %d, Slice %d to %d)", useStream, iSlice, iSlice + runSlices); } runKernel(GetGridBlk(getKernelProperties().minBlocks * BlockCount(), useStream), {iSlice, runSlices}); - runKernel({1, ThreadCount(), useStream}, {iSlice}, {}, runSlices); + runKernel({1, -ThreadCount(), useStream}, {iSlice}, {}, runSlices); for (unsigned int k = iSlice; k < iSlice + runSlices; k++) { TransferMemoryResourceLinkToHost(RecoStep::TPCSliceTracking, processors()->tpcTrackers[k].MemoryResCommon(), useStream, &mEvents->selector[k]); streamMap[k] = useStream; @@ -1490,7 +1490,7 @@ void GPUChainTracking::RunTPCTrackingMerger_MergeBorderTracks(char withinSlice, int n = withinSlice == -1 ? NSLICES / 2 : NSLICES; for (int i = 0; i < n; i++) { runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i, withinSlice, mergeMode); - runKernel({2, WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, i, withinSlice, mergeMode); + runKernel({2, -WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, i, withinSlice, mergeMode); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i, withinSlice, mergeMode); } } @@ -1542,18 +1542,18 @@ int GPUChainTracking::RunTPCTrackingMerger() for (unsigned int i = 0; i < NSLICES; i++) { runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i); - runKernel({1, WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, i); + runKernel({1, -WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, i); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i); } for (unsigned int i = 0; i < NSLICES; i++) { - runKernel({1, WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, NSLICES + i); + runKernel({1, -WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, NSLICES + i); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i); } - runKernel({1, WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, 2 * NSLICES); + runKernel({1, -WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, 2 * NSLICES); DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpSliceTracks, mDebugFile); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); - runKernel({1, WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), NSLICES * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType); @@ -1561,21 +1561,21 @@ int GPUChainTracking::RunTPCTrackingMerger() DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); - runKernel({1, WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); - runKernel({1, WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); - runKernel({1, WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0); DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile); - runKernel({1, WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); @@ -1601,8 +1601,8 @@ int GPUChainTracking::RunTPCTrackingMerger() } if (!param().rec.NonConsecutiveIDs) { unsigned int* sharedCount = (unsigned int*)MergerShadow.TmpMem() + CAMath::nextMultipleOf<4>(Merger.Memory()->nOutputTracks); - runKernel({BlockCount(), ThreadCount(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, sharedCount, maxId * sizeof(*sharedCount)); - runKernel({BlockCount(), ThreadCount(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); + runKernel({BlockCount(), -ThreadCount(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, sharedCount, maxId * sizeof(*sharedCount)); + runKernel({BlockCount(), -ThreadCount(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); @@ -1615,7 +1615,7 @@ int GPUChainTracking::RunTPCTrackingMerger() TransferMemoryResourcesToGPU(RecoStep::TPCMerging, &Merger); } if (GetDeviceProcessingSettings().fitSlowTracksInOtherPass && GetDeviceProcessingSettings().mergerSortTracks) { - runKernel(GetGrid(Merger.NSlowTracks(), WarpSize(), 0), krnlRunRangeNone, krnlEventNone, -1); + runKernel(GetGrid(Merger.NSlowTracks(), -WarpSize(), 0), krnlRunRangeNone, krnlEventNone, -1); runKernel(GetGridBlk(Merger.NOutputTracks() - Merger.NSlowTracks(), 0), krnlRunRangeNone, krnlEventNone, 1); } else { runKernel(GetGridBlk(Merger.NOutputTracks(), 0), krnlRunRangeNone, krnlEventNone, 0); From e52f928a83d20d4d31ff1fe19bba2e1e372bc80e Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 16:11:48 +0200 Subject: [PATCH 11/22] GPU: Add propagation of register/unregister DMA memory functions --- .../include/TPCReconstruction/GPUCATracking.h | 3 +++ Detectors/TPC/reconstruction/src/GPUCATracking.cxx | 10 ++++++++++ GPU/GPUTracking/Interface/GPUO2Interface.cxx | 10 ++++++++++ GPU/GPUTracking/Interface/GPUO2Interface.h | 3 +++ 4 files changed, 26 insertions(+) diff --git a/Detectors/TPC/reconstruction/include/TPCReconstruction/GPUCATracking.h b/Detectors/TPC/reconstruction/include/TPCReconstruction/GPUCATracking.h index 3657bd0080061..cc58621cad8c7 100644 --- a/Detectors/TPC/reconstruction/include/TPCReconstruction/GPUCATracking.h +++ b/Detectors/TPC/reconstruction/include/TPCReconstruction/GPUCATracking.h @@ -61,6 +61,9 @@ class GPUCATracking int getNTracksASide() { return mNTracksASide; } void GetClusterErrors2(int row, float z, float sinPhi, float DzDs, short clusterState, float& ErrY2, float& ErrZ2) const; + int registerMemoryForGPU(const void* ptr, size_t size); + int unregisterMemoryForGPU(const void* ptr); + private: std::unique_ptr mTrackingCAO2Interface; //Pointer to Interface class in HLT O2 CA Tracking library. //The tracking code itself is not included in the O2 package, but contained in the CA library. diff --git a/Detectors/TPC/reconstruction/src/GPUCATracking.cxx b/Detectors/TPC/reconstruction/src/GPUCATracking.cxx index eefbc34b9b9d9..33fa68861c248 100644 --- a/Detectors/TPC/reconstruction/src/GPUCATracking.cxx +++ b/Detectors/TPC/reconstruction/src/GPUCATracking.cxx @@ -305,3 +305,13 @@ void GPUCATracking::GetClusterErrors2(int row, float z, float sinPhi, float DzDs } mTrackingCAO2Interface->GetClusterErrors2(row, z, sinPhi, DzDs, clusterState, ErrY2, ErrZ2); } + +int GPUCATracking::registerMemoryForGPU(const void* ptr, size_t size) +{ + return mTrackingCAO2Interface->registerMemoryForGPU(ptr, size); +} + +int GPUCATracking::unregisterMemoryForGPU(const void* ptr) +{ + return mTrackingCAO2Interface->unregisterMemoryForGPU(ptr); +} diff --git a/GPU/GPUTracking/Interface/GPUO2Interface.cxx b/GPU/GPUTracking/Interface/GPUO2Interface.cxx index 0f51b81734520..4c0ee9ba4c43c 100644 --- a/GPU/GPUTracking/Interface/GPUO2Interface.cxx +++ b/GPU/GPUTracking/Interface/GPUO2Interface.cxx @@ -122,3 +122,13 @@ void GPUTPCO2Interface::GetClusterErrors2(int row, float z, float sinPhi, float mRec->GetParam().GetClusterErrors2(row, z, sinPhi, DzDs, ErrY2, ErrZ2); mRec->GetParam().UpdateClusterError2ByState(clusterState, ErrY2, ErrZ2); } + +int GPUTPCO2Interface::registerMemoryForGPU(const void* ptr, size_t size) +{ + return mRec->registerMemoryForGPU(ptr, size); +} + +int GPUTPCO2Interface::unregisterMemoryForGPU(const void* ptr) +{ + return mRec->unregisterMemoryForGPU(ptr); +} diff --git a/GPU/GPUTracking/Interface/GPUO2Interface.h b/GPU/GPUTracking/Interface/GPUO2Interface.h index abdbc6f721147..3add50e4055d9 100644 --- a/GPU/GPUTracking/Interface/GPUO2Interface.h +++ b/GPU/GPUTracking/Interface/GPUO2Interface.h @@ -55,6 +55,9 @@ class GPUTPCO2Interface bool GetParamContinuous() { return (mContinuous); } void GetClusterErrors2(int row, float z, float sinPhi, float DzDs, short clusterState, float& ErrY2, float& ErrZ2) const; + int registerMemoryForGPU(const void* ptr, size_t size); + int unregisterMemoryForGPU(const void* ptr); + const GPUO2InterfaceConfiguration& getConfig() const { return *mConfig; } private: From 0b2d7823d24cb0224a4d6f4d8b6b07778beb14c8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 4 May 2020 13:38:34 +0200 Subject: [PATCH 12/22] GPU: Run Creation of TPC fast access grid on GPU --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 27 ++-- .../SliceTracker/GPUTPCCreateSliceData.cxx | 5 +- .../SliceTracker/GPUTPCCreateSliceData.h | 4 + .../SliceTracker/GPUTPCSliceData.cxx | 138 ++++++++++++------ .../SliceTracker/GPUTPCSliceData.h | 4 +- 5 files changed, 115 insertions(+), 63 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 3e6c6d26d28ad..74d1c6c4dabd2 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -730,8 +730,6 @@ void GPUChainTracking::WriteOutput(int iSlice, int threadId) if (GetDeviceProcessingSettings().debugLevel >= 5) { GPUInfo("Running WriteOutput for slice %d on thread %d\n", iSlice, threadId); } - HighResTimer& timer = getTimer("WriteOutput", threadId); - timer.Start(); if (GetDeviceProcessingSettings().nDeviceHelperThreads) { while (mLockAtomic.test_and_set(std::memory_order_acquire)) { ; @@ -742,7 +740,6 @@ void GPUChainTracking::WriteOutput(int iSlice, int threadId) mLockAtomic.clear(); } processors()->tpcTrackers[iSlice].WriteOutput(); - timer.Stop(); if (GetDeviceProcessingSettings().debugLevel >= 5) { GPUInfo("Finished WriteOutput for slice %d on thread %d\n", iSlice, threadId); } @@ -1132,6 +1129,7 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() GPUInfo("Running TPC Slice Tracker"); } bool doGPU = GetRecoStepsGPU() & RecoStep::TPCSliceTracking; + bool doSliceDataOnGPU = doGPU && (GetRecoStepsGPU() & RecoStep::TPCConversion) && mIOPtrs.clustersNative; bool streamInit[GPUCA_MAX_STREAMS] = {false}; for (unsigned int iSlice = 0; iSlice < NSLICES; iSlice++) { @@ -1150,7 +1148,9 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() #endif } - RunHelperThreads(&GPUChainTracking::HelperReadEvent, this, NSLICES); + if (!doSliceDataOnGPU) { + RunHelperThreads(&GPUChainTracking::HelperReadEvent, this, NSLICES); + } if (PrepareTextures()) { return (2); } @@ -1183,11 +1183,16 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() for (unsigned int iSlice = 0; iSlice < NSLICES; iSlice++) { GPUTPCTracker& trk = processors()->tpcTrackers[iSlice]; GPUTPCTracker& trkShadow = doGPU ? processorsShadow()->tpcTrackers[iSlice] : trk; + int useStream = (iSlice % mRec->NStreams()); if (GetDeviceProcessingSettings().debugLevel >= 3) { GPUInfo("Creating Slice Data (Slice %d)", iSlice); } - if (!doGPU || iSlice % (GetDeviceProcessingSettings().nDeviceHelperThreads + 1) == 0) { + if (doSliceDataOnGPU) { + TransferMemoryResourcesToGPU(RecoStep::TPCSliceTracking, &trk); + runKernel(GetGridBlk(GPUCA_ROW_COUNT, useStream), {iSlice}, {nullptr, streamInit[useStream] ? nullptr : &mEvents->init}); + streamInit[useStream] = true; + } else if (!doGPU || iSlice % (GetDeviceProcessingSettings().nDeviceHelperThreads + 1) == 0) { if (ReadEvent(iSlice, 0)) { GPUError("Error reading event"); error = 1; @@ -1216,7 +1221,6 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() } } - int useStream = (iSlice % mRec->NStreams()); // Initialize temporary memory where needed if (GetDeviceProcessingSettings().debugLevel >= 3) { GPUInfo("Copying Slice Data to GPU and initializing temporary memory"); @@ -1228,13 +1232,12 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() } // Copy Data to GPU Global Memory - HighResTimer& timer = getTimer("ReadEvent", useStream); - timer.Start(); - TransferMemoryResourcesToGPU(RecoStep::TPCSliceTracking, &trk); + if (!doSliceDataOnGPU) { + TransferMemoryResourcesToGPU(RecoStep::TPCSliceTracking, &trk); + } if (GPUDebug("Initialization (3)", useStream)) { throw std::runtime_error("memcpy failure"); } - timer.Stop(); runKernel(GetGridBlk(GPUCA_ROW_COUNT, useStream), {iSlice}, {nullptr, streamInit[useStream] ? nullptr : &mEvents->init}); streamInit[useStream] = true; @@ -1295,7 +1298,9 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() if (doGPU || GetDeviceProcessingSettings().debugLevel >= 1) { ReleaseEvent(&mEvents->init); - WaitForHelperThreads(); + if (!doSliceDataOnGPU) { + WaitForHelperThreads(); + } if (!GetDeviceProcessingSettings().trackletSelectorInPipeline) { if (GetDeviceProcessingSettings().trackletConstructorInPipeline) { diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx index a0314e1f5ff51..c884f685a23b9 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.cxx @@ -20,8 +20,5 @@ using namespace GPUCA_NAMESPACE::gpu; template <> GPUdii() void GPUTPCCreateSliceData::Thread<0>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() MEM_LOCAL(GPUSharedMemory) & GPUrestrict() s, processorType& GPUrestrict() tracker) { - if (iThread || iBlock) { - return; - } - tracker.Data().InitFromClusterData(nBlocks, nThreads, iBlock, iThread, tracker.GetConstantMem(), tracker.ISlice()); + tracker.Data().InitFromClusterData(nBlocks, nThreads, iBlock, iThread, tracker.GetConstantMem(), tracker.ISlice(), s.tmp); } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h index dc7f97fb4775c..13704097f3b41 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCCreateSliceData.h @@ -28,6 +28,10 @@ class GPUTPCTracker; class GPUTPCCreateSliceData : public GPUKernelTemplate { public: + struct GPUSharedMemory { + float tmp[4]; + }; + typedef GPUconstantref() MEM_GLOBAL(GPUTPCTracker) processorType; GPUhdi() CONSTEXPRRET static GPUDataTypes::RecoStep GetRecoStep() { return GPUCA_RECO_STEP::TPCSliceTracking; } MEM_TEMPLATE() diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index 87b586bb3bc3d..bc0fbfced61e6 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -104,7 +104,7 @@ GPUd() unsigned int GPUTPCSliceData::GetGridSize(unsigned int nHits, unsigned in return 26 * nRows + 4 * nHits; } -GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, const float2* GPUrestrict() data, int ClusterDataHitNumberOffset, float yMin, float yMax, float zMin, float zMax) +GPUdi() void GPUTPCSliceData::CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, GPUTPCRow* GPUrestrict() row, float yMin, float yMax, float zMin, float zMax) { float dz = zMax - zMin; float tfFactor = 1.; @@ -138,7 +138,7 @@ GPUdi() static void UpdateMinMaxYZ(float& yMin, float& yMax, float& zMin, float& } } -GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int iBlock, int iThread, GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice) +GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int iBlock, int iThread, GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * GPUrestrict() mem, int iSlice, float* tmpMinMax) { #ifdef GPUCA_GPUCODE constexpr bool EarlyTransformWithoutClusterNative = false; @@ -154,16 +154,17 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i const unsigned int* RowOffsets = nullptr; #ifndef GPUCA_GPUCODE - std::unique_ptr YZData_p(new float2[mNumberOfHits]); - float2* YZData = YZData_p.get(); + vecpod YZData(mNumberOfHits); + vecpod binMemory(mNumberOfHits); unsigned int RowOffsetsA[GPUCA_ROW_COUNT]; unsigned int NumberOfClustersInRowA[GPUCA_ROW_COUNT]; - std::unique_ptr tmpHitIndex_p(new int[mNumberOfHits]); + vecpod tmpHitIndexA; if (EarlyTransformWithoutClusterNative) { // Implies mem->param.earlyTpcTransform but no ClusterNative present NumberOfClustersInRow = NumberOfClustersInRowA; RowOffsets = RowOffsetsA; - tmpHitIndex = tmpHitIndex_p.get(); + tmpHitIndexA.resize(mNumberOfHits); + tmpHitIndex = tmpHitIndexA.data(); memset(NumberOfClustersInRowA, 0, GPUCA_ROW_COUNT * sizeof(NumberOfClustersInRowA[0])); for (int i = 0; i < mNumberOfHits; i++) { @@ -188,15 +189,25 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i } } // Other cases below in loop over rows #else - float2* YZData = (float2*)mLinkUpData; + float2* YZData = (float2*)mLinkUpData; // TODO: we can do this as well on the CPU, just must make sure that CPU has the scratch memory + calink* binMemory = (calink*)mHitWeights; static_assert(sizeof(*YZData) <= (sizeof(*mLinkUpData) + sizeof(*mLinkDownData)), "Cannot reuse memory"); + static_assert(sizeof(*binMemory) <= sizeof(*mHitWeights), "Cannot reuse memory"); #endif - for (int rowIndex = 0; rowIndex < GPUCA_ROW_COUNT; ++rowIndex) { + for (int rowIndex = iBlock; rowIndex < GPUCA_ROW_COUNT; rowIndex += nBlocks) { float yMin = 1.e6f; float yMax = -1.e6f; float zMin = 1.e6f; float zMax = -1.e6f; + GPUbarrier(); + if (iThread == 0) { + tmpMinMax[0] = -yMin; + tmpMinMax[1] = yMax; + tmpMinMax[2] = -zMin; + tmpMinMax[3] = zMax; + } + GPUbarrier(); const unsigned int NumberOfClusters = EarlyTransformWithoutClusterNative ? NumberOfClustersInRow[rowIndex] : mem->ioPtrs.clustersNative->nClusters[iSlice][rowIndex]; const unsigned int RowOffset = EarlyTransformWithoutClusterNative ? RowOffsets[rowIndex] : (mem->ioPtrs.clustersNative->clusterOffset[iSlice][rowIndex] - mem->ioPtrs.clustersNative->clusterOffset[iSlice][0]); @@ -211,26 +222,28 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i GPUTPCRow& row = mRows[rowIndex]; if (NumberOfClusters == 0) { - row.mGrid.CreateEmpty(); - row.mNHits = 0; - row.mHitNumberOffset = 0; - row.mFirstHitInBinOffset = 0; - row.mHy0 = 0.f; - row.mHz0 = 0.f; - row.mHstepY = 1.f; - row.mHstepZ = 1.f; - row.mHstepYi = 1.f; - row.mHstepZi = 1.f; + if (iThread == 0) { + row.mGrid.CreateEmpty(); + row.mNHits = 0; + row.mHitNumberOffset = 0; + row.mFirstHitInBinOffset = 0; + row.mHy0 = 0.f; + row.mHz0 = 0.f; + row.mHstepY = 1.f; + row.mHstepZ = 1.f; + row.mHstepYi = 1.f; + row.mHstepZi = 1.f; + } continue; } if (EarlyTransformWithoutClusterNative) { - for (unsigned int i = 0; i < NumberOfClusters; i++) { - UpdateMinMaxYZ(yMin, yMax, zMin, zMax, YZData[RowOffset - i].x, YZData[RowOffset - i].y); + for (unsigned int i = iThread; i < NumberOfClusters; i += nThreads) { + UpdateMinMaxYZ(yMin, yMax, zMin, zMax, YZData[RowOffset + i].x, YZData[RowOffset + i].y); } } else { if (mem->param.earlyTpcTransform) { // Early transform case with ClusterNative present - for (unsigned int i = 0; i < NumberOfClusters; i++) { + for (unsigned int i = iThread; i < NumberOfClusters; i += nThreads) { float2 tmp; tmp.x = mClusterData[RowOffset + i].y; tmp.y = mClusterData[RowOffset + i].z; @@ -238,7 +251,7 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i YZData[RowOffset + i] = tmp; } } else { - for (unsigned int i = 0; i < NumberOfClusters; i++) { + for (unsigned int i = iThread; i < NumberOfClusters; i += nThreads) { float x, y, z; GPUTPCConvertImpl::convert(*mem, iSlice, rowIndex, mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getPad(), mem->ioPtrs.clustersNative->clusters[iSlice][rowIndex][i].getTime(), x, y, z); UpdateMinMaxYZ(yMin, yMax, zMin, zMax, y, z); @@ -247,11 +260,38 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i } } - row.mNHits = NumberOfClusters; - row.mHitNumberOffset = CAMath::nextMultipleOf(RowOffset + rowIndex * GPUCA_ROWALIGNMENT / sizeof(calink)); - row.mFirstHitInBinOffset = CAMath::nextMultipleOf(GetGridSize(RowOffset, rowIndex) + rowIndex * GPUCA_ROWALIGNMENT / sizeof(int)); + if (iThread == 0) { + row.mNHits = NumberOfClusters; + row.mHitNumberOffset = CAMath::nextMultipleOf(RowOffset + rowIndex * GPUCA_ROWALIGNMENT / sizeof(calink)); + row.mFirstHitInBinOffset = CAMath::nextMultipleOf(GetGridSize(RowOffset, rowIndex) + rowIndex * GPUCA_ROWALIGNMENT / sizeof(int)); + } - CreateGrid(mem, &row, YZData, RowOffset, yMin, yMax, zMin, zMax); + /* CAMath::AtomicMaxShared(&tmpMinMax[0], -yMin); // Atomic max not supported for float + CAMath::AtomicMaxShared(&tmpMinMax[1], yMax); + CAMath::AtomicMaxShared(&tmpMinMax[2], -zMin); + CAMath::AtomicMaxShared(&tmpMinMax[3], zMax); */ + for (int i = 0; i < nThreads; i++) { // Todo: Implement a better version of this stupid atomic max replacement + GPUbarrier(); + if (iThread == i) { + if (tmpMinMax[0] < -yMin) { + tmpMinMax[0] = -yMin; + } + if (tmpMinMax[1] < yMax) { + tmpMinMax[1] = yMax; + } + if (tmpMinMax[2] < -zMin) { + tmpMinMax[2] = -zMin; + } + if (tmpMinMax[3] < zMax) { + tmpMinMax[3] = zMax; + } + } + } + GPUbarrier(); + if (iThread == 0) { + CreateGrid(mem, &row, -tmpMinMax[0], tmpMinMax[1], -tmpMinMax[2], tmpMinMax[3]); + } + GPUbarrier(); const GPUTPCGrid& grid = row.mGrid; const int numberOfBins = grid.N(); if ((long long int)numberOfBins >= ((long long int)1 << (sizeof(calink) * 8))) { @@ -264,29 +304,33 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i return 1; } - calink* c = mFirstHitInBin + row.mFirstHitInBinOffset; // number of hits in all previous bins - calink* bins = (calink*)mHitWeights + RowOffset; // Reuse mLinkUpData memory as temporary memory - static_assert(sizeof(*bins) <= sizeof(*mHitWeights), "Cannot reuse memory"); + GPUAtomic(calink)* c = (GPUAtomic(calink)*)mFirstHitInBin + row.mFirstHitInBinOffset; // number of hits in all previous bins + calink* bins = &binMemory[RowOffset]; // Reuse mLinkUpData memory as temporary memory - for (int bin = 0; bin < numberOfBins; ++bin) { - c[bin] = 0; // initialize filled[] to 0 + for (int bin = iThread; bin < numberOfBins; bin += nThreads) { + c[bin] = 0; // initialize to 0 } - for (int hitIndex = 0; hitIndex < row.mNHits; ++hitIndex) { + GPUbarrier(); + for (int hitIndex = iThread; hitIndex < row.mNHits; hitIndex += nThreads) { const int globalHitIndex = RowOffset + hitIndex; const calink bin = row.mGrid.GetBin(YZData[globalHitIndex].x, YZData[globalHitIndex].y); bins[hitIndex] = bin; - ++c[bin]; + CAMath::AtomicAdd(&c[bin], 1u); } + GPUbarrier(); - calink n = 0; - for (int bin = 0; bin < numberOfBins; ++bin) { - n += c[bin]; - c[bin] = n; - } - for (unsigned int bin = numberOfBins; bin < nn; bin++) { - c[bin] = n; + if (iThread == 0) { + calink n = 0; + for (int bin = 0; bin < numberOfBins; ++bin) { // TODO: Parallelize + n += c[bin]; + c[bin] = n; + } + for (unsigned int bin = numberOfBins; bin < nn; bin++) { + c[bin] = n; + } } + GPUbarrier(); constexpr float maxVal = (((long long int)1 << (sizeof(cahit) < 3 ? sizeof(cahit) * 8 : 24)) - 1); // Stay within float precision in any case! constexpr float packingConstant = 1.f / (maxVal - 2.); @@ -304,9 +348,9 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i row.mHstepYi = stepYi; row.mHstepZi = stepZi; - for (int hitIndex = 0; hitIndex < row.mNHits; ++hitIndex) { + for (int hitIndex = iThread; hitIndex < row.mNHits; hitIndex += nThreads) { const calink bin = bins[hitIndex]; - const calink ind = --c[bin]; // generate an index for this hit that is >= c[bin] and < c[bin + 1] + const calink ind = CAMath::AtomicAdd(&c[bin], (calink)-1) - 1; // generate an index for this hit that is >= c[bin] and < c[bin + 1] const int globalBinsortedIndex = row.mHitNumberOffset + ind; const int globalHitIndex = RowOffset + hitIndex; @@ -326,10 +370,12 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i mHitData[globalBinsortedIndex].y = (cahit)yy; } - const float maxAbsZ = CAMath::Max(CAMath::Abs(zMin), CAMath::Abs(zMax)); - if (maxAbsZ > 300 && !mem->param.ContinuousTracking) { - GPUError("Need to set continuous tracking mode for data outside of the TPC volume!"); // TODO: Set GPU error code - return 1; + if (iThread == 0) { + const float maxAbsZ = CAMath::Max(CAMath::Abs(tmpMinMax[2]), CAMath::Abs(tmpMinMax[3])); + if (maxAbsZ > 300 && !mem->param.ContinuousTracking) { + GPUError("Need to set continuous tracking mode for data outside of the TPC volume!"); // TODO: Set GPU error code + return 1; + } } } diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h index c247b1e611638..3ff210b661db7 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.h @@ -44,7 +44,7 @@ class GPUTPCSliceData void* SetPointersRows(void* mem); #endif - GPUd() int InitFromClusterData(int nBlocks, int nThreads, int iBlock, int iThread, GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int iSlice); + GPUd() int InitFromClusterData(int nBlocks, int nThreads, int iBlock, int iThread, GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, int iSlice, float* tmpMinMax); /** * Return the number of hits in this slice. @@ -137,7 +137,7 @@ class GPUTPCSliceData GPUTPCSliceData& operator=(const GPUTPCSliceData&) CON_DELETE; // ROOT 5 tries to use this if it is not private GPUTPCSliceData(const GPUTPCSliceData&) CON_DELETE; // #endif - GPUd() void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, const float2* data, int ClusterDataHitNumberOffset, float yMin, float yMax, float zMin, float zMax); + GPUd() void CreateGrid(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, float yMin, float yMax, float zMin, float zMax); GPUd() static void GetMaxNBins(GPUconstantref() const MEM_CONSTANT(GPUConstantMem) * mem, MEM_GLOBAL(GPUTPCRow) * GPUrestrict() row, int& maxY, int& maxZ); GPUd() unsigned int GetGridSize(unsigned int nHits, unsigned int nRows); From ac1c3bdfd80b8b420de5152185111977012595c1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 11:10:08 +0200 Subject: [PATCH 13/22] GPU: Fix track merging when only part of kernels runs on GPU --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 74d1c6c4dabd2..ade0f6dd4f548 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1523,6 +1523,7 @@ int GPUChainTracking::RunTPCTrackingMerger() GPUTPCGMMerger& Merger = processors()->tpcMerger; GPUTPCGMMerger& MergerShadow = doGPU ? processorsShadow()->tpcMerger : Merger; + GPUTPCGMMerger& MergerShadowAll = doGPUall ? processorsShadow()->tpcMerger : Merger; if (GetDeviceProcessingSettings().debugLevel >= 2) { GPUInfo("Running TPC Merger"); } @@ -1558,7 +1559,7 @@ int GPUChainTracking::RunTPCTrackingMerger() DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpSliceTracks, mDebugFile); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); - runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType); @@ -1566,21 +1567,21 @@ int GPUChainTracking::RunTPCTrackingMerger() DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); - runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); - runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); - runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0); DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile); - runKernel({1, -WarpSize(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadow.TmpCounter())); + runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); @@ -1605,9 +1606,9 @@ int GPUChainTracking::RunTPCTrackingMerger() throw std::runtime_error("mNMaxClusters too small"); } if (!param().rec.NonConsecutiveIDs) { - unsigned int* sharedCount = (unsigned int*)MergerShadow.TmpMem() + CAMath::nextMultipleOf<4>(Merger.Memory()->nOutputTracks); - runKernel({BlockCount(), -ThreadCount(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, sharedCount, maxId * sizeof(*sharedCount)); - runKernel({BlockCount(), -ThreadCount(), 0, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); + unsigned int* sharedCount = (unsigned int*)MergerShadowAll.TmpMem() + CAMath::nextMultipleOf<4>(Merger.Memory()->nOutputTracks); + runKernel({BlockCount(), -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, sharedCount, maxId * sizeof(*sharedCount)); + runKernel({BlockCount(), -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.ClusterAttachment(), maxId * sizeof(*MergerShadowAll.ClusterAttachment())); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); From 4ba29243a53da5b1cf4a047291c9f8af1cb2f3c2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 13:56:38 +0200 Subject: [PATCH 14/22] GPU: Fix - after cluster overflow during CE tracking fill bogus clusters must still have a valid ID to avoid segfaults --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 198fbc85ae5e4..5d9ae35d2dde7 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1060,6 +1060,10 @@ GPUd() void GPUTPCGMMerger::MergeCE(int nBlocks, int nThreads, int iBlock, int i #else // TODO: proper overflow handling #endif + for (unsigned int k = newRef; k < mNMaxOutputTrackClusters; k++) { + mClusters[k].num = 0; + mClusters[k].state = 0; + } CAMath::AtomicExch(&mMemory->nOutputTrackClusters, mNMaxOutputTrackClusters); return; } From a4f0aa500bfe1895abf9dbb6d67290537bb7edc9 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 13:59:13 +0200 Subject: [PATCH 15/22] GPU: Fix CE tracking, reserve cluster IDs only after CE track is confirmed --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 5d9ae35d2dde7..84d9e0cc6fb65 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1052,6 +1052,10 @@ GPUd() void GPUTPCGMMerger::MergeCE(int nBlocks, int nThreads, int iBlock, int i if (!trk[1]->OK() || trk[1]->CCE()) { continue; } + bool looper = trk[0]->Looper() || trk[1]->Looper() || (trk[0]->GetParam().GetQPt() > 1 && trk[0]->GetParam().GetQPt() * trk[1]->GetParam().GetQPt() < 0); + if (!looper && trk[0]->GetParam().GetPar(3) * trk[1]->GetParam().GetPar(3) < 0) { + continue; + } unsigned int newRef = CAMath::AtomicAdd(&mMemory->nOutputTrackClusters, trk[0]->NClusters() + trk[1]->NClusters()); if (newRef + trk[0]->NClusters() + trk[1]->NClusters() >= mNMaxOutputTrackClusters) { @@ -1068,10 +1072,6 @@ GPUd() void GPUTPCGMMerger::MergeCE(int nBlocks, int nThreads, int iBlock, int i return; } - bool looper = trk[0]->Looper() || trk[1]->Looper() || (trk[0]->GetParam().GetQPt() > 1 && trk[0]->GetParam().GetQPt() * trk[1]->GetParam().GetQPt() < 0); - if (!looper && trk[0]->GetParam().GetPar(3) * trk[1]->GetParam().GetPar(3) < 0) { - continue; - } bool needswap = false; if (looper) { float z0max, z1max; From e118d31cd46b4761ecb0be61df3aa53a4b9a1f83 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 15:10:00 +0200 Subject: [PATCH 16/22] GPU: Fix TPC Merging with Run 2 data, improve debug output --- GPU/GPUTracking/Base/GPUReconstructionCPU.h | 2 +- GPU/GPUTracking/Global/GPUChain.h | 13 +++- GPU/GPUTracking/Global/GPUChainTracking.cxx | 75 +++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 2 +- GPU/GPUTracking/Standalone/qa/GPUQA.cxx | 4 +- 5 files changed, 50 insertions(+), 46 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.h b/GPU/GPUTracking/Base/GPUReconstructionCPU.h index a8fe8ac17d711..b2a523a9084da 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.h +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.h @@ -258,7 +258,7 @@ inline int GPUReconstructionCPU::runKernel(const krnlExec& x, const krnlRunRange throw std::runtime_error("GPUCA_MAX_THREADS exceeded"); } if (mDeviceProcessingSettings.debugLevel >= 3) { - GPUInfo("Running %s (Stream %d, Range %d/%d, Grid %d/%d) on %s", GetKernelName(), x.stream, y.start, y.num, nBlocks, nThreads, cpuFallback == 2 ? "CPU (forced)" : cpuFallback ? "CPU (fallback)" : mDeviceName.c_str()); + GPUInfo("Running kernel %s (Stream %d, Range %d/%d, Grid %d/%d) on %s", GetKernelName(), x.stream, y.start, y.num, nBlocks, nThreads, cpuFallback == 2 ? "CPU (forced)" : cpuFallback ? "CPU (fallback)" : mDeviceName.c_str()); } if (nThreads == 0 || nBlocks == 0) { return 0; diff --git a/GPU/GPUTracking/Global/GPUChain.h b/GPU/GPUTracking/Global/GPUChain.h index 1c78976f32692..e6cf5d8108757 100644 --- a/GPU/GPUTracking/Global/GPUChain.h +++ b/GPU/GPUTracking/Global/GPUChain.h @@ -200,7 +200,12 @@ class GPUChain virtual int DoStuckProtection(int stream, void* event) { return 0; } template - bool DoDebugAndDump(RecoStep step, int mask, T& processor, S T::*func, Args&&... args); + bool DoDebugAndDump(RecoStep step, int mask, T& processor, S T::*func, Args&&... args) + { + return DoDebugAndDump(step, mask, true, processor, func, args...); + } + template + bool DoDebugAndDump(RecoStep step, int mask, bool transfer, T& processor, S T::*func, Args&&... args); private: template @@ -240,10 +245,12 @@ inline void GPUChain::timeCpy(RecoStep step, bool toGPU, S T::*func, Args... arg } template -bool GPUChain::DoDebugAndDump(GPUChain::RecoStep step, int mask, T& processor, S T::*func, Args&&... args) +bool GPUChain::DoDebugAndDump(GPUChain::RecoStep step, int mask, bool transfer, T& processor, S T::*func, Args&&... args) { if (GetDeviceProcessingSettings().keepAllMemory) { - TransferMemoryResourcesToHost(step, &processor, -1, true); + if (transfer) { + TransferMemoryResourcesToHost(step, &processor, -1, true); + } if (GetDeviceProcessingSettings().debugLevel >= 6 && (mask == 0 || (GetDeviceProcessingSettings().debugMask & mask))) { (processor.*func)(args...); return true; diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index ade0f6dd4f548..e546f6eeb1117 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -795,7 +795,9 @@ int GPUChainTracking::GlobalTracking(unsigned int iSlice, int threadId) GPUReconstruction::krnlDeviceType deviceType = GetDeviceProcessingSettings().fullMergerOnGPU ? GPUReconstruction::krnlDeviceType::Auto : GPUReconstruction::krnlDeviceType::CPU; runKernel(GetGridBlk(256, 0, deviceType), {iSlice}); - TransferMemoryResourceLinkToHost(RecoStep::TPCSliceTracking, processors()->tpcTrackers[iSlice].MemoryResCommon(), 0); + if (GetDeviceProcessingSettings().fullMergerOnGPU) { + TransferMemoryResourceLinkToHost(RecoStep::TPCSliceTracking, processors()->tpcTrackers[iSlice].MemoryResCommon(), 0); + } SynchronizeGPU(); if (GetDeviceProcessingSettings().debugLevel >= 5) { @@ -1285,12 +1287,6 @@ int GPUChainTracking::RunTPCTrackingSlices_internal() } DoDebugAndDump(RecoStep::TPCSliceTracking, 512, trk, &GPUTPCTracker::DumpTrackHits, mDebugFile); } - - if (!(doGPU || GetDeviceProcessingSettings().debugLevel >= 1)) { - if (!param().rec.GlobalTracking && (GetRecoStepsOutputs() & GPUDataTypes::InOutType::TPCSectorTracks)) { - WriteOutput(iSlice, 0); - } - } } if (error) { return (3); @@ -1520,6 +1516,7 @@ int GPUChainTracking::RunTPCTrackingMerger() bool doGPU = GetRecoStepsGPU() & RecoStep::TPCMerging; bool doGPUall = doGPU && GetDeviceProcessingSettings().fullMergerOnGPU; GPUReconstruction::krnlDeviceType deviceType = GetDeviceProcessingSettings().fullMergerOnGPU ? GPUReconstruction::krnlDeviceType::Auto : GPUReconstruction::krnlDeviceType::CPU; + unsigned int numBlocks = doGPUall ? BlockCount() : 1; GPUTPCGMMerger& Merger = processors()->tpcMerger; GPUTPCGMMerger& MergerShadow = doGPU ? processorsShadow()->tpcMerger : Merger; @@ -1547,58 +1544,58 @@ int GPUChainTracking::RunTPCTrackingMerger() } for (unsigned int i = 0; i < NSLICES; i++) { - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, i); runKernel({1, -WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, i); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, i); } for (unsigned int i = 0; i < NSLICES; i++) { runKernel({1, -WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, NSLICES + i); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, i); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, i); } runKernel({1, -WarpSize(), 0, deviceType}, krnlRunRangeNone, krnlEventNone, 2 * NSLICES); - DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpSliceTracks, mDebugFile); + DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpSliceTracks, mDebugFile); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), NSLICES * sizeof(*MergerShadowAll.TmpCounter())); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1); - DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1); + DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); - runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0); + runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); - runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0); + runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); - runKernel(GetGridBlk(std::max(2u, BlockCount()), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1); + runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0); - DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0); + DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); - DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpCollected, mDebugFile); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); + DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpCollected, mDebugFile); if (doGPUall) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResMemory()); } - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone, 1); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 1); RunTPCTrackingMerger_MergeBorderTracks(-1, 1, deviceType); RunTPCTrackingMerger_MergeBorderTracks(-1, 2, deviceType); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); - DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpMergeCE, mDebugFile); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); + DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergeCE, mDebugFile); if (GetDeviceProcessingSettings().mergerSortTracks) { - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); } unsigned int maxId = param().rec.NonConsecutiveIDs ? Merger.Memory()->nOutputTrackClusters : Merger.NMaxClusters(); @@ -1607,15 +1604,15 @@ int GPUChainTracking::RunTPCTrackingMerger() } if (!param().rec.NonConsecutiveIDs) { unsigned int* sharedCount = (unsigned int*)MergerShadowAll.TmpMem() + CAMath::nextMultipleOf<4>(Merger.Memory()->nOutputTracks); - runKernel({BlockCount(), -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, sharedCount, maxId * sizeof(*sharedCount)); - runKernel({BlockCount(), -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.ClusterAttachment(), maxId * sizeof(*MergerShadowAll.ClusterAttachment())); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); - runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel({numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, sharedCount, maxId * sizeof(*sharedCount)); + runKernel({numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.ClusterAttachment(), maxId * sizeof(*MergerShadowAll.ClusterAttachment())); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); } - DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpFitPrepare, mDebugFile); + DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpFitPrepare, mDebugFile); if (doGPU && !doGPUall) { TransferMemoryResourcesToGPU(RecoStep::TPCMerging, &Merger); @@ -1643,7 +1640,7 @@ int GPUChainTracking::RunTPCTrackingMerger() runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); } - DoDebugAndDump(RecoStep::TPCMerging, 0, Merger, &GPUTPCGMMerger::DumpFinal, mDebugFile); + DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpFinal, mDebugFile); if (doGPUall) { TransferMemoryResourcesToHost(RecoStep::TPCMerging, &Merger); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 77d55c436de37..70d565d42704a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -40,7 +40,7 @@ void GPUTPCGMMerger::DumpSliceTracks(std::ostream& out) out << std::setprecision(2); out << "\nTPC Merger Slice Tracks\n"; for (int iSlice = 0; iSlice < NSLICES; iSlice++) { - out << "Slice Track Info Index" << mSliceTrackInfoIndex[iSlice] << " / " << mSliceTrackInfoIndex[NSLICES + iSlice] << "\n"; + out << "Slice Track Info Index " << (mSliceTrackInfoIndex[iSlice + 1] - mSliceTrackInfoIndex[iSlice]) << " / " << (mSliceTrackInfoIndex[NSLICES + iSlice + 1] - mSliceTrackInfoIndex[NSLICES + iSlice]) << "\n"; for (int iGlobal = 0; iGlobal < 2; iGlobal++) { out << " Track type " << iGlobal << "\n"; for (int j = mSliceTrackInfoIndex[iSlice + NSLICES * iGlobal]; j < mSliceTrackInfoIndex[iSlice + NSLICES * iGlobal + 1]; j++) { diff --git a/GPU/GPUTracking/Standalone/qa/GPUQA.cxx b/GPU/GPUTracking/Standalone/qa/GPUQA.cxx index a033552384300..76778b9976a52 100644 --- a/GPU/GPUTracking/Standalone/qa/GPUQA.cxx +++ b/GPU/GPUTracking/Standalone/qa/GPUQA.cxx @@ -571,8 +571,6 @@ void GPUQA::RunQA(bool matchOnly) mcEffBuffer[mNEvents - 1].resize(GetNMCTracks(0)); mcLabelBuffer[mNEvents - 1].resize(merger.NOutputTracks()); } - std::vector& effBuffer = mcEffBuffer[mNEvents - 1]; - std::vector& labelBuffer = mcLabelBuffer[mNEvents - 1]; bool mcAvail = mcPresent(); @@ -836,6 +834,7 @@ void GPUQA::RunQA(bool matchOnly) mc2.eta = -std::log(std::tan(0.5 * mc2.theta)); } if (mConfig.writeMCLabels) { + std::vector& effBuffer = mcEffBuffer[mNEvents - 1]; effBuffer[i] = mRecTracks[iCol][i] * 1000 + mFakeTracks[iCol][i]; } } @@ -933,6 +932,7 @@ void GPUQA::RunQA(bool matchOnly) for (int i = 0; i < merger.NOutputTracks(); i++) { if (mConfig.writeMCLabels) { + std::vector& labelBuffer = mcLabelBuffer[mNEvents - 1]; labelBuffer[i] = mTrackMCLabels[i].getTrackID(); } if (mTrackMCLabels[i].isFake()) { From 974e223ce43f543404d416f8f71295e581733f44 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 16:30:55 +0200 Subject: [PATCH 17/22] GPU: Fix TPC track pointers when all tracks come from adjacent sectors but none from seeds --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 84d9e0cc6fb65..e595e64c19809 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -508,8 +508,8 @@ GPUd() void GPUTPCGMMerger::RefitSliceTracks(int nBlocks, int nThreads, int iBlo TrackIds[iSlice * mNMaxSingleSliceTracks + sliceTr->LocalTrackId()] = myTrack; mSliceTrackInfos[myTrack] = track; } - if (!Param().rec.mergerReadFromTrackerDirectly && nLocalTracks) { - mMemory->firstGlobalTracks[iSlice] = sliceTr->GetNextTrack(); + if (!Param().rec.mergerReadFromTrackerDirectly) { + mMemory->firstGlobalTracks[iSlice] = nLocalTracks ? sliceTr->GetNextTrack() : mkSlices[iSlice]->GetFirstTrack(); } } From 8804f493a47928a08df8e7efb94e5f12313f775d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 19:02:35 +0200 Subject: [PATCH 18/22] GPU: Fix: firstHitInBin must be initialized properly also for empty TPC rows --- .../SliceTracker/GPUTPCSliceData.cxx | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx index bc0fbfced61e6..29212d648f75b 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCSliceData.cxx @@ -200,14 +200,6 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i float yMax = -1.e6f; float zMin = 1.e6f; float zMax = -1.e6f; - GPUbarrier(); - if (iThread == 0) { - tmpMinMax[0] = -yMin; - tmpMinMax[1] = yMax; - tmpMinMax[2] = -zMin; - tmpMinMax[3] = zMax; - } - GPUbarrier(); const unsigned int NumberOfClusters = EarlyTransformWithoutClusterNative ? NumberOfClustersInRow[rowIndex] : mem->ioPtrs.clustersNative->nClusters[iSlice][rowIndex]; const unsigned int RowOffset = EarlyTransformWithoutClusterNative ? RowOffsets[rowIndex] : (mem->ioPtrs.clustersNative->clusterOffset[iSlice][rowIndex] - mem->ioPtrs.clustersNative->clusterOffset[iSlice][0]); @@ -221,18 +213,29 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i } GPUTPCRow& row = mRows[rowIndex]; + if (iThread == 0) { + tmpMinMax[0] = -yMin; + tmpMinMax[1] = yMax; + tmpMinMax[2] = -zMin; + tmpMinMax[3] = zMax; + row.mFirstHitInBinOffset = CAMath::nextMultipleOf(GetGridSize(RowOffset, rowIndex) + rowIndex * GPUCA_ROWALIGNMENT / sizeof(int)); + } + GPUbarrier(); + GPUAtomic(calink)* c = (GPUAtomic(calink)*)mFirstHitInBin + row.mFirstHitInBinOffset; if (NumberOfClusters == 0) { if (iThread == 0) { row.mGrid.CreateEmpty(); row.mNHits = 0; row.mHitNumberOffset = 0; - row.mFirstHitInBinOffset = 0; row.mHy0 = 0.f; row.mHz0 = 0.f; row.mHstepY = 1.f; row.mHstepZ = 1.f; row.mHstepYi = 1.f; row.mHstepZi = 1.f; + for (int i = 0; i < 4; i++) { + c[i] = 0; + } } continue; } @@ -263,7 +266,6 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i if (iThread == 0) { row.mNHits = NumberOfClusters; row.mHitNumberOffset = CAMath::nextMultipleOf(RowOffset + rowIndex * GPUCA_ROWALIGNMENT / sizeof(calink)); - row.mFirstHitInBinOffset = CAMath::nextMultipleOf(GetGridSize(RowOffset, rowIndex) + rowIndex * GPUCA_ROWALIGNMENT / sizeof(int)); } /* CAMath::AtomicMaxShared(&tmpMinMax[0], -yMin); // Atomic max not supported for float @@ -304,7 +306,6 @@ GPUd() int GPUTPCSliceData::InitFromClusterData(int nBlocks, int nThreads, int i return 1; } - GPUAtomic(calink)* c = (GPUAtomic(calink)*)mFirstHitInBin + row.mFirstHitInBinOffset; // number of hits in all previous bins calink* bins = &binMemory[RowOffset]; // Reuse mLinkUpData memory as temporary memory for (int bin = iThread; bin < numberOfBins; bin += nThreads) { From 6498d507a3d1f726d0bffe939d5e545624774e40 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 19:56:24 +0200 Subject: [PATCH 19/22] GPU: Keep 36 timers for all GPU kernels, overhead is negligible and distinguishing too complicated --- GPU/GPUTracking/Base/GPUReconstructionCPU.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.h b/GPU/GPUTracking/Base/GPUReconstructionCPU.h index b2a523a9084da..3fc4ef8f14f8d 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.h +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.h @@ -316,8 +316,7 @@ HighResTimer& GPUReconstructionCPU::getKernelTimer(RecoStep step, int num, size_ static int id = getNextTimerId(); timerMeta* timer = getTimerById(id); if (timer == nullptr) { - int max = step == GPUCA_RECO_STEP::NoRecoStep || step == GPUCA_RECO_STEP::TPCSliceTracking || step == GPUCA_RECO_STEP::TPCClusterFinding ? NSLICES : 1; - timer = insertTimer(id, GetKernelName(), J, max, 0, step); + timer = insertTimer(id, GetKernelName(), J, NSLICES, 0, step); } if (addMemorySize) { timer->memSize += addMemorySize; From 68bfc3b9b5facc731834ecfe65e9ab05ecbed49e Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 21:07:17 +0200 Subject: [PATCH 20/22] GPU: Use at least parallel thrust sort for track ordering --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 50 +++++++++-------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index e595e64c19809..e818fe03be22c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1192,34 +1192,6 @@ struct GPUTPCGMMerger_CompareClusterIds { } }; -struct GPUTPCGMMerger_CompareTracksAttachWeight { - const GPUTPCGMMergedTrack* const mCmp; - GPUd() GPUTPCGMMerger_CompareTracksAttachWeight(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {} - GPUd() bool operator()(const int aa, const int bb) - { - const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa]; - const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb]; - return (CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt())); - } -}; - -struct GPUTPCGMMerger_CompareTracksProcess { - const GPUTPCGMMergedTrack* const mCmp; - GPUd() GPUTPCGMMerger_CompareTracksProcess(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {} - GPUd() bool operator()(const int aa, const int bb) - { - const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa]; - const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb]; - if (a.CCE() != b.CCE()) { - return a.CCE() > b.CCE(); - } - if (a.Legs() != b.Legs()) { - return a.Legs() > b.Legs(); - } - return a.NClusters() > b.NClusters(); - } -}; - GPUd() void GPUTPCGMMerger::LinkGlobalTracks(int nBlocks, int nThreads, int iBlock, int iThread) { for (int itr = SliceTrackInfoGlobalFirst(0) + iBlock * nThreads + iThread; itr < SliceTrackInfoGlobalLast(NSLICES - 1); itr += nThreads * nBlocks) { @@ -1536,7 +1508,19 @@ GPUd() void GPUTPCGMMerger::SortTracksPrepare(int nBlocks, int nThreads, int iBl GPUd() void GPUTPCGMMerger::SortTracks(int nBlocks, int nThreads, int iBlock, int iThread) { - GPUCommonAlgorithm::sort(mTrackOrderProcess, mTrackOrderProcess + mMemory->nOutputTracks, GPUTPCGMMerger_CompareTracksProcess(mOutputTracks)); + auto comp = [cmp = mOutputTracks](const int aa, const int bb) { + const GPUTPCGMMergedTrack& GPUrestrict() a = cmp[aa]; + const GPUTPCGMMergedTrack& GPUrestrict() b = cmp[bb]; + if (a.CCE() != b.CCE()) { + return a.CCE() > b.CCE(); + } + if (a.Legs() != b.Legs()) { + return a.Legs() > b.Legs(); + } + return a.NClusters() > b.NClusters(); + }; + + GPUCommonAlgorithm::sortDeviceDynamic(mTrackOrderProcess, mTrackOrderProcess + mMemory->nOutputTracks, comp); } GPUd() void GPUTPCGMMerger::PrepareClustersForFit0(int nBlocks, int nThreads, int iBlock, int iThread) @@ -1550,7 +1534,13 @@ GPUd() void GPUTPCGMMerger::PrepareClustersForFit0(int nBlocks, int nThreads, in GPUd() void GPUTPCGMMerger::SortTracksQPt(int nBlocks, int nThreads, int iBlock, int iThread) { unsigned int* trackSort = (unsigned int*)mTmpMem; - GPUCommonAlgorithm::sort(trackSort, trackSort + mMemory->nOutputTracks, GPUTPCGMMerger_CompareTracksAttachWeight(mOutputTracks)); + auto comp = [cmp = mOutputTracks](const int aa, const int bb) { + const GPUTPCGMMergedTrack& GPUrestrict() a = cmp[aa]; + const GPUTPCGMMergedTrack& GPUrestrict() b = cmp[bb]; + return (CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt())); + }; + + GPUCommonAlgorithm::sortDeviceDynamic(trackSort, trackSort + mMemory->nOutputTracks, comp); } GPUd() void GPUTPCGMMerger::PrepareClustersForFit1(int nBlocks, int nThreads, int iBlock, int iThread) From 517547ebc3a9aecd62ae6cf295dedcea9758ef2d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 22:19:30 +0200 Subject: [PATCH 21/22] GPU: Correct reporting for number of events used in benchmark statistic in standalone tool --- GPU/GPUTracking/Base/GPUReconstruction.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.cxx | 4 ++-- GPU/GPUTracking/Standalone/standalone.cxx | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstruction.h b/GPU/GPUTracking/Base/GPUReconstruction.h index 745f5c28813ef..3a9b1d70e39e0 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.h +++ b/GPU/GPUTracking/Base/GPUReconstruction.h @@ -165,6 +165,7 @@ class GPUReconstruction void PrepareEvent(); virtual int RunChains() = 0; unsigned int getNEventsProcessed() { return mNEventsProcessed; } + unsigned int getNEventsProcessedInStat() { return mStatNEvents; } virtual int registerMemoryForGPU(const void* ptr, size_t size) = 0; virtual int unregisterMemoryForGPU(const void* ptr) = 0; diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index e546f6eeb1117..704f2fee9110c 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1854,9 +1854,8 @@ int GPUChainTracking::RunChain() } } static HighResTimer timerTracking, timerMerger, timerQA, timerTransform, timerCompression, timerClusterer; - int nCount = mRec->getNEventsProcessed(); if (GetDeviceProcessingSettings().debugLevel >= 6) { - mDebugFile << "\n\nProcessing event " << nCount << std::endl; + mDebugFile << "\n\nProcessing event " << mRec->getNEventsProcessed() << std::endl; } if (mRec->slavesExist() && mRec->IsGPU()) { const auto threadContext = GetThreadContext(); @@ -1930,6 +1929,7 @@ int GPUChainTracking::RunChain() } if (GetDeviceProcessingSettings().debugLevel >= 0) { + int nCount = mRec->getNEventsProcessedInStat(); char nAverageInfo[16] = ""; if (nCount > 1) { sprintf(nAverageInfo, " (%d)", nCount); diff --git a/GPU/GPUTracking/Standalone/standalone.cxx b/GPU/GPUTracking/Standalone/standalone.cxx index a088dc79c45dd..e55db5d61d3b5 100644 --- a/GPU/GPUTracking/Standalone/standalone.cxx +++ b/GPU/GPUTracking/Standalone/standalone.cxx @@ -701,6 +701,7 @@ int main(int argc, char** argv) rec->SetResetTimers(j1 < configStandalone.runsInit); if (configStandalone.testSyncAsync) { + recAsync->SetResetTimers(j1 < configStandalone.runsInit); printf("Running synchronous phase\n"); } chainTracking->mIOPtrs = ioPtrSave; From a9e83f11c1c9502e2c53aae319ff77e2b30dfce3 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 May 2020 22:47:35 +0200 Subject: [PATCH 22/22] GPU: Read counters from GPU memory not host --- .../DataCompression/GPUTPCCompressionKernels.cxx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index 263fd03627542..60f6404c6113b 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -329,7 +329,7 @@ GPUdii() void GPUTPCCompressionKernels::ThreadnSliceRowClusters, compressor.mPtrs.nSliceRowClusters, compressor.NSLICES * GPUCA_ROW_COUNT * sizeof(compressor.mOutput->nSliceRowClusters[0])); - compressorMemcpy(compressor.mOutput->nTrackClusters, compressor.mPtrs.nTrackClusters, compressor.mOutput->nTracks * sizeof(compressor.mOutput->nTrackClusters[0])); + compressorMemcpy(compressor.mOutput->nTrackClusters, compressor.mPtrs.nTrackClusters, compressor.mMemory->nStoredTracks * sizeof(compressor.mOutput->nTrackClusters[0])); unsigned int offset = 0; for (unsigned int i = 0; i < compressor.NSLICES; i++) { for (unsigned int j = 0; j < GPUCA_ROW_COUNT; j++) { @@ -344,7 +344,7 @@ GPUdii() void GPUTPCCompressionKernels::ThreadnTracks; i++) { + for (unsigned int i = 0; i < compressor.mMemory->nStoredTracks; i++) { compressorMemcpy(compressor.mOutput->qTotA + offset, compressor.mPtrs.qTotA + compressor.mAttachedClusterFirstIndex[i], compressor.mPtrs.nTrackClusters[i] * sizeof(compressor.mOutput->qTotA[0])); compressorMemcpy(compressor.mOutput->qMaxA + offset, compressor.mPtrs.qMaxA + compressor.mAttachedClusterFirstIndex[i], compressor.mPtrs.nTrackClusters[i] * sizeof(compressor.mOutput->qMaxA[0])); compressorMemcpy(compressor.mOutput->flagsA + offset, compressor.mPtrs.flagsA + compressor.mAttachedClusterFirstIndex[i], compressor.mPtrs.nTrackClusters[i] * sizeof(compressor.mOutput->flagsA[0])); @@ -358,9 +358,9 @@ GPUdii() void GPUTPCCompressionKernels::ThreadtimeResA + offset - i, compressor.mPtrs.timeResA + compressor.mAttachedClusterFirstIndex[i] + 1, (compressor.mPtrs.nTrackClusters[i] - 1) * sizeof(compressor.mOutput->timeResA[0])); offset += compressor.mPtrs.nTrackClusters[i]; } - compressorMemcpy(compressor.mOutput->qPtA, compressor.mPtrs.qPtA, compressor.mOutput->nTracks * sizeof(compressor.mOutput->qPtA[0])); - compressorMemcpy(compressor.mOutput->rowA, compressor.mPtrs.rowA, compressor.mOutput->nTracks * sizeof(compressor.mOutput->rowA[0])); - compressorMemcpy(compressor.mOutput->sliceA, compressor.mPtrs.sliceA, compressor.mOutput->nTracks * sizeof(compressor.mOutput->sliceA[0])); - compressorMemcpy(compressor.mOutput->timeA, compressor.mPtrs.timeA, compressor.mOutput->nTracks * sizeof(compressor.mOutput->timeA[0])); - compressorMemcpy(compressor.mOutput->padA, compressor.mPtrs.padA, compressor.mOutput->nTracks * sizeof(compressor.mOutput->padA[0])); + compressorMemcpy(compressor.mOutput->qPtA, compressor.mPtrs.qPtA, compressor.mMemory->nStoredTracks * sizeof(compressor.mOutput->qPtA[0])); + compressorMemcpy(compressor.mOutput->rowA, compressor.mPtrs.rowA, compressor.mMemory->nStoredTracks * sizeof(compressor.mOutput->rowA[0])); + compressorMemcpy(compressor.mOutput->sliceA, compressor.mPtrs.sliceA, compressor.mMemory->nStoredTracks * sizeof(compressor.mOutput->sliceA[0])); + compressorMemcpy(compressor.mOutput->timeA, compressor.mPtrs.timeA, compressor.mMemory->nStoredTracks * sizeof(compressor.mOutput->timeA[0])); + compressorMemcpy(compressor.mOutput->padA, compressor.mPtrs.padA, compressor.mMemory->nStoredTracks * sizeof(compressor.mOutput->padA[0])); }