Add option to register only selected shm segment IDs on the GPU, to speed up workflow startup

Author: davidrohr

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -416,6 +416,7 @@ AddOption(mutexMemReg, bool, false, "", 0, "Global mutex to serialize GPU memory
 AddOption(printSettings, int, 0, "", 0, "Print all settings", def(1))
 AddOption(gpuDisplayfilterMacro, std::string, "", "", 0, "File name of ROOT macro for GPU display filter")
 AddOption(benchmarkMemoryRegistration, bool, false, "", 0, "Time-benchmark for memory registration")
+AddOption(registerSelectedSegmentIds, int, -1, "", 0, "Register only a specific managed shm segment id (-1 = all)")
 EndConfig()
 #endif // GPUCA_O2_LIB
 #endif // !GPUCA_GPUCODE_DEVICE

GPU/Workflow/src/GPUWorkflowSpec.cxx

@@ -309,38 +309,42 @@ DataProcessorSpec getGPURecoWorkflowSpec(gpuworkflow::CompletionPolicyData* poli
 
     auto& callbacks = ic.services().get<CallbackService>();
     callbacks.set(CallbackService::Id::RegionInfoCallback, [&processAttributes, confParam](FairMQRegionInfo const& info) {
-      if (info.size) {
-        int fd = 0;
-        if (confParam.mutexMemReg) {
-          mode_t mask = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
-          fd = open("/tmp/o2_gpu_memlock_mutex.lock", O_RDWR | O_CREAT | O_CLOEXEC, mask);
-          if (fd == -1) {
-            throw std::runtime_error("Error opening lock file");
-          }
-          fchmod(fd, mask);
-          if (lockf(fd, F_LOCK, 0)) {
-            throw std::runtime_error("Error locking file");
-          }
-        }
-        auto& tracker = processAttributes->tracker;
-        std::chrono::time_point<std::chrono::high_resolution_clock> start, end;
-        if (confParam.benchmarkMemoryRegistration) {
-          start = std::chrono::high_resolution_clock::now();
-        }
-        if (tracker->registerMemoryForGPU(info.ptr, info.size)) {
-          throw std::runtime_error("Error registering memory for GPU");
+      if (info.size == 0) {
+        return;
+      }
+      if (confParam.registerSelectedSegmentIds != -1 && info.managed && info.id != confParam.registerSelectedSegmentIds) {
+        return;
+      }
+      int fd = 0;
+      if (confParam.mutexMemReg) {
+        mode_t mask = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
+        fd = open("/tmp/o2_gpu_memlock_mutex.lock", O_RDWR | O_CREAT | O_CLOEXEC, mask);
+        if (fd == -1) {
+          throw std::runtime_error("Error opening lock file");
         }
-        if (confParam.benchmarkMemoryRegistration) {
-          end = std::chrono::high_resolution_clock::now();
-          std::chrono::duration<double> elapsed_seconds = end - start;
-          LOG(info) << "Memory registration time (0x" << info.ptr << ", " << info.size << " bytes): " << elapsed_seconds.count() << " s";
+        fchmod(fd, mask);
+        if (lockf(fd, F_LOCK, 0)) {
+          throw std::runtime_error("Error locking file");
         }
-        if (confParam.mutexMemReg) {
-          if (lockf(fd, F_ULOCK, 0)) {
-            throw std::runtime_error("Error unlocking file");
-          }
-          close(fd);
+      }
+      auto& tracker = processAttributes->tracker;
+      std::chrono::time_point<std::chrono::high_resolution_clock> start, end;
+      if (confParam.benchmarkMemoryRegistration) {
+        start = std::chrono::high_resolution_clock::now();
+      }
+      if (tracker->registerMemoryForGPU(info.ptr, info.size)) {
+        throw std::runtime_error("Error registering memory for GPU");
+      }
+      if (confParam.benchmarkMemoryRegistration) {
+        end = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> elapsed_seconds = end - start;
+        LOG(info) << "Memory registration time (0x" << info.ptr << ", " << info.size << " bytes): " << elapsed_seconds.count() << " s";
+      }
+      if (confParam.mutexMemReg) {
+        if (lockf(fd, F_ULOCK, 0)) {
+          throw std::runtime_error("Error unlocking file");
         }
+        close(fd);
       }
     });
 

prodtests/full-system-test/dpl-workflow.sh

@@ -74,7 +74,7 @@ if [[ $EPNSYNCMODE == 1 ]]; then
 elif [[ "0$ENABLE_METRICS" != "01" ]]; then
   ARGS_ALL+=" --monitoring-backend no-op://"
 fi
-( [[ $EXTINPUT == 1 ]] || [[ $NUMAGPUIDS == 1 ]] ) && ARGS_ALL+=" --no-cleanup"
+( [[ $EXTINPUT == 1 ]] || [[ $NUMAGPUIDS != 0 ]] ) && ARGS_ALL+=" --no-cleanup"
 ( [[ $GPUTYPE != "CPU" ]] || [[ $OPTIMIZED_PARALLEL_ASYNC != 0 ]] ) && ARGS_ALL+=" --shm-mlock-segment-on-creation 1"
 [[ $SHMTHROW == 0 ]] && ARGS_ALL+=" --shm-throw-bad-alloc 0"
 [[ $NORATELOG == 1 ]] && ARGS_ALL+=" --fairmq-rate-logging 0"
@@ -149,6 +149,10 @@ if [[ $EPNSYNCMODE == 1 ]]; then
   if [[ $EXTINPUT == 1 ]] && [[ $GPUTYPE != "CPU" ]] && [[ -z "$GPU_NUM_MEM_REG_CALLBACKS" ]]; then GPU_NUM_MEM_REG_CALLBACKS=4; fi
 fi
 
+if [[ $GPUTYPE != "CPU" && $NUMAGPUIDS != 0 ]]; then
+  GPU_CONFIG_KEY+="GPU_global.registerSelectedSegmentIds=$NUMAID;"
+fi
+
 if [[ $GPUTYPE == "HIP" ]]; then
   if [[ $NUMAID == 0 ]] || [[ $NUMAGPUIDS == 0 ]]; then
     export TIMESLICEOFFSET=0
@@ -237,7 +241,7 @@ N_TPCTRK=$NGPUS
 if [[ $OPTIMIZED_PARALLEL_ASYNC != 0 ]]; then
   # Tuned multiplicities for async Pb-Pb processing
   if [[ $SYNCMODE == "1" ]]; then echo "Must not use OPTIMIZED_PARALLEL_ASYNC with GPU or SYNCMODE" 1>&2; exit 1; fi
-  if [[ $NUMAGPUIDS == 1 ]]; then N_NUMAFACTOR=1; else N_NUMAFACTOR=2; fi
+  if [[ $NUMAGPUIDS != 0 ]]; then N_NUMAFACTOR=1; else N_NUMAFACTOR=2; fi
   GPU_CONFIG_KEY+="GPU_proc.ompThreads=6;"
   TRD_CONFIG_KEY+="GPU_proc.ompThreads=2;"
   if [[ $GPUTYPE == "CPU" ]]; then
@@ -271,7 +275,7 @@ elif [[ $EPNPIPELINES != 0 ]]; then
     GPU_CONFIG_KEY+="GPU_proc.ompThreads=4;"
   fi
   # Scale some multiplicities with the number of nodes
-  RECO_NUM_NODES_WORKFLOW_CMP=$((($RECO_NUM_NODES_WORKFLOW > 15 ? $RECO_NUM_NODES_WORKFLOW : 15) * ($NUMAGPUIDS == 1 ? 2 : 1))) # Limit the lower scaling factor, multiply by 2 if we have 2 NUMA domains
+  RECO_NUM_NODES_WORKFLOW_CMP=$((($RECO_NUM_NODES_WORKFLOW > 15 ? $RECO_NUM_NODES_WORKFLOW : 15) * ($NUMAGPUIDS != 0 ? 2 : 1))) # Limit the lower scaling factor, multiply by 2 if we have 2 NUMA domains
   N_ITSRAWDEC=$(math_max $((3 * 60 / $RECO_NUM_NODES_WORKFLOW_CMP)) ${N_ITSRAWDEC:-1}) # This means, if we have 60 EPN nodes, we need at least 3 ITS RAW decoders
   N_MFTRAWDEC=$(math_max $((3 * 60 / $RECO_NUM_NODES_WORKFLOW_CMP)) ${N_MFTRAWDEC:-1})
   N_ITSTRK=$(math_max $((1 * 200 / $RECO_NUM_NODES_WORKFLOW_CMP)) ${N_ITSTRK:-1})