Merge branch 'dev' into rof-processing

Author: mpuccio

CCDB/include/CCDB/BasicCCDBManager.h

@@ -48,6 +48,8 @@ class CCDBManagerInstance
     std::string uuid;
     long startvalidity = 0;
     long endvalidity = -1;
+    size_t minSize = -1ULL;
+    size_t maxSize = 0;
     int queries = 0;
     int fetches = 0;
     int failures = 0;
@@ -176,7 +178,9 @@ class CCDBManagerInstance
 
   std::string getSummaryString() const;
 
-  void endOfStream();
+  size_t getFetchedSize() const { return mFetchedSize; }
+
+  void report(bool longrep = false);
 
  private:
   // method to print (fatal) error
@@ -190,10 +194,11 @@ class CCDBManagerInstance
   bool mCanDefault = false;                             // whether default is ok --> useful for testing purposes done standalone/isolation
   bool mCachingEnabled = true;                          // whether caching is enabled
   bool mCheckObjValidityEnabled = false;                // wether the validity of cached object is checked before proceeding to a CCDB API query
+  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
   long mCreatedNotAfter = 0;                            // upper limit for object creation timestamp (TimeMachine mode) - If-Not-After HTTP header
   long mCreatedNotBefore = 0;                           // lower limit for object creation timestamp (TimeMachine mode) - If-Not-Before HTTP header
   long mTimerMS = 0;                                    // timer for queries
-  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
+  size_t mFetchedSize = 0;                              // total fetched size
   int mQueries = 0;                                     // total number of object queries
   int mFetches = 0;                                     // total number of succesful fetches from CCDB
   int mFailures = 0;                                    // total number of failed fetches
@@ -218,11 +223,16 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
       mFailures++;
     } else {
       mFetches++;
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+      }
     }
   } else {
     auto& cached = mCache[path];
+    cached.queries++;
     if (mCheckObjValidityEnabled && cached.isValid(timestamp)) {
-      cached.queries++;
       return reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
     }
     ptr = mCCDBAccessor.retrieveFromTFileAny<T>(path, mMetaData, timestamp, &mHeaders, cached.uuid,
@@ -254,6 +264,13 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
       } catch (std::exception const& e) {
         reportFatal("Failed to read validity from CCDB response (Valid-From :  " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")"));
       }
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+        cached.minSize = std::min(s, cached.minSize);
+        cached.maxSize = std::max(s, cached.minSize);
+      }
     } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set
       cached.failures++;
       cached.clear(); // in case of any error clear cache for this object

CCDB/include/CCDB/CcdbApi.h

@@ -542,7 +542,7 @@ class CcdbApi //: public DatabaseInterface
    * @param tcl The TClass object describing the serialized type
    * @return raw pointer to created object
    */
-  void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo) const;
+  void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map<string, string>* headers) const;
 
   // initialize the TGrid (Alien connection)
   bool initTGrid() const;

CCDB/src/BasicCCDBManager.cxx

@@ -54,7 +54,7 @@ std::pair<int64_t, int64_t> CCDBManagerInstance::getRunDuration(int runnumber, b
 
 std::string CCDBManagerInstance::getSummaryString() const
 {
-  std::string res = fmt::format("{} queries", mQueries);
+  std::string res = fmt::format("{} queries, {} bytes", mQueries, fmt::group_digits(mFetchedSize));
   if (mCachingEnabled) {
     res += fmt::format(" for {} objects", mCache.size());
   }
@@ -72,9 +72,15 @@ std::string CCDBManagerInstance::getSummaryString() const
   return res;
 }
 
-void CCDBManagerInstance::endOfStream()
+void CCDBManagerInstance::report(bool longrep)
 {
   LOG(info) << "CCDBManager summary: " << getSummaryString();
+  if (longrep && mCachingEnabled) {
+    LOGP(info, "CCDB cache miss/hit/failures");
+    for (const auto& obj : mCache) {
+      LOGP(info, "  {}: {}/{}/{} ({}-{} bytes)", obj.first, obj.second.fetches, obj.second.queries - obj.second.fetches - obj.second.failures, obj.second.failures, obj.second.minSize, obj.second.maxSize);
+    }
+  }
 }
 
 } // namespace ccdb

CCDB/src/CCDBDownloader.cxx

@@ -529,6 +529,10 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
             (*requestData->headers)["Error"] = "An error occurred during retrieval";
           }
           LOGP(alarm, "Curl request to {}, response code: {}", url, httpCode);
+        } else {
+          if (requestData->headers && requestData->headers->find("fileSize") == requestData->headers->end()) {
+            (*requestData->headers)["fileSize"] = fmt::format("{}", requestData->hoPair.object ? requestData->hoPair.object->size() : 0);
+          }
         }
         --(*performData->requestsLeft);
         delete requestData;

CCDB/src/CcdbApi.cxx

@@ -832,6 +832,9 @@ void* CcdbApi::extractFromLocalFile(std::string const& filename, std::type_info
     if ((isSnapshotMode() || mPreferSnapshotCache) && headers->find("ETag") == headers->end()) { // generate dummy ETag to profit from the caching
       (*headers)["ETag"] = filename;
     }
+    if (headers->find("fileSize") == headers->end()) {
+      (*headers)["fileSize"] = fmt::format("{}", f.GetEND());
+    }
   }
   return extractFromTFile(f, tcl);
 }
@@ -857,7 +860,7 @@ bool CcdbApi::initTGrid() const
   return mAlienInstance != nullptr;
 }
 
-void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo) const
+void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map<string, string>* headers) const
 {
   if ((url.find("alien:/", 0) != std::string::npos) && !initTGrid()) {
     return nullptr;
@@ -867,6 +870,9 @@ void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info
   if (memfile) {
     auto cl = tinfo2TClass(tinfo);
     auto content = extractFromTFile(*memfile, cl);
+    if (headers && headers->find("fileSize") == headers->end()) {
+      (*headers)["fileSize"] = fmt::format("{}", memfile->GetEND());
+    }
     delete memfile;
     return content;
   }
@@ -902,7 +908,7 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con
 
   // let's see first of all if the url is something specific that curl cannot handle
   if ((url.find("alien:/", 0) != std::string::npos) || (url.find("file:/", 0) != std::string::npos)) {
-    return downloadFilesystemContent(url, tinfo);
+    return downloadFilesystemContent(url, tinfo, headers);
   }
   // add other final cases here
   // example root://
@@ -933,6 +939,9 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con
     if (200 <= response_code && response_code < 300) {
       // good response and the content is directly provided and should have been dumped into "chunk"
       content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, tinfo);
+      if (headers && headers->find("fileSize") == headers->end()) {
+        (*headers)["fileSize"] = fmt::format("{}", chunk.size);
+      }
     } else if (response_code == 304) {
       // this means the object exist but I am not serving
       // it since it's already in your possession
@@ -1773,6 +1782,9 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& p
     if ((isSnapshotMode() || mPreferSnapshotCache) && localHeaders->find("ETag") == localHeaders->end()) { // generate dummy ETag to profit from the caching
       (*localHeaders)["ETag"] = path;
     }
+    if (localHeaders->find("fileSize") == localHeaders->end()) {
+      (*localHeaders)["fileSize"] = fmt::format("{}", memFile.GetEND());
+    }
   }
   return;
 }

DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h

@@ -306,8 +306,8 @@ template <typename value_T>
 template <typename T>
 GPUdi() auto TrackParametrizationWithError<value_T>::getPredictedChi2(const BaseCluster<T>& p) const -> value_t
 {
-  const dim2_t pyz = {p.getY(), p.getZ()};
-  const dim3_t cov = {p.getSigmaY2(), p.getSigmaYZ(), p.getSigmaZ2()};
+  const dim2_t pyz = {value_T(p.getY()), value_T(p.getZ())};
+  const dim3_t cov = {value_T(p.getSigmaY2()), value_T(p.getSigmaYZ()), value_T(p.getSigmaZ2())};
   return getPredictedChi2(pyz, cov);
 }
 
@@ -337,8 +337,8 @@ template <typename value_T>
 template <typename T>
 GPUdi() bool TrackParametrizationWithError<value_T>::update(const BaseCluster<T>& p)
 {
-  const dim2_t pyz = {p.getY(), p.getZ()};
-  const dim3_t cov = {p.getSigmaY2(), p.getSigmaYZ(), p.getSigmaZ2()};
+  const dim2_t pyz = {value_T(p.getY()), value_T(p.getZ())};
+  const dim3_t cov = {value_T(p.getSigmaY2()), value_T(p.getSigmaYZ()), value_T(p.getSigmaZ2())};
   return update(pyz, cov);
 }
 

Detectors/Align/CMakeLists.txt

@@ -9,7 +9,7 @@
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
-add_compile_options(-O0 -g -fPIC)
+#add_compile_options(-O0 -g -fPIC)
 
 o2_add_library(Align
                SOURCES  src/GeometricalConstraint.cxx

Detectors/Align/Workflow/src/BarrelAlignmentSpec.cxx

@@ -85,9 +85,13 @@ class BarrelAlignmentSpec : public Task
                   CheckConstaints = 0x1 << 1,
                   GenPedeFiles = 0x1 << 2,
                   LabelPedeResults = 0x1 << 3 };
-  BarrelAlignmentSpec(GTrackID::mask_t srcMP, std::shared_ptr<DataRequest> dr, std::shared_ptr<o2::base::GRPGeomRequest> ggrec,
+  BarrelAlignmentSpec(GTrackID::mask_t srcMP, std::shared_ptr<DataRequest> dr, std::shared_ptr<o2::base::GRPGeomRequest> ggrec, const o2::tpc::CorrectionMapsLoaderGloOpts& tpcOpt,
                       DetID::mask_t detmask, bool cosmic, int postprocess, bool useMC, bool loadTPCCalib)
-    : mDataRequest(dr), mGRPGeomRequest(ggrec), mMPsrc{srcMP}, mDetMask{detmask}, mCosmic(cosmic), mPostProcessing(postprocess), mUseMC(useMC), mLoadTPCCalib(loadTPCCalib) {}
+    : mDataRequest(dr), mGRPGeomRequest(ggrec), mMPsrc{srcMP}, mDetMask{detmask}, mCosmic(cosmic), mPostProcessing(postprocess), mUseMC(useMC), mLoadTPCCalib(loadTPCCalib)
+  {
+    mTPCCorrMapsLoader.setLumiScaleType(tpcOpt.lumiType);
+    mTPCCorrMapsLoader.setLumiScaleMode(tpcOpt.lumiMode);
+  }
   ~BarrelAlignmentSpec() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
@@ -401,7 +405,7 @@ DataProcessorSpec getBarrelAlignmentSpec(GTrackID::mask_t srcMP, GTrackID::mask_
     "barrel-alignment",
     dataRequest->inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask<BarrelAlignmentSpec>(srcMP, dataRequest, ccdbRequest, dets, enableCosmic, postprocess, useMC, loadTPCCalib)},
+    AlgorithmSpec{adaptFromTask<BarrelAlignmentSpec>(srcMP, dataRequest, ccdbRequest, sclOpts, dets, enableCosmic, postprocess, useMC, loadTPCCalib)},
     opts};
 }
 

Detectors/Align/include/Align/AlignConfig.h

@@ -52,6 +52,13 @@ struct AlignConfig : public o2::conf::ConfigurableParamHelper<AlignConfig> {
   int minTRDTracklets = 3; // min TRD tracklets to accept the track
   int minTPCClusters = 10; // discard tracks with less clusters
   int minTOFClusters = 1;  // min TOF clusters to accept track
+  int maxTPCRowsCombined = 1;        // allow combining clusters on so many rows to a single cluster
+  int discardEdgePadrows = 3;        // discard padrow if its distance to stack edge padrow < this
+  float discardSectorEdgeDepth = 2.; // discard clusters too close to the sector edge
+  float ITSOverlapMargin = 0.15;     // consider for overlaps only clusters within this marging from the chip edge (in cm)
+  float ITSOverlapMaxChi2 = 16;      // max chi2 between track and overlapping cluster
+  int ITSOverlapEdgeRows = 1;        // require clusters to not have pixels closer than this distance from the edge
+  float ITSOverlapMaxDZ = 0.3;       // max difference in Z for clusters on overlapping ITS chips to consider as candidate for a double hit
 
   int minPointTotalCosm = 4;      // total min number of alignment point to account cosmic track
   int minDetectorsCosm = 1;       // min number of detectors per cosmic track

Detectors/Align/include/Align/AlignableDetectorITS.h

@@ -21,6 +21,7 @@
 #include "Align/utils.h"
 #include "ReconstructionDataFormats/TrackParametrizationWithError.h"
 #include "ReconstructionDataFormats/BaseCluster.h"
+#include "ITSMFTReconstruction/ChipMappingITS.h"
 
 namespace o2
 {
@@ -39,6 +40,11 @@ class AlignableDetectorITS : public AlignableDetector
  public:
   //
   using ClusterD = o2::BaseCluster<double>;
+  using OVL = o2::itsmft::ChipMappingITS::Overlaps;
+  enum EdgeFlags : int8_t { NONE = -1,
+                            LowRow = OVL::LowRow,
+                            HighRow = OVL::HighRow,
+                            Biased = 2 };
   AlignableDetectorITS() = default; // RS FIXME do we need default c-tor?
   AlignableDetectorITS(Controller* ctr);
   ~AlignableDetectorITS() override = default;
@@ -71,6 +77,9 @@ class AlignableDetectorITS : public AlignableDetector
  protected:
   //
   std::vector<ClusterD> mITSClustersArray;
+  std::vector<int> mOverlapCandidateID; // pool of indices for potentially overlapping clusters
+  std::vector<int> mOverlapClusRef;     // 1st entry in mOverlapCandidateID for the overlapping cluster indices of each cluster
+  std::vector<o2::itsmft::ChipMappingITS::Overlaps> mOverlaps;
   const o2::itsmft::TopologyDictionary* mITSDict{nullptr}; // cluster patterns dictionary
   //
   ClassDefOverride(AlignableDetectorITS, 1);