Macro to extract per-detector CTF dictionaries good for CCDB

Author: shahor02

Detectors/CTF/README.md

@@ -18,7 +18,7 @@ o2-its-reco-workflow --entropy-encoding | o2-ctf-writer-workflow --onlyDet ITS
 
 For the storage optimization reason one can request multiple CTFs stored in the same output file (as entries of the `ctf` tree):
 ```bash
-o2-ctf-writer --min-file-size <min> --max-file-size <max> ...
+o2-ctf-writer-workflow --min-file-size <min> --max-file-size <max> ...
 ```
 will accumulate CTFs in entries of the same tree/file until its size fits exceeds `min` and does not exceed `max` (`max` check is disabled if `max<=min`) or EOS received.
 The `--max-file-size` limit will be ignored if the very first CTF already exceeds it.
@@ -42,6 +42,14 @@ In the dictionaries creation mode their data are accumulated over all CTFs procs
 
 Option `--ctf-dict-dir <dir>` can be provided to indicate the (existing) directory where the dictionary will be stored.
 
+The external dictionaries created by the `o2-ctf-writer-workflow` containes a TTree (one for all participating detectos or single file per detector if `--dict-per-det` was provided). Since the TTrees cannot be used with CcdbAPI, one can
+run the macro `O2/Detectors/CTF/utils/CTFdict2CCDBfiles.C` (installed to $O2_ROOT/share/macro/CTFdict2CCDBfiles.C) which extracts the dictionary for every detector into separate file containing plain `vector<char>`. These files can be directly
+uploaded to CCDB and accessed via `CcdbAPI` (the reference of the vector should be provided to corresponding detector CTFCoder::createCoders method to build the run-time dictionary). These files can be also used as per-detector command-line
+parameters, on the same footing as tree-based dictionaries, e.g.
+```
+o2-ctf-reader-workflow --ctf-input input.lst --onlyDet ITS,TPC,TOF --its-entropy-decoder ' --ctf-dict ctfdict_ITS_v1.0_1626472046.root' --tpc-entropy-decoder ' --ctf-dict ctfdict_TPC_v1.0_1626472048.root' --tof-entropy-decoder  ' --ctf-dict ctfdict_TOF_v1.0_1626472048.root'
+```
+
 ## CTF reader workflow
 
 `o2-ctf-reader-workflow` should be the 1st workflow in the piped chain of CTF processing.

Detectors/CTF/utils/CMakeLists.txt

@@ -15,6 +15,7 @@
 
 install(FILES extractCTF.C
               dumpCTF.C
+              CTFdict2CCDBfiles.C
         DESTINATION share/macro/)
 
 o2_add_test_root_macro(extractCTF.C
@@ -24,3 +25,7 @@ o2_add_test_root_macro(extractCTF.C
 o2_add_test_root_macro(dumpCTF.C
                        PUBLIC_LINK_LIBRARIES O2::CTFWorkflow
                        LABELS ctf)
+
+o2_add_test_root_macro(CTFdict2CCDBfiles.C
+                       PUBLIC_LINK_LIBRARIES O2::CTFWorkflow
+                       LABELS ctf COMPILE_ONLY)

Detectors/CTF/utils/CTFdict2CCDBfiles.C

@@ -0,0 +1,97 @@
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include "DetectorsBase/CTFCoderBase.h"
+#include "DetectorsCommonDataFormats/CTFDictHeader.h"
+#include "DetectorsCommonDataFormats/CTFHeader.h"
+#include "DetectorsCommonDataFormats/NameConf.h"
+#include "DataFormatsITSMFT/CTF.h"
+#include "DataFormatsTPC/CTF.h"
+#include "DataFormatsTRD/CTF.h"
+#include "DataFormatsFT0/CTF.h"
+#include "DataFormatsFV0/CTF.h"
+#include "DataFormatsFDD/CTF.h"
+#include "DataFormatsTOF/CTF.h"
+#include "DataFormatsMID/CTF.h"
+#include "DataFormatsMCH/CTF.h"
+#include "DataFormatsEMCAL/CTF.h"
+#include "DataFormatsPHOS/CTF.h"
+#include "DataFormatsCPV/CTF.h"
+#include "DataFormatsZDC/CTF.h"
+#include "DataFormatsHMP/CTF.h"
+#include "DataFormatsCTP/CTF.h"
+#endif
+
+using DetID = o2::detectors::DetID;
+
+template <typename T>
+bool readFromTree(TTree& tree, const std::string brname, T& dest, int ev = 0)
+{
+  auto* br = tree.GetBranch(brname.c_str());
+  if (br && br->GetEntries() > ev) {
+    auto* ptr = &dest;
+    br->SetAddress(&ptr);
+    br->GetEntry(ev);
+    br->ResetAddress();
+    return true;
+  }
+  return false;
+}
+
+template <typename C>
+void extractDictionary(TTree& tree, o2::detectors::DetID det, DetID::mask_t detMask)
+{
+  std::vector<char> bufVec;
+  if (!detMask[det]) {
+    return;
+  }
+  o2::ctf::CTFHeader ctfHeader;
+  readFromTree(tree, "CTFHeader", ctfHeader);
+  if (!ctfHeader.detectors[det]) {
+    LOGP(warning, "Dictionary for {} was requested but absent", det.getName());
+    return;
+  }
+  C::readFromTree(bufVec, tree, det.getName());
+  auto& dictHeader = static_cast<o2::ctf::CTFDictHeader&>(C::get(bufVec.data())->getHeader());
+  dictHeader.det = det; // impose detector, since in early versions it is absent
+  std::string outName = fmt::format("ctfdict_{}_v{}.{}_{}.root", det.getName(), int(dictHeader.majorVersion), int(dictHeader.minorVersion), dictHeader.dictTimeStamp);
+  TFile flout(outName.c_str(), "recreate");
+  flout.WriteObject(&bufVec, o2::base::NameConf::CCDBOBJECT.data());
+  flout.Close();
+  LOG(info) << "Wrote " << dictHeader.asString() << " to " << outName;
+}
+
+// This macro allows to convert tree-based CTF dictionary (produced by the ctf-writer-workflow) to per-detector files with plain vector, suitable for the CCDB use.
+void CTFdict2CCDBfiles(const std::string& fname = "ctf_dictionary.root", const std::string dets = "all")
+{
+  std::string allowedDetectors = "ITS,TPC,TRD,TOF,PHS,CPV,EMC,HMP,MFT,MCH,MID,ZDC,FT0,FV0,FDD,CTP";
+  auto detMask = DetID::getMask(dets) & DetID::getMask(allowedDetectors);
+
+  std::unique_ptr<TFile> dictFile(TFile::Open(fname.c_str()));
+  if (!dictFile) {
+    LOG(error) << "Failed to open CTF dictionary file " << fname;
+    return;
+  }
+  std::unique_ptr<TTree> tree((TTree*)dictFile->Get(std::string(o2::base::NameConf::CTFDICT).c_str()));
+  if (!tree) {
+    LOG(error) << "Did not find CTF dictionary tree in " << fname;
+    return;
+  }
+  extractDictionary<o2::itsmft::CTF>(*tree, DetID::ITS, detMask);
+  extractDictionary<o2::itsmft::CTF>(*tree, DetID::MFT, detMask);
+  extractDictionary<o2::emcal::CTF>(*tree, DetID::EMC, detMask);
+  extractDictionary<o2::hmpid::CTF>(*tree, DetID::HMP, detMask);
+  extractDictionary<o2::phos::CTF>(*tree, DetID::PHS, detMask);
+  extractDictionary<o2::tpc::CTF>(*tree, DetID::TPC, detMask);
+  extractDictionary<o2::trd::CTF>(*tree, DetID::TRD, detMask);
+  extractDictionary<o2::ft0::CTF>(*tree, DetID::FT0, detMask);
+  extractDictionary<o2::fv0::CTF>(*tree, DetID::FV0, detMask);
+  extractDictionary<o2::fdd::CTF>(*tree, DetID::FDD, detMask);
+  extractDictionary<o2::tof::CTF>(*tree, DetID::TOF, detMask);
+  extractDictionary<o2::mid::CTF>(*tree, DetID::MID, detMask);
+  extractDictionary<o2::mch::CTF>(*tree, DetID::MCH, detMask);
+  extractDictionary<o2::cpv::CTF>(*tree, DetID::CPV, detMask);
+  extractDictionary<o2::zdc::CTF>(*tree, DetID::ZDC, detMask);
+  extractDictionary<o2::ctp::CTF>(*tree, DetID::CTP, detMask);
+
+  tree.reset();
+  dictFile.reset();
+}