[EMCAL-834] Checking active links per trigger

For each trigger checking from which DDLs the
reconstruction received pages. If we don't receive
pages from all active links the page is discarded.

The link checker can be disabled via a corresponding
config option and is by default disabled in case of
running on simulated data.

Author: mfasDa

DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ErrorTypeFEE.h

@@ -57,6 +57,7 @@ class ErrorTypeFEE
     FIT_ERROR,         ///< Raw fit failed
     GEOMETRY_ERROR,    ///< Decoded position outside EMCAL
     GAIN_ERROR,        ///< Error due to gain type
+    LINK_ERROR,        ///< Error due to missing DDL links
     STU_ERROR,         ///< Error from STU data
     UNDEFINED          ///< Error source undefined
   };
@@ -98,6 +99,10 @@ class ErrorTypeFEE
   /// \param gainError Error code of the gain type error
   void setGainErrorType(int gainError) { setError(ErrorSource_t::GAIN_ERROR, gainError); }
 
+  /// \brief Set the error type as link error and store the error code
+  /// \param linkError Error code of the link error
+  void setLinkErrorTYpe(int linkError) { setError(ErrorSource_t::LINK_ERROR, linkError); }
+
   /// \brief Set the error as STU decoder error and store the error code
   /// \param gainError Error code of the STU decoder error
   void setSTUDecoderErrorType(int gainError) { setError(ErrorSource_t::STU_ERROR, gainError); }
@@ -162,6 +167,10 @@ class ErrorTypeFEE
   /// \return Error code (-1 in case the object is not a gain type error)
   int getGainTypeErrorType() const { return getRawErrorForType(ErrorSource_t::GAIN_ERROR); }
 
+  /// \brief Get the error code of the obect in case the object is a link error
+  /// \return Error code (-1 in case the object is not a gain type error)
+  int getLinkErrorType() const { return getRawErrorForType(ErrorSource_t::LINK_ERROR); }
+
   /// \brief Get the error code of the obect in case the object is a STU decoder error
   /// \return Error code (-1 in case the object is not a STU decoder error)
   int getSTUDecoderErrorType() const { return getRawErrorForType(ErrorSource_t::STU_ERROR); }

Detectors/EMCAL/workflow/include/EMCALWorkflow/RawToCellConverterSpec.h

@@ -239,6 +239,7 @@ class RawToCellConverterSpec : public framework::Task
   int mErrorMessagesSuppressed = 0;                                  ///< Counter of suppressed error messages
   int mMaxErrorMessages = 100;                                       ///< Max. number of error messages
   bool mMergeLGHG = true;                                            ///< Merge low and high gain cells
+  bool mActiveLinkCheck = true;                                      ///< Run check for active links
   bool mPrintTrailer = false;                                        ///< Print RCU trailer
   bool mDisablePedestalEvaluation = false;                           ///< Disable pedestal evaluation independent of settings in the RCU trailer
   bool mCreateRawDataErrors = false;                                 ///< Create raw data error objects for monitoring

Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx

@@ -31,6 +31,7 @@
 #include "DetectorsRaw/RDHUtils.h"
 #include "EMCALBase/Geometry.h"
 #include "EMCALBase/Mapper.h"
+#include "EMCALCalib/FeeDCS.h"
 #include "EMCALReconstruction/CaloFitResults.h"
 #include "EMCALReconstruction/Bunch.h"
 #include "EMCALReconstruction/CaloRawFitterStandard.h"
@@ -93,8 +94,11 @@ void RawToCellConverterSpec::init(framework::InitContext& ctx)
 
   mMergeLGHG = !ctx.options().get<bool>("no-mergeHGLG");
   mDisablePedestalEvaluation = ctx.options().get<bool>("no-evalpedestal");
+  mActiveLinkCheck = !ctx.options().get<bool>("no-checkactivelinks");
 
   LOG(info) << "Running gain merging mode: " << (mMergeLGHG ? "yes" : "no");
+  LOG(info) << "Checking for active links: " << (mActiveLinkCheck ? "yes" : "no");
+  LOG(info) << "Calculate pedestals:       " << (mDisablePedestalEvaluation ? "no" : "yes");
   LOG(info) << "Using L0LM delay: " << o2::ctp::TriggerOffsetsParam::Instance().LM_L0 << " BCs";
 
   mRawFitter->setAmpCut(mNoiseThreshold);
@@ -107,6 +111,9 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
   mCalibHandler->checkUpdates(ctx);
   updateCalibrationObjects();
 
+  // container with BCid and feeID
+  std::unordered_map<int64_t, std::bitset<46>> bcFreq;
+
   double timeshift = RecoParam::Instance().getCellTimeShiftNanoSec(); // subtract offset in ns in order to center the time peak around the nominal delay
   constexpr auto originEMC = o2::header::gDataOriginEMC;
   constexpr auto descRaw = o2::header::gDataDescriptionRawData;
@@ -181,6 +188,9 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
           continue;
         }
       }
+
+      bcFreq[currentIR.toLong()].set(feeID, true);
+
       // Correct the cell time for the bc mod 4 (LHC: 40 MHz clock - ALTRO: 10 MHz clock)
       // Convention: All times shifted with respect to BC % 4 = 0 for trigger BC
       // Attention: Correction only works for the permutation (0 1 2 3) of the BC % 4, if the permutation is
@@ -315,6 +325,43 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
     }
   }
 
+  if (mActiveLinkCheck) {
+    // build expected active mask from DCS
+    FeeDCS* feedcs = mCalibHandler->getFEEDCS();
+    auto list0 = feedcs->getDDLlist0();
+    auto list1 = feedcs->getDDLlist1();
+    //  links 21 and 39 do not exist, but they are set active in DCS
+    list0.set(21, false);
+    list1.set(7, false);
+    // must be 0x307FFFDFFFFF if all links are active
+    std::bitset<46> bitSetActiveLinks((list1.to_ullong() << 32) + list0.to_ullong());
+
+    // Check if we have received pages from all active links
+    // If not we cannot trust the timeframe and must send
+    // empty containers
+    bool hasMissingLinks = false;
+    for (const auto& [globalBC, activelinks] : bcFreq) {
+      if (activelinks != bitSetActiveLinks) {
+        hasMissingLinks = true;
+        LOG(error) << "Not all EMC active links contributed in global BCid=" << globalBC << ": mask=" << (activelinks ^ bitSetActiveLinks);
+        if (mCreateRawDataErrors) {
+          for (std::size_t ilink = 0; ilink < bitSetActiveLinks.size(); ilink++) {
+            if (!bitSetActiveLinks.test(ilink)) {
+              continue;
+            }
+            if (!activelinks.test(ilink)) {
+              mOutputDecoderErrors.emplace_back(ilink, ErrorTypeFEE::ErrorSource_t::LINK_ERROR, 0, -1, -1);
+            }
+          }
+        }
+      }
+    }
+    if (hasMissingLinks) {
+      sendData(ctx, mOutputCells, mOutputTriggerRecords, mOutputDecoderErrors);
+      return;
+    }
+  }
+
   // Loop over BCs, sort cells with increasing tower ID and write to output containers
   RecoContainerReader eventIterator(mCellHandler);
   while (eventIterator.hasNext()) {
@@ -355,9 +402,9 @@ void RawToCellConverterSpec::updateCalibrationObjects()
     LOG(info) << "RecoParams updated";
     o2::emcal::RecoParam::Instance().printKeyValues(true, true);
   }
-  // if (mCalibHandler->hasUpdateFEEDCS()) {
-  //   LOG(info) << "DCS params updated";
-  // }
+  if (mCalibHandler->hasUpdateFEEDCS()) {
+    LOG(info) << "DCS params updated";
+  }
 }
 
 bool RawToCellConverterSpec::isLostTimeframe(framework::ProcessingContext& ctx) const
@@ -677,17 +724,19 @@ o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getRawToCellConverter
   // CCDB objects
   auto calibhandler = std::make_shared<o2::emcal::CalibLoader>();
   calibhandler->enableRecoParams(true);
-  // calibhandler->enableFEEDCS(true);
+  calibhandler->enableFEEDCS(true);
   calibhandler->defineInputSpecs(inputs);
 
-  return o2::framework::DataProcessorSpec{"EMCALRawToCellConverterSpec",
-                                          inputs,
-                                          outputs,
-                                          o2::framework::adaptFromTask<o2::emcal::reco_workflow::RawToCellConverterSpec>(subspecification, !disableDecodingErrors, calibhandler),
-                                          o2::framework::Options{
-                                            {"fitmethod", o2::framework::VariantType::String, "gamma2", {"Fit method (standard or gamma2)"}},
-                                            {"maxmessage", o2::framework::VariantType::Int, 100, {"Max. amout of error messages to be displayed"}},
-                                            {"printtrailer", o2::framework::VariantType::Bool, false, {"Print RCU trailer (for debugging)"}},
-                                            {"no-mergeHGLG", o2::framework::VariantType::Bool, false, {"Do not merge HG and LG channels for same tower"}},
-                                            {"no-evalpedestal", o2::framework::VariantType::Bool, false, {"Disable pedestal evaluation"}}}};
+  return o2::framework::DataProcessorSpec{
+    "EMCALRawToCellConverterSpec",
+    inputs,
+    outputs,
+    o2::framework::adaptFromTask<o2::emcal::reco_workflow::RawToCellConverterSpec>(subspecification, !disableDecodingErrors, calibhandler),
+    o2::framework::Options{
+      {"fitmethod", o2::framework::VariantType::String, "gamma2", {"Fit method (standard or gamma2)"}},
+      {"maxmessage", o2::framework::VariantType::Int, 100, {"Max. amout of error messages to be displayed"}},
+      {"printtrailer", o2::framework::VariantType::Bool, false, {"Print RCU trailer (for debugging)"}},
+      {"no-mergeHGLG", o2::framework::VariantType::Bool, false, {"Do not merge HG and LG channels for same tower"}},
+      {"no-checkactivelinks", o2::framework::VariantType::Bool, false, {"Do not check for active links per BC"}},
+      {"no-evalpedestal", o2::framework::VariantType::Bool, false, {"Disable pedestal evaluation"}}}};
 }

prodtests/full-system-test/dpl-workflow.sh

@@ -258,7 +258,7 @@ fi
 ( workflow_has_parameter AOD || [[ -z "$DISABLE_ROOT_OUTPUT" ]] || needs_root_output o2-emcal-cell-writer-workflow ) && has_detector EMC && RAW_EMC_SUBSPEC=" --subspecification 1 "
 has_detector_reco MID && has_detector_matching MCHMID && MFTMCHConf="FwdMatching.useMIDMatch=true;" || MFTMCHConf="FwdMatching.useMIDMatch=false;"
 
-[[ $IS_SIMULATED_DATA == "1" ]] && EMCRAW2C_CONFIG+=" --no-mergeHGLG"
+[[ $IS_SIMULATED_DATA == "1" ]] && EMCRAW2C_CONFIG+=" --no-mergeHGLG --no-checkactivelinks"
 
 # ---------------------------------------------------------------------------------------------------------------------
 # Temporary extra options