Apply pixel overflow masking to ITS,MFT before skimming

Author: shahor02

DATA/production/configurations/CTFSkimming/ctf-skim-workflow.sh

@@ -5,27 +5,28 @@ set -o pipefail
 
 # ---------------------------------------------------------------------------------------------------------------------
 # Get this script's directory and load common settings (use zsh first (e.g. on Mac) and fallback on `readlink -f` if zsh is not there)
-[[ -z $GEN_TOPO_MYDIR ]] && GEN_TOPO_MYDIR="$O2DPG_ROOT/DATA/production"
+: ${GEN_TOPO_MYDIR:="$O2DPG_ROOT/DATA/production"}
 source $GEN_TOPO_MYDIR/gen_topo_helper_functions.sh || { echo "gen_topo_helper_functions.sh failed" 1>&2 && exit 1; }
 source $GEN_TOPO_MYDIR/setenv.sh || { echo "setenv.sh failed" 1>&2 && exit 1; }
 
-if [[ -z $CTF_MINSIZE ]];              then CTF_MINSIZE="2000000000"; fi          # accumulate CTFs until file size reached
-if [[ -z $CTF_MAX_PER_FILE ]];         then CTF_MAX_PER_FILE="10000"; fi          # but no more than given number of CTFs per file
+: ${CTF_MINSIZE:="2000000000"}                    # accumulate CTFs until file size reached
+: ${CTF_MAX_PER_FILE:="1000000"}                  # but no more than given number of CTFs per file
 
 # Set general arguments
 source $GEN_TOPO_MYDIR/getCommonArgs.sh || { echo "getCommonArgs.sh failed" 1>&2 && exit 1; }
 
 TIMEFRAME_RATE_LIMIT=2
-[[ -z $NUMAID ]] && NUMAID="0"
-if [[ $ALIEN_JDL_CPUCORES == 8 ]]; then
+: ${NUMAID:="0"}
+
+if [[ ${ALIEN_JDL_CPUCORES:-} == 8 ]]; then
   export MULTIPLICITY_PROCESS_tpc_entropy_decoder=2
   export MULTIPLICITY_PROCESS_tpc_entropy_encoder=2
   TIMEFRAME_RATE_LIMIT=3
 fi
 
-[[ -z $SHM_MANAGER_SHMID ]] && ( [[ $EXTINPUT == 1 ]] || [[ $NUMAGPUIDS != 0 ]] ) && ARGS_ALL+=" --no-cleanup"
-[[ ! -z $TIMEFRAME_RATE_LIMIT ]] && [[ $TIMEFRAME_RATE_LIMIT != 0 ]] && ARGS_ALL+=" --timeframes-rate-limit $TIMEFRAME_RATE_LIMIT --timeframes-rate-limit-ipcid $(($NUMAID + 10 * ${O2JOBID:-0}))"
-[[ ! -z $TIMEFRAME_SHM_LIMIT ]] && ARGS_ALL+=" --timeframes-shm-limit $TIMEFRAME_SHM_LIMIT"
+[[ -z ${SHM_MANAGER_SHMID:-} ]] && ( [[ ${EXTINPUT:-} == 1 ]] || [[ ${NUMAGPUIDS:-} != 0 ]] ) && ARGS_ALL+=" --no-cleanup"
+[[ ! -z ${TIMEFRAME_RATE_LIMIT:-} ]] && [[ ${TIMEFRAME_RATE_LIMIT:-} != 0 ]] && ARGS_ALL+=" --timeframes-rate-limit $TIMEFRAME_RATE_LIMIT --timeframes-rate-limit-ipcid $(($NUMAID + 10 * ${O2JOBID:-0}))"
+[[ ! -z ${TIMEFRAME_SHM_LIMIT:-} ]] && ARGS_ALL+=" --timeframes-shm-limit $TIMEFRAME_SHM_LIMIT"
 
 { source $O2DPG_ROOT/DATA/production/workflow-multiplicities.sh; [[ $? != 0 ]] && echo "workflow-multiplicities.sh failed" 1>&2 && exit 1; }
 
@@ -52,19 +53,29 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-if [[ -z $IRFRAMES ]] || [[ -z $CTFLIST ]] ; then
+if [[ -z ${IRFRAMES:-} ]] || [[ -z ${CTFLIST:-} ]] ; then
   echo "Format: ${0##*/} -f <IRFramesFile> -c <CTFsList>"
   exit 1  
 fi
 
-[[ "0$ALLOW_MISSING_DET" == "00" ]] && ALLOW_MISSING_DET= || ALLOW_MISSING_DET="--allow-missing-detectors"
-[[ "0$SKIP_SKIMMED_OUT_TF" == "00" ]] && SKIP_SKIMMED_OUT_TF= || SKIP_SKIMMED_OUT_TF="--skip-skimmed-out-tf"
+[[ "0${ALLOW_MISSING_DET:-}" == "00" ]] && ALLOW_MISSING_DET= || ALLOW_MISSING_DET="--allow-missing-detectors"
+[[ "0${SKIP_SKIMMED_OUT_TF:-}" == "00" ]] && SKIP_SKIMMED_OUT_TF= || SKIP_SKIMMED_OUT_TF="--skip-skimmed-out-tf"
+
+: ${DEF_MARGIN_BWD:=55}                           # default backward margin to account for time misalignments
+: ${DEF_MARGIN_FWD:=55}                           # default forward margin to account for time misalignments
 
-[[ -z $DEF_MARGIN_BWD ]] && DEF_MARGIN_BWD=55 # default backward margin to account for time misalignments
-[[ -z $DEF_MARGIN_FWD ]] && DEF_MARGIN_FWD=55 # default backward margin to account for time misalignments
+if [[ -z ${NO_ITSMFT_MASKING:-} ]] ; then
+  has_detector_ctf ITS && export ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow="$ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow --its-digits"
+  has_detector_ctf MFT && export ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow="$ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow --mft-digits"
+fi
 
 add_W o2-ctf-reader-workflow "--ctf-data-subspec 1 --ir-frames-files $IRFRAMES $SKIP_SKIMMED_OUT_TF --ctf-input $CTFLIST ${INPUT_FILE_COPY_CMD+--copy-cmd} ${INPUT_FILE_COPY_CMD} --onlyDet $WORKFLOW_DETECTORS $ALLOW_MISSING_DET --pipeline $(get_N tpc-entropy-decoder TPC REST 1 TPCENTDEC)"
 
+if [[ -z ${NO_ITSMFT_MASKING:-} ]] ; then
+  has_detector_ctf ITS && add_W o2-its-reco-workflow "--digits-from-upstream --disable-mc --disable-tracking --disable-root-output --pipeline $(get_N its-tracker ITS REST 1 ITSTRK)" "ITSClustererParam.maxBCDiffToMaskBias=10;"
+  has_detector_ctf MFT && add_W o2-mft-reco-workflow "--digits-from-upstream --disable-mc --disable-tracking --disable-root-output --pipeline $(get_N mft-tracker MFT REST 1 MFTTRK)" "MFTClustererParam.maxBCDiffToMaskBias=10;"    
+fi
+
 has_detector_ctf ITS && add_W o2-itsmft-entropy-encoder-workflow "--select-ir-frames --irframe-margin-bwd ${ITS_MARGIN_BWD:-$DEF_MARGIN_BWD} --irframe-margin-fwd ${ITS_MARGIN_FWD:-$DEF_MARGIN_FWD} --mem-factor ${ITS_ENC_MEMFACT:-1.5} --pipeline $(get_N its-entropy-encoder ITS CTF 1)"
 has_detector_ctf MFT && add_W o2-itsmft-entropy-encoder-workflow "--select-ir-frames --irframe-margin-bwd ${MFT_MARGIN_BWD:-$DEF_MARGIN_BWD} --irframe-margin-fwd ${MFT_MARGIN_FWD:-$DEF_MARGIN_FWD} --mem-factor ${MFT_ENC_MEMFACT:-1.5} --runmft true --pipeline $(get_N mft-entropy-encoder MFT CTF 1)"
 has_detector_ctf TPC && add_W o2-tpc-reco-workflow "--select-ir-frames --irframe-margin-bwd ${TPC_MARGIN_BWD:-$DEF_MARGIN_BWD} --irframe-margin-fwd ${TPC_MARGIN_FWD:-$DEF_MARGIN_FWD} --mem-factor ${TPC_ENC_MEMFACT:-1.} --input-type compressed-clusters-flat --output-type encoded-clusters,disable-writer --pipeline $(get_N tpc-entropy-encoder TPC CTF 1 TPCENT)"
@@ -86,7 +97,7 @@ add_W o2-ctf-writer-workflow "--output-dir $CTF_DIR --min-file-size ${CTF_MINSIZ
 
 # ---------------------------------------------------------------------------------------------------------------------
 # DPL run binary
-WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT -b"
+WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT -b --run"
 
-[[ $WORKFLOWMODE == "print" || "0$PRINT_WORKFLOW" == "01" ]] && echo "#Workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ $WORKFLOWMODE == "dds" ]] && echo '1>&2')
-if [[ $WORKFLOWMODE != "print" ]]; then eval $WORKFLOW; else true; fi
+[[ ${WORKFLOWMODE:-} == "print" || "0${PRINT_WORKFLOW:-}" == "01" ]] && echo "#Workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ ${WORKFLOWMODE:-} == "dds" ]] && echo '1>&2')
+if [[ ${WORKFLOWMODE:-} != "print" ]]; then eval $WORKFLOW; else true; fi