Merge branch 'master' into swenzel/interactionRateCTPScalers

Author: sawenzel

DATA/production/calib/ft0-timeoffset-aggregator.sh

@@ -11,7 +11,7 @@ PROXY_INSPEC="calib:FT0/TIME_SPECTRA/0;${PROXY_INSPEC_EOS}"
 CALIB_TIME_OFFSET_CONFIG="FT0CalibParam.mNExtraSlots=0"
 
 WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --proxy-name ft0-timeoffset-input-proxy --dataspec \"${PROXY_INSPEC}\" --network-interface ib0 --channel-config \"name=ft0-timeoffset-input-proxy,method=bind,type=pull,rateLogging=1,transport=zeromq\" | "
-WORKFLOW+="o2-calibration-ft0-time-offset-calib --configKeyValues \"${CALIB_TIME_OFFSET_CONFIG}\" ${ARGS_ALL} --tf-per-slot 26400 | "
+WORKFLOW+="o2-calibration-ft0-time-offset-calib --configKeyValues \"${CALIB_TIME_OFFSET_CONFIG}\" ${ARGS_ALL} --tf-per-slot 105600 | "
 WORKFLOW+="o2-calibration-ccdb-populator-workflow ${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://ccdb-test.cern.ch:8080\" --sspec-min 1 --sspec-max 1 | "
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 

DATA/production/calib/hmp-pedestals-processing.sh

@@ -72,7 +72,7 @@ then
   WORKFLOW+="--use-files "
 fi
 
-WORKFLOW+="--files-basepath 'HMP/Calib/Pedestals' "
+WORKFLOW+="--files-basepath 'HMP' "
 WORKFLOW+="--pedestals-tag ${HMP_PED_TAG} --sigmacut ${HMP_SIGMACUT} | "
 
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"

DATA/production/common/getTPCvdrift.C

@@ -0,0 +1,75 @@
+#include <cmath>
+#include <fmt/format.h>
+#include <string_view>
+#include <fstream>
+
+#include "TSystem.h"
+
+#include "CCDB/CcdbApi.h"
+#include "DataFormatsTPC/LtrCalibData.h"
+#include "TPCBase/ParameterGas.h"
+#include <TJAlienCredentials.h>
+
+float getTPCvdrift(int run, std::string_view ltrUrl = "http://alice-ccdb.cern.ch")
+{
+  o2::ccdb::CcdbApi c;
+  c.init("http://alice-ccdb.cern.ch");
+  std::map<std::string, std::string> headers, metadataRCT, metadata, mm;
+  headers = c.retrieveHeaders(fmt::format("RCT/Info/RunInformation/{}", run), metadataRCT, -1);
+  printf("\nLooking for vdrift for run %d\n", run);
+  const auto sor = std::stol(headers["SOR"].data());
+
+  const auto defaultDriftV = o2::tpc::ParameterGas::Instance().DriftV;
+
+  std::string_view calibType = "TPC/Calib/LaserTracks";
+  //
+  // query present run up to +-3days
+  const auto queryInterval = 3l * 24l * 60l * 60l * 1000l;
+  TJAlienCredentials* cred = new TJAlienCredentials();
+  cred->loadCredentials();
+  cred->selectPreferedCredentials();
+  CredentialsKind cmk = cred->getPreferedCredentials();
+  TJAlienCredentialsObject cmo = cred->get(cmk);
+  
+  
+  const auto queryString = fmt::format("curl --cert {} --key {} --insecure -H \"If-Not-Before: {}\" -H \"If-Not-After: {}\" -H \"Accept: application/json\" {}/browse/{}", cmo.certpath.c_str(), cmo.keypath.c_str(), sor - queryInterval, sor + queryInterval, ltrUrl.data(), calibType.data());
+  fmt::print("Query: {}\n", queryString);
+  const auto queryResultTString = gSystem->GetFromPipe(queryString.data());
+  std::string queryResult(queryResultTString);
+
+  // find closest entry in time
+  long minDist = 9999999999999;
+  long minTime = sor;
+  size_t pos = 0;
+  const std::string_view searchString("validFrom");
+  while ((pos = queryResult.find(searchString.data(), pos)) < queryResult.size()) {
+    const auto startPosTime = queryResult.find(":", pos) + 1;
+    const auto endPosTime = queryResult.find(",", pos);
+    const auto startValidity = std::atol(queryResult.substr(startPosTime, endPosTime - startPosTime).data());
+    fmt::print("add object {}\n", startValidity);
+    if (std::abs(startValidity - sor) < minDist) {
+      minTime = startValidity;
+      minDist = std::abs(startValidity - sor);
+    }
+    pos = endPosTime;
+  }
+  fmt::print("{} closest to {} is at {}\n", calibType, sor, minTime);
+
+  //
+  // Get object closest to present run and return the drfit veloctiy calibration factor
+  c.init(ltrUrl.data());
+  const auto ltrCalib = c.retrieveFromTFileAny<o2::tpc::LtrCalibData>(calibType.data(), metadata, minTime); /// timestamp in the run of interest
+  const auto corr = ltrCalib->getDriftVCorrection();
+  float vcorr = defaultDriftV / corr;
+  if (ltrCalib->refVDrift != 0) {
+    printf("refVDrift different from zero: %f (default was %f)\n", ltrCalib->refVDrift, defaultDriftV);
+    vcorr = ltrCalib->refVDrift / corr;
+  }
+  printf("vdrift = %f\n", vcorr);
+
+  ofstream fp("vdrift.txt");
+  fp << vcorr << endl;
+  fp.close();
+
+  return vcorr;
+}

DATA/production/configurations/CTFSkimming/ctf-skim-workflow.sh

@@ -5,27 +5,28 @@ set -o pipefail
 
 # ---------------------------------------------------------------------------------------------------------------------
 # Get this script's directory and load common settings (use zsh first (e.g. on Mac) and fallback on `readlink -f` if zsh is not there)
-[[ -z $GEN_TOPO_MYDIR ]] && GEN_TOPO_MYDIR="$O2DPG_ROOT/DATA/production"
+: ${GEN_TOPO_MYDIR:="$O2DPG_ROOT/DATA/production"}
 source $GEN_TOPO_MYDIR/gen_topo_helper_functions.sh || { echo "gen_topo_helper_functions.sh failed" 1>&2 && exit 1; }
 source $GEN_TOPO_MYDIR/setenv.sh || { echo "setenv.sh failed" 1>&2 && exit 1; }
 
-if [[ -z $CTF_MINSIZE ]];              then CTF_MINSIZE="2000000000"; fi          # accumulate CTFs until file size reached
-if [[ -z $CTF_MAX_PER_FILE ]];         then CTF_MAX_PER_FILE="10000"; fi          # but no more than given number of CTFs per file
+: ${CTF_MINSIZE:="2000000000"}                    # accumulate CTFs until file size reached
+: ${CTF_MAX_PER_FILE:="1000000"}                  # but no more than given number of CTFs per file
 
 # Set general arguments
 source $GEN_TOPO_MYDIR/getCommonArgs.sh || { echo "getCommonArgs.sh failed" 1>&2 && exit 1; }
 
 TIMEFRAME_RATE_LIMIT=2
-[[ -z $NUMAID ]] && NUMAID="0"
-if [[ $ALIEN_JDL_CPUCORES == 8 ]]; then
+: ${NUMAID:="0"}
+
+if [[ ${ALIEN_JDL_CPUCORES:-} == 8 ]]; then
   export MULTIPLICITY_PROCESS_tpc_entropy_decoder=2
   export MULTIPLICITY_PROCESS_tpc_entropy_encoder=2
   TIMEFRAME_RATE_LIMIT=3
 fi
 
-[[ -z $SHM_MANAGER_SHMID ]] && ( [[ $EXTINPUT == 1 ]] || [[ $NUMAGPUIDS != 0 ]] ) && ARGS_ALL+=" --no-cleanup"
-[[ ! -z $TIMEFRAME_RATE_LIMIT ]] && [[ $TIMEFRAME_RATE_LIMIT != 0 ]] && ARGS_ALL+=" --timeframes-rate-limit $TIMEFRAME_RATE_LIMIT --timeframes-rate-limit-ipcid $(($NUMAID + 10 * ${O2JOBID:-0}))"
-[[ ! -z $TIMEFRAME_SHM_LIMIT ]] && ARGS_ALL+=" --timeframes-shm-limit $TIMEFRAME_SHM_LIMIT"
+[[ -z ${SHM_MANAGER_SHMID:-} ]] && ( [[ ${EXTINPUT:-} == 1 ]] || [[ ${NUMAGPUIDS:-} != 0 ]] ) && ARGS_ALL+=" --no-cleanup"
+[[ ! -z ${TIMEFRAME_RATE_LIMIT:-} ]] && [[ ${TIMEFRAME_RATE_LIMIT:-} != 0 ]] && ARGS_ALL+=" --timeframes-rate-limit $TIMEFRAME_RATE_LIMIT --timeframes-rate-limit-ipcid $(($NUMAID + 10 * ${O2JOBID:-0}))"
+[[ ! -z ${TIMEFRAME_SHM_LIMIT:-} ]] && ARGS_ALL+=" --timeframes-shm-limit $TIMEFRAME_SHM_LIMIT"
 
 { source $O2DPG_ROOT/DATA/production/workflow-multiplicities.sh; [[ $? != 0 ]] && echo "workflow-multiplicities.sh failed" 1>&2 && exit 1; }
 
@@ -52,19 +53,29 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-if [[ -z $IRFRAMES ]] || [[ -z $CTFLIST ]] ; then
+if [[ -z ${IRFRAMES:-} ]] || [[ -z ${CTFLIST:-} ]] ; then
   echo "Format: ${0##*/} -f <IRFramesFile> -c <CTFsList>"
   exit 1  
 fi
 
-[[ "0$ALLOW_MISSING_DET" == "00" ]] && ALLOW_MISSING_DET= || ALLOW_MISSING_DET="--allow-missing-detectors"
-[[ "0$SKIP_SKIMMED_OUT_TF" == "00" ]] && SKIP_SKIMMED_OUT_TF= || SKIP_SKIMMED_OUT_TF="--skip-skimmed-out-tf"
+[[ "0${ALLOW_MISSING_DET:-}" == "00" ]] && ALLOW_MISSING_DET= || ALLOW_MISSING_DET="--allow-missing-detectors"
+[[ "0${SKIP_SKIMMED_OUT_TF:-}" == "00" ]] && SKIP_SKIMMED_OUT_TF= || SKIP_SKIMMED_OUT_TF="--skip-skimmed-out-tf"
+
+: ${DEF_MARGIN_BWD:=55}                           # default backward margin to account for time misalignments
+: ${DEF_MARGIN_FWD:=55}                           # default forward margin to account for time misalignments
 
-[[ -z $DEF_MARGIN_BWD ]] && DEF_MARGIN_BWD=55 # default backward margin to account for time misalignments
-[[ -z $DEF_MARGIN_FWD ]] && DEF_MARGIN_FWD=55 # default backward margin to account for time misalignments
+if [[ -z ${NO_ITSMFT_MASKING:-} ]] ; then
+  has_detector_ctf ITS && export ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow="$ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow --its-digits"
+  has_detector_ctf MFT && export ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow="$ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow --mft-digits"
+fi
 
 add_W o2-ctf-reader-workflow "--ctf-data-subspec 1 --ir-frames-files $IRFRAMES $SKIP_SKIMMED_OUT_TF --ctf-input $CTFLIST ${INPUT_FILE_COPY_CMD+--copy-cmd} ${INPUT_FILE_COPY_CMD} --onlyDet $WORKFLOW_DETECTORS $ALLOW_MISSING_DET --pipeline $(get_N tpc-entropy-decoder TPC REST 1 TPCENTDEC)"
 
+if [[ -z ${NO_ITSMFT_MASKING:-} ]] ; then
+  has_detector_ctf ITS && add_W o2-its-reco-workflow "--digits-from-upstream --disable-mc --disable-tracking --disable-root-output --pipeline $(get_N its-tracker ITS REST 1 ITSTRK)" "ITSClustererParam.maxBCDiffToMaskBias=10;"
+  has_detector_ctf MFT && add_W o2-mft-reco-workflow "--digits-from-upstream --disable-mc --disable-tracking --disable-root-output --pipeline $(get_N mft-tracker MFT REST 1 MFTTRK)" "MFTClustererParam.maxBCDiffToMaskBias=10;"    
+fi
+
 has_detector_ctf ITS && add_W o2-itsmft-entropy-encoder-workflow "--select-ir-frames --irframe-margin-bwd ${ITS_MARGIN_BWD:-$DEF_MARGIN_BWD} --irframe-margin-fwd ${ITS_MARGIN_FWD:-$DEF_MARGIN_FWD} --mem-factor ${ITS_ENC_MEMFACT:-1.5} --pipeline $(get_N its-entropy-encoder ITS CTF 1)"
 has_detector_ctf MFT && add_W o2-itsmft-entropy-encoder-workflow "--select-ir-frames --irframe-margin-bwd ${MFT_MARGIN_BWD:-$DEF_MARGIN_BWD} --irframe-margin-fwd ${MFT_MARGIN_FWD:-$DEF_MARGIN_FWD} --mem-factor ${MFT_ENC_MEMFACT:-1.5} --runmft true --pipeline $(get_N mft-entropy-encoder MFT CTF 1)"
 has_detector_ctf TPC && add_W o2-tpc-reco-workflow "--select-ir-frames --irframe-margin-bwd ${TPC_MARGIN_BWD:-$DEF_MARGIN_BWD} --irframe-margin-fwd ${TPC_MARGIN_FWD:-$DEF_MARGIN_FWD} --mem-factor ${TPC_ENC_MEMFACT:-1.} --input-type compressed-clusters-flat --output-type encoded-clusters,disable-writer --pipeline $(get_N tpc-entropy-encoder TPC CTF 1 TPCENT)"
@@ -86,7 +97,7 @@ add_W o2-ctf-writer-workflow "--output-dir $CTF_DIR --min-file-size ${CTF_MINSIZ
 
 # ---------------------------------------------------------------------------------------------------------------------
 # DPL run binary
-WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT -b"
+WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT -b --run"
 
-[[ $WORKFLOWMODE == "print" || "0$PRINT_WORKFLOW" == "01" ]] && echo "#Workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ $WORKFLOWMODE == "dds" ]] && echo '1>&2')
-if [[ $WORKFLOWMODE != "print" ]]; then eval $WORKFLOW; else true; fi
+[[ ${WORKFLOWMODE:-} == "print" || "0${PRINT_WORKFLOW:-}" == "01" ]] && echo "#Workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ ${WORKFLOWMODE:-} == "dds" ]] && echo '1>&2')
+if [[ ${WORKFLOWMODE:-} != "print" ]]; then eval $WORKFLOW; else true; fi

DATA/production/configurations/asyncReco/async_pass.sh

@@ -17,6 +17,8 @@ run_AOD_merging() {
   return $exitcode
 }
 
+timeStartFullProcessing=`date +%s`
+
 # to skip positional arg parsing before the randomizing part.
 inputarg="${1}"
 
@@ -409,7 +411,7 @@ if [[ $ALIEN_JDL_SPLITWF != "1" ]]; then
       exit $exitcode
     fi
     mv latest.log latest_reco_1.log
-    ./$STATSCRIPT latest_reco_1.log
+    $STATSCRIPT latest_reco_1.log
   fi
 else
   # running the wf in split mode
@@ -440,7 +442,7 @@ else
 	exit $exitcode
       fi
       mv latest.log latest_reco_1.log
-      ./$STATSCRIPT latest_reco_1.log reco_1
+      $STATSCRIPT latest_reco_1.log reco_1
     fi
   fi
 
@@ -469,7 +471,7 @@ else
 	exit $exitcode
       fi
       mv latest.log latest_reco_2.log
-      ./$STATSCRIPT latest_reco_2.log reco_2
+      $STATSCRIPT latest_reco_2.log reco_2
       # let's compare to previous step
       if [[ -f latest_reco_1.log ]]; then
 	nCTFsFilesInspected_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/\(^[0-9]*\)_.*/\1/'`
@@ -740,8 +742,11 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then
   fi
 fi
 
-timeUsed=$(( $timeUsed+$timeUsedCheck+$timeUsedMerge+$timeUsedCheckMergedAOD+$timeUsedAnalysisQC ))
-echo "Time used for processing = $timeUsed s"
+
+timeEndFullProcessing=`date +%s`
+timeUsedFullProcessing=$(( $timeEndFullProcessing+$timeStartFullProcessing ))
+
+echo "Time used for processing = $timeUsedFullProcessing s"
 
 if [[ $ALIEN_JDL_QCOFF != 1 ]]; then
   # copying the QC json file here

DATA/production/configurations/asyncReco/setenv_extra.sh

@@ -5,8 +5,10 @@
 
 export SETENV_NO_ULIMIT=1
 
-# to avoid memory issues
-export DPL_DEFAULT_PIPELINE_LENGTH=16
+# to avoid memory issues - we don't do this on the EPNs, since it can affect the performance
+if [[ $ALIEN_JDL_USEGPUS != 1 ]]; then
+  export DPL_DEFAULT_PIPELINE_LENGTH=16
+fi
 
 # detector list
 if [[ -n $ALIEN_JDL_WORKFLOWDETECTORS ]]; then
@@ -159,6 +161,24 @@ else
     echo "************************************************************"
 fi
 
+# TPC vdrift
+PERIODLETTER=${PERIOD: -1}
+VDRIFTPARAMOPTION=
+if [[ $$ALIEN_JDL_LPMANCHORYEAR == "2022" ]] && [[ $PERIODLETTER < m ]]; then
+  echo "In setenv_extra: time used so far = $timeUsed s"
+  timeStart=`date +%s`
+  time root -b -q "$O2DPG_ROOT/DATA/production/common/getTPCvdrift.C+($RUNNUMBER)"
+  timeEnd=`date +%s`
+  timeUsed=$(( $timeUsed+$timeEnd-$timeStart ))
+  delta=$(( $timeEnd-$timeStart ))
+  echo "Time spent to get VDrift for TPC = $delta s"
+  export VDRIFT=`cat vdrift.txt`
+  VDRIFTPARAMOPTION="TPCGasParam.DriftV=$VDRIFT"
+  echo "Setting TPC vdrift to $VDRIFT"
+else
+  echo "TPC vdrift will be taken from CCDB"
+fi
+
 # IR
 if [[ -z $RUN_IR ]] || [[ -z $RUN_DURATION ]] || [[ -z $RUN_BFIELD ]]; then
   cp $O2DPG_ROOT/DATA/production/common/getIRandDuration.C ./
@@ -280,7 +300,7 @@ if [[ $ALIGNLEVEL != 0 ]]; then
 fi
 
 # ad-hoc options for GPU reco workflow
-export CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow+=";GPU_global.dEdxDisableResidualGainMap=1;$TRACKTUNETPCINNER;"
+export CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow+=";GPU_global.dEdxDisableResidualGainMap=1;$TRACKTUNETPCINNER;$VDRIFTPARAMOPTION;"
 if [[ $ALIEN_JDL_LPMPRODUCTIONTYPE == "MC" ]]; then
   export CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow+=";GPU_global.dEdxDisableResidualGain=1"
 fi
@@ -315,10 +335,10 @@ fi
 # secondary vertexing
 export SVTX="svertexer.checkV0Hypothesis=false;svertexer.checkCascadeHypothesis=false"
 
-export CONFIG_EXTRA_PROCESS_o2_primary_vertexing_workflow+=";$PVERTEXER;"
+export CONFIG_EXTRA_PROCESS_o2_primary_vertexing_workflow+=";$PVERTEXER;$VDRIFTPARAMOPTION;"
 export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";$SVTX"
 
-export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATCH;$TRACKTUNETPCINNER;"
+export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATCH;$TRACKTUNETPCINNER;$VDRIFTPARAMOPTION;"
 [[ ! -z "${TPCITSTIMEBIAS}" ]] && export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";tpcitsMatch.globalTimeBiasMUS=$TPCITSTIMEBIAS;"
 [[ ! -z "${TPCITSTIMEERR}" ]] && export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";tpcitsMatch.globalTimeExtraErrorMUS=$TPCITSTIMEERR;"
 
@@ -327,10 +347,10 @@ has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EX
 
 # ad-hoc settings for TOF matching
 export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia"
-export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPCINNER;"
+export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPCINNER;$VDRIFTPARAMOPTION;"
 
 # ad-hoc settings for TRD matching
-export CONFIG_EXTRA_PROCESS_o2_trd_global_tracking+=";$ITSEXTRAERR;$TRACKTUNETPCINNER;"
+export CONFIG_EXTRA_PROCESS_o2_trd_global_tracking+=";$ITSEXTRAERR;$TRACKTUNETPCINNER;$VDRIFTPARAMOPTION;"
 
 # ad-hoc settings for FT0
 export ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow="$ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow --ft0-reconstructor"

DATA/production/qc-async/mft.json

@@ -64,7 +64,7 @@
         "dataSource": {
           "type": "direct",
           "query_comment" : "100% sampling",
-          "query": "tracks:MFT/TRACKS/0;tracksrofs:MFT/MFTTrackROF/0"
+          "query": "tracks:MFT/TRACKS/0;tracksrofs:MFT/MFTTrackROF/0;clustersrofs:MFT/CLUSTERSROF/0"
         },
         "taskParameters": {
           "ROFLengthInBC": "594",

DATA/production/qc-sync/mft-full.json

@@ -107,7 +107,7 @@
         "maxNumberCycles": "-1",
         "dataSource" : {
           "type" : "direct",
-          "query" : "tracks:MFT/TRACKS/0;tracksrofs:MFT/MFTTrackROF/0"
+          "query" : "tracks:MFT/TRACKS/0;tracksrofs:MFT/MFTTrackROF/0;clustersrofs:MFT/CLUSTERSROF/0"
         },
         "location": "local",
         "localMachines": [

GRID/utils/grid_submit.sh

@@ -306,15 +306,15 @@ EOF
       echo "cp file:${PWD}/${MY_JOBNAMEDATE}.jdl alien://${MY_JOBWORKDIR}/${MY_JOBNAMEDATE}.jdl@DISK=1" >> ${command_file}  # copy the jdl
       echo "cp file:${THIS_SCRIPT} alien://${MY_BINDIR}/${MY_JOBNAMEDATE}.sh@DISK=1" >> ${command_file}  # copy current job script to AliEn
       [ ! "${CONTINUE_WORKDIR}" ] && echo "cp file:${MY_JOBSCRIPT} alien://${MY_JOBWORKDIR}/alien_jobscript.sh" >> ${command_file}
-    ) &> alienlog.txt
+    ) > alienlog.txt 2>&1
 
     pok "Submitting job \"${MY_JOBNAMEDATE}\" from $PWD"
     (
       echo "submit ${MY_JOBWORKDIR}/${MY_JOBNAMEDATE}.jdl" >> ${command_file}
 
       # finally we do a single call to alien:
       alien.py < ${command_file}
-    ) &>> alienlog.txt
+    ) >> alienlog.txt 2>&1
 
     MY_JOBID=$( (grep 'Your new job ID is' alienlog.txt | grep -oE '[0-9]+' || true) | sort -n | tail -n1)
     if [[ $MY_JOBID ]]; then

MC/bin/o2dpg_sim_workflow.py

@@ -107,6 +107,7 @@
 parser.add_argument('--combine-tpc-clusterization', action='store_true', help=argparse.SUPPRESS) #<--- useful for small productions (pp, low interaction rate, small number of events)
 parser.add_argument('--first-orbit', default=0, type=int, help=argparse.SUPPRESS)  # to set the first orbit number of the run for HBFUtils (only used when anchoring)
                                                             # (consider doing this rather in O2 digitization code directly)
+parser.add_argument('--sor', default=-1, type=int, help=argparse.SUPPRESS) # may pass start of run with this (otherwise it is autodetermined from run number)
 parser.add_argument('--run-anchored', action='store_true', help=argparse.SUPPRESS)
 parser.add_argument('--alternative-reco-software', default="", help=argparse.SUPPRESS) # power feature to set CVFMS alienv software version for reco steps (different from default)
 parser.add_argument('--dpl-child-driver', default="", help="Child driver to use in DPL processes (export mode)")
@@ -266,9 +267,12 @@ def extractVertexArgs(configKeyValuesStr, finalDiamondDict):
 # ----------- START WORKFLOW CONSTRUCTION -----------------------------
 
 # set the time to start of run (if no timestamp specified)
+if args.sor==-1:
+   args.sor = retrieve_sor(args.run)
+   assert (args.sor != 0)
+
 if args.timestamp==-1:
-   args.timestamp = retrieve_sor(args.run)
-   assert (args.timestamp != 0)
+   args.timestamp = args.sor
 
 NTIMEFRAMES=int(args.tf)
 NWORKERS=args.j
@@ -715,8 +719,8 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True):
                             "HBFUtils.runNumber" : args.run }
    # we set the timestamp here only if specified explicitely (otherwise it will come from
    # the simulation GRP and digitization)
-   if (args.timestamp != -1):
-      globalTFConfigValues["HBFUtils.startTime"] = args.timestamp
+   if (args.sor != -1):
+      globalTFConfigValues["HBFUtils.startTime"] = args.sor
 
    def putConfigValues(localCF = {}):
      """