Improvements in failure log processing

* ... in particular less connections to AliEn

Author: sawenzel

GRID/utils/extractErroredLogFiles.sh

@@ -1,6 +1,15 @@
 # We determine the O2DPG task that failed (as listed in stdout) and extract the relevant log automatically
 # Beware that errors might occur outside of O2DPG tasks such as in preprocessing etc or not visible in logs
 
+alias mytar=tar
+if [[ $(uname) == "Darwin" ]]; then
+    echo "Running on macOS. This needs gnu-tar"
+    alias mytar=gtar
+fi
+
+# show aliases
+alias
+
 errored_tasks=""
 find ./ -name "stdout*" -exec grep -H "failed.*retry" {} ';' | sed 's/ failed.*//' | tr ":" " " | while IFS= read -r line; do
   stdoutpath=$(echo "$line" | awk '{print $1}')  # Extracting the first column
@@ -17,12 +26,12 @@ find ./ -name "stdout*" -exec grep -H "failed.*retry" {} ';' | sed 's/ failed.*/
   # extract the general log archive
   unzip -n log_archive.zip
   # extract specific task log from debug archive
-  tar -xvzf debug_log_archive.tgz --wildcards "*${logfile}.log"
+  mytar -xvzf debug_log_archive.tgz --wildcards "*${logfile}.log"
   if [[ ${logfile} == *"qedsim"* || ${logfile} == *"sgnsim"* || ${logfile} == *"bkgsim"* ]]; then
     # simulation has few more files to inspect
-    tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*serverlog*"
-    tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*workerlog*"
-    tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*mergerlog*"
+    mytar -xvzf debug_log_archive.tgz --wildcards "*${tf}*serverlog*"
+    mytar -xvzf debug_log_archive.tgz --wildcards "*${tf}*workerlog*"
+    mytar -xvzf debug_log_archive.tgz --wildcards "*${tf}*mergerlog*"
   fi
   cd $OLDPWD
 done

GRID/utils/extractFailureMessage.sh

@@ -23,6 +23,7 @@ pattern="-e \"\<[Ee]xception\"                         \
          -e \"terminate called without an active\"     \
          -e \"\]\[FATAL\]\"                            \
          -e \"\*\*\* Program crashed\"                 \
+         -e \"\*\*\* Fatal Exception\"                 \
          -e \"\*\*\* Error in\"" # <--- LIBC fatal error messages
 
 CMD="grep -m 1 -B2 -A10 ${pattern} $filetocheck"
@@ -32,4 +33,4 @@ if [ "$?" -eq 0 ]; then
   echo "==============================ERROR in ${filetocheck} =============================="
   echo ""
   echo "${T}"
-fi
+fi

GRID/utils/fetch_output_onfailure.sh

@@ -8,27 +8,40 @@ if [ ! "${MY_JOBID}" ]; then
   exit 1
 fi
 
+# these are all subjobids
 SUBJOBIDS=($(alien.py ps --trace ${MY_JOBID} | awk '/Subjob submitted/' | sed 's/.*submitted: //' | tr '\n' ' '))
 
+# these are the one that failed
+FAILEDSUBJOBIDS=($(alien.py ps -a -m ${MY_JOBID} -f ERROR_ALL | awk '/EE/{print $2}' | tr '\n' ' '))
+
 CurrDir=${PWD}
 OutputDir=/tmp/AlienLogs_${MY_JOBID}
 
-# 
-job_number=${#SUBJOBIDS[@]}
-
-
-# We fetch the error files locally
+job_number=${#FAILEDSUBJOBIDS[@]}
+echo "Found ${job_number} failed job ids"
+echo "Registering output for retrieval"
+RecycleBase=""
+# First pass to do register output
 for ((i = 0; i < job_number; i++)); do
-  jobid=${SUBJOBIDS[i]}
-  STATUS=$(alien.py ps -j ${jobid} | awk '//{print $4}')
-  echo "Status of ${jobid} is $STATUS"
-  if [ ${STATUS} == "EE" ]; then
-    # nothing
+  jobid=${FAILEDSUBJOBIDS[i]}
+  if [ ! "${RecycleBase}" ]; then
     RecycleOutputDir=$(alien.py ps --trace ${jobid} | awk '/Going to uploadOutputFiles/' | sed 's/.*outputDir=//' | sed 's/)//')
-    alien.py registerOutput ${jobid}
-    echo "Recycle out is ${RecycleOutputDir}"
-    alien.py cp ${RecycleOutputDir}/'*' file:${OutputDir}/${jobid}
+    # /alice/cern.ch/user/a/aliprod/recycle/alien-job-2974093751
+    RecycleBase=${RecycleOutputDir%-${jobid}} # Removes the ${jobid} and yields the recycle base path
   fi
+  $(alien.py registerOutput ${jobid}) 2> /dev/null
+done
+
+# wait a bit to allow propagation of "registerOutput"
+sleep 1
+
+echo "Downloading output"
+# Second pass to copy files
+for ((i = 0; i < job_number; i++)); do
+  jobid=${FAILEDSUBJOBIDS[i]}
+  RecycleOutputDir="${RecycleBase}-${jobid}"
+  alien.py cp ${RecycleOutputDir}/'*archive*' file:${OutputDir}/${jobid}
+  [ -f ${OutputDir}/${jobid}/log_archive.zip ] && unzip -q -o ${OutputDir}/${jobid}/log_archive.zip -d ${OutputDir}/${jobid}
 done
 
 echo " ... Going to automatic extraction of log files ... "