farmoutSummary

#!/bin/bash
#
# Usage:
#   farmoutSummary <jobName>
#      or
#   farmoutSummary all
#
# Lists number of jobs that had 
# return values == 0, and also != 0
#
# Michael Anderson
# www.hep.wisc.edu/~mbanderson
# Sept 07, 2007

##################################################
FARMOUT_USER=${FARMOUT_USER:-${USER}}
farmoutDir="/scratch/${FARMOUT_USER}"
dCacheDir="/hdfs/store/user/${FARMOUT_USER}"
##################################################


##################################################
printUsage() {
 echo 
 echo "For more information:"
 echo "  farmoutSummary <jobName>"
 echo "  farmoutSummary all"
}
##################################################


##################################################
basename() {
  # This shell function is faster than calling /bin/basename
  path=$1
  suffix=$2
  path=${path##*/}  # get everything after the final '/'
  if [ ! -z $suffix ]; then
    path=${path%$suffix}
  fi
  echo $path
}
##################################################


##################################################
# Check number of folders in user's dCache area.
# If there's too many folders, this will be too
# slow. So quit.
checkNumFolders() {
  dirCount=`ls $dCacheDir | wc -l`

  if [ "$dirCount" -gt 50 ]; then
    echo "Too many folders (greater than 50) in"
    echo "  $dCacheDir"
    echo "to summarize in reasonable time."
    exit
  fi
}

##################################################


##################################################
# Gives summary information about a workflow
# Example:
#   $ farmoutSummary aTestThing-photonjetanalyzer
#   6.7M  /scratch/mbanderson/aTestThing-photonjetanalyzer/
#     5 jobs with 5 normal terminations, 0 abnormal terminations, and 0 not done.
checkThisJob() {

 dirCount=0
 notDoneCount=0
 goodCount=0
 badCount=0
 thisBaseDir=$1

 echo -e `du -hs $thisBaseDir | awk '{print $1}'`"  \c"
 echo $thisBaseDir"/"

 for dir in "$thisBaseDir/"*; do

  # Store the name of this directory alone
  dir=`basename $dir`

  # Make sure this is a directory
  if [[ -d "$thisBaseDir/$dir" ]]; then   

   # Increment Directory Count
   dirCount=`expr $dirCount + 1`

   # Store name of the log file
   logFile="$thisBaseDir/$dir/$dir.log"

   # Grab the return value for a terminated job
   returnVal=`tail -12 "$logFile" | grep "termination" | awk '{print $6}' | cut -d")" -f1`
   
   if [[ -n "$returnVal" ]]; then
    if [[ "$returnVal" -eq "0" ]]; then

     goodCount=`expr $goodCount + 1`

    else
   
     badCount=`expr $badCount + 1`

     # Figure out the last host it ran on
     # host=`grep "Job executing" "$logFile" | tail -1 | awk '{print $9}'` 

     # Print out the returnVal from host (should be optional)
     # echo "Return value" $returnVal "from host" $host
    fi
   else
    notDoneCount=`expr $notDoneCount + 1`
   fi
  fi
 done

 echo -e "  "$dirCount "jobs with\c"
 echo -e " "$goodCount "normal terminations,\c"
 echo -e " "$badCount  "abnormal terminations, and\c"
 echo -e " "$notDoneCount "not done."
}
##################################################


##################################################
checkAllJobs() {
 for jobDir in "$farmoutDir/"*; do
  if [[ -d "$jobDir" && -e "$jobDir/submit" ]]; then
   checkThisJob $jobDir
  fi
 done
}
##################################################


##################################################
listAllJobs() {
 echo "In" $farmoutDir"/ on "$HOSTNAME
 echo -e "\n"Jobs"\t"Job Name

 dirCount=0 

 for jobDir in "$farmoutDir/"*; do
  if [[ -d "$jobDir" && -e "$jobDir/submit" ]]; then

   dirCount=`expr $dirCount + 1`

   jobDir=`basename $jobDir`
   #echo -e `du -hs $farmoutDir/$jobDir | awk '{print $1}'`"\c\t"
   echo -e `ls $farmoutDir/$jobDir/ | wc | awk '{print $1}'`"\c"
   echo -e "\t"$jobDir"\t"

  fi
 done

 if [[ "$dirCount" -eq 0 ]]; then
  echo -e "\t No available jobs."
 fi
}
##################################################


##################################################
listCondorJobs() {
 echo "Currently on condor:"
 echo `condor_q "${FARMOUT_USER}" 2> /dev/null | tail -1`
 echo
}
##################################################


##################################################
listDCacheFolders() {
 echo
 echo "In" $dCacheDir
 echo ROOT
 echo -e Files"\t"Job Name

 dirCount=0

 for jobDir in "$dCacheDir/"*; do
  if [[ -d "$jobDir" ]]; then

   dirCount=`expr $dirCount + 1`

   jobDir=`basename $jobDir`
   #echo -e `du -hs $dCacheDir/$jobDir | awk '{print $1}'`"\c\t"
   echo -e `ls $dCacheDir/$jobDir/*.root | wc | awk '{print $1}'`"\c"
   echo -e "\t"$jobDir"\t"

  fi
 done

}
##################################################


##################################################
# Looks for submit files on this machine
# and finds if there is corresponding jobs 
# in the dCache folder.
#
# Example output:
#  Submitted   Completed
#       Jobs   ROOT Files  Job Name
#        134   134         JetET80-photonjetanalyzer
#         25   13          PhotonJet20-200-photonjetanalyzer
#         11   11          RelValSingleElectronPt35-photonjetanalyzer
listSubAndOut() {

 echo -e "Submit folder: $HOSTNAME:$farmoutDir   \c" 
 echo -e `du -hs $farmoutDir | awk '{print $1}'`
 echo -e "Output folder:  $dCacheDir\n"
 echo "Submitted   Completed"
 printf "%9s" "Jobs"
 echo -e "   "ROOT Files"\t"Job Name

 dirCount=0

 for jobDir in "$farmoutDir/"*; do

  jobDir=`basename $jobDir`

  if [[ -d "$farmoutDir/$jobDir" && -e "$farmoutDir/$jobDir/submit" ]]; then

   # We have found a submitted job on this machine & its corresponding dCacheDir

   dirCount=`expr $dirCount + 1`

   # Print number of submitted jobs
   #printf "%9s" "`ls -d $farmoutDir/$jobDir/*/        | wc | awk '{print $1}'`"
   printf "%9s" "`find $farmoutDir/$jobDir/*/ -type d | wc -l `"
   #printf "%9s" "`tail -8 $farmoutDir/$jobDir/submit | grep InitialDir | awk '{print $3}' | awk -F'-' '{print $2+1}'`"

   # Print  number of completed ROOT files
   if [[ -d "$dCacheDir/$jobDir" ]]; then
    #echo -e "   "`ls  $dCacheDir/$jobDir/*.root | wc | awk '{print $1}'`"\c" 
    #echo -e "   "`find $dCacheDir/$jobDir/ -name \*.root | wc -l `"\c"
    printf "   %-12s" `find $dCacheDir/$jobDir/ -name \*.root | wc -l `
   else
    printf "   %-12s" " "
   fi

   # Print the name of the job
   echo -e "$jobDir\t"

  fi
 done
}
##################################################


##################################################
# Main method
if [ $# -ne 1 ]; then

 #listAllJobs
 #listDCacheFolders
 listCondorJobs
 checkNumFolders
 listSubAndOut
 printUsage

else

 if [[ "$1" == "all" ]]; then
  checkAllJobs
 else
  jobDir="$farmoutDir/$1"
 
  # Make sure it is a valid job
  if [[ -d "$jobDir" && -e "$jobDir/submit" ]]; then
   checkThisJob "$jobDir"
  else
   echo $jobDir "is not a valid job to check!"
   listAllJobs
   printUsage
  fi
 fi
fi