#!/bin/bash
#set -x
TimeLeft=${1:-0}
((TimeLeft < 60)) && echo LESS than ${TimeLeft} wall clock seconds left, EXITING && exit 1
JobFile=""
for i in $(ls -1rt)
do
  mv $i $StreamFlagDir
  Job=${StreamFlagDir}/${i}
  if [[ -r $Job ]]
  then
    ((StreamCpus=OMP_NUM_THREADS*BATCH_MPI_CPUS))
    JobWall=$(grep walltime= ${Job} | head -1)
    JobWall=${JobWall#*walltime=}
    JobCpus=$(grep :ppn= ${Job} | head -1)
    JobNodes=${JobCpus#*nodes=}
    JobNodes=${JobNodes%:*}
    JobCpus=${JobCpus#*:ppn=}
    ((JobCpus=JobCpus*JobNodes))
    echo JOB ${Job##*/} scanned: JobCpus=$JobCpus, StreamCpus=$StreamCpus, JobWall=$JobWall, TimeLeft=$TimeLeft 
    ((TimeLeft-JobWall>60)) && ((JobCpus<=StreamCpus)) && JobFile=${Job} && break
    echo JOB ${Job##*/} rejected: JobCpus=$JobCpus, StreamCpus=$StreamCpus, JobWall=$JobWall, TimeLeft=$TimeLeft 
    echo  moving ${Job##*/} back into ${StreamFlagDir%/*}
    (cd $StreamFlagDir ; mv $Job ..)
  fi
done
[[ -z ${JobFile} ]] && exit 1
cd ${HOME}
ulimit -t $((JobWall*OMP_NUM_THREADS))
((runs=0))
((rerun=1))
set -x
export STREAM_STATUS=$StreamFlagFile.status
while ((runs<rerun))
do
  ((runs=runs+1))
  ((rerun=1))
  rm -f $STREAM_STATUS
  echo "${runs}:${JobFile}" >> $StreamFlagFile
  export STREAM_RUN=${runs}
  bash ${JobFile}
  [[ -r $STREAM_STATUS ]] && grep -q 'rerun=' $STREAM_STATUS && . $STREAM_STATUS
done
[[ -r $STREAM_STATUS ]] && grep -q 'rerun=' $STREAM_STATUS && exit 1
rm ${JobFile}
