#!/bin/bash
#set -x
TimeLeft=${1:-0}
((TimeLeft < 60)) && echo LESS than ${TimeLeft} wall clock seconds left, EXITING && exit 1
JobFile=""
for i in $(ls -1rt)
do
  mv $i $StreamFlagDir
  Job=${StreamFlagDir}/${i}
  if [[ -r $Job ]]
  then
    ((StreamCpus=OMP_NUM_THREADS*BATCH_MPI_CPUS))
# get information from insertion by soumet
    eval $(grep '^#SOUMET_RESOURCE_PROFILE_FOR_JOB ' ${Job} | sed 's/.* //')
#   MaxNumberOfCores,MaxNumberOfThreads,MaxWallTime for job now available
#   JobWall=$(grep walltime= ${Job} | head -1)
#   JobWall=${JobWall#*walltime=}
    JobWall=${MaxWallTime}
#   JobCpus=$(grep :ppn= ${Job} | head -1)
#   JobNodes=${JobCpus#*nodes=}
#   JobNodes=${JobNodes%:*}
#   JobCpus=${JobCpus#*:ppn=}
#   ((JobCpus=JobCpus*JobNodes))
    JobCpus=${MaxNumberOfCores}
    echo JOB ${Job##*/} scanned: JobCpus=$JobCpus, StreamCpus=$StreamCpus, JobWall=$JobWall, TimeLeft=$TimeLeft 
    ((TimeLeft-JobWall>60)) && ((JobCpus<=StreamCpus)) && JobFile=${Job} && break
    echo JOB ${Job##*/} rejected: JobCpus=$JobCpus, StreamCpus=$StreamCpus, JobWall=$JobWall, TimeLeft=$TimeLeft 
    echo  moving ${Job##*/} back into ${StreamFlagDir%/*}
    (cd $StreamFlagDir ; mv $Job ..)
  fi
done
[[ -z ${JobFile} ]] && exit 1
cd ${HOME}
# safety code below: limit one process to what the "job" requested
#ulimit -t $((JobWall*OMP_NUM_THREADS)) # previous bad code, we want OMP_NUM_THREADS from job to run, not stream controller
ulimit -t $((JobWall*MaxNumberOfThreads))  # OMP_NUM_THREADS from job extracted from SOUMET_RESOURCE_PROFILE_FOR_JOB
((runs=0))
((rerun=1))
set -x
export STREAM_STATUS=$StreamFlagFile.status
while ((runs<rerun))
do
  ((runs=runs+1))
  ((rerun=1))
  rm -f $STREAM_STATUS
  echo "${runs}:${JobFile}" >> $StreamFlagFile
  export STREAM_RUN=${runs}
  bash ${JobFile}
  [[ -r $STREAM_STATUS ]] && grep -q 'rerun=' $STREAM_STATUS && . $STREAM_STATUS
done
[[ -r $STREAM_STATUS ]] && grep -q 'rerun=' $STREAM_STATUS && exit 1
rm ${JobFile}
