#!/bin/bash
# usage: run_work_stream -name stream_name -maxidle nnn -queues name-1 name-2 ... name-N
#
#set -x
[[ "$1" == -name ]] || exit1
[[ -n "$2" ]] || exit 1
StreamProcessor="${2}_${JOB_ID}"
shift ; shift
#
[[ "$1" == -maxidle ]] || exit1
[[ -n "$2" ]] || exit 1
MaxIdle=$2
shift ; shift
#
[[ "$1" == -queues ]] || exit1
[[ -n "$2" ]] || exit 1
Streams="$*"
shift
#
IdleSince=$(date +%s)

cd ${HOME}/.job_queues                       || exit 1
printf "MaxIdle=${MaxIdle}\nTotalIdleTime=${IdleTime}\n" > ${HOME}/.job_queues/.active_${StreamProcessor}  || exit 1

for Stream in $Streams
do
    [[ -d $Stream ]] || continue
    touch ${HOME}/.job_queues/${Stream}/.active_${StreamProcessor}     || exit 1
    mkdir ${HOME}/.job_queues/${Stream}/.active_${StreamProcessor}.d   || exit 1
done

#
# launch monitor
#
${0%/*}/u.job-monitor &

TimeNow=$(date +%s)
((IdleTime=TimeNow-IdleSince))
while ((IdleTime<MaxIdle))
do

    ((ProcessedJobs=0))
    for Stream in $Streams    # loop over streams
    do
	[[ -d ${HOME}/.job_queues/$Stream ]] || continue
	cd ${HOME}/.job_queues/$Stream       || continue

	export StreamDir=`pwd -P`
	export StreamFlagFile=${StreamDir}/.active_${StreamProcessor}
	export StreamFlagDir=${StreamFlagFile}.d

	if [[ -r $StreamFlagFile ]]  # try to get a job from stream if stream is active
	then
            [[ -r ${HOME}/.job_queues/.active_${StreamProcessor} ]] || break 10  # stop signal from user
	    TimeNow=$(date +%s)
	    ((TimeLeft=JobStartTime+JobTimeLimit-TimeNow))
	    ((TimeLeft<75)) && \
              echo "INFO: less than 75 seconds left, exiting" && \
              break 10   # too little time left, break and cleanup
	    ${0%/*}/pick_and_run_work  $TimeLeft  && \
              ((ProcessedJobs=ProcessedJobs+1)) && \
              IdleSince=$(date +%s) && \
              break  # got a job, try again to pick from highest queue
	fi

    done

    if((ProcessedJobs==0))  # no job found in any stream, run a filler job
    then
      TimeNow=$(date +%s)
      ((IdleTime=TimeNow-IdleSince))
      echo "IDLE=$IdleTime, MAXIDLE=$MaxIdle" 1>&2
      ((TimeLeft=JobStartTime+JobTimeLimit-TimeNow))
#      ((TimeLeft>=90))                && time run_simu_opt30_nompi
      ((TimeLeft>=90))                && sleep 10
      ((TimeLeft>=15 && TimeLeft<90)) && sleep 10
    fi

    TimeNow=$(date +%s)
    ((IdleTime=TimeNow-IdleSince))

    [[ -r ${HOME}/.job_queues/.active_${StreamProcessor} ]] || break   # stop signal from user
    . ${HOME}/.job_queues/.active_${StreamProcessor}                   # update MaxIdle in case user changed it
    printf "MaxIdle=${MaxIdle}\nTotalIdleTime=${IdleTime}\n" > ${HOME}/.job_queues/.active_${StreamProcessor}
    ((TimeLeft<75)) && echo "INFO: less than 75 seconds left, exiting" && break      # too little time left, break and cleanup
done
#
rm -f ${HOME}/.job_queues/.active_${StreamProcessor}   # cleanup at end of streams
for Stream in $Streams
do
    [[ -d ${HOME}/.job_queues/$Stream ]] || continue
    echo ==== JOBS run from stream $Stream ====
    cat ${HOME}/.job_queues/${Stream}/.active_${StreamProcessor}
    rm -f ${HOME}/.job_queues/${Stream}/.active_${StreamProcessor}
    rmdir ${HOME}/.job_queues/${Stream}/.active_${StreamProcessor}.d
done
