#!/bin/bash # Define QUEUE in RNS space ... not local file system space #QUEUE=/home/xsede.org/prodhan/queues/new-jsdl-queue QUEUE=/resources/CCC/queues/NormalQueue INCOUNT=0 OUTCOUNT=0 # HOSTNAME=`hostname` # Modify the GENII_ROOT to point to your mounted home grid home # GENII_ROOT=/home/xsede.org/prodhan/ GPATHS_IN=() IN_NAME=() GPATHS_OUT=() OUT_NAME=() # BLOCK_SIZE=1 preprocessing(){ file=$1 while IFS=, read -r lpath gpath name do if [ ! -z "$lpath" ] then fastgrid cp local:$lpath $gpath fi GPATHS_IN[INCOUNT]=$gpath IN_NAME[INCOUNT]=$name INCOUNT=$(( $INCOUNT + 1 )) done <"$file" file1=$2 while IFS=, read -r lpath_o gpath_o name_o do GPATHS_OUT[OUTCOUNT]=$gpath_o OUT_NAME[OUTCOUNT]=$name_o OUTCOUNT=$(( $OUTCOUNT + 1 )) done <"$file1" } run_jobs() { # *************************************************************** # param $1 is the "name" of the test directory # param $2 is the number of iterations # param $3 is the jsdl file to modify echo "generating jobs" i=0 temp_file=$(mktemp) while (( $i < $2 )) do echo "Starting $i" cat $3 >tmp echo $INCOUNT $OUTCOUNT k=0 for ((k=0;k${temp_file} i=$(( $i + 1 )) # m=$((i%BLOCK_SIZE)) # if ((m == 0)); then # echo "hit the mod" # echo "transfering jobs to queue" PWD=`pwd` echo "fastgrid qsub ${QUEUE} "local:"${temp_file}" fastgrid qsub ${QUEUE} "local:"${temp_file} echo "job $i submitted" #fi done #echo "jobs submitted" rm ${temp_file} rm tmp } monitor () { # Now monitor the jobs # *************************************************************** # param $1 is the number of iterations sleep 10 # First, get my session cert EPI my_cert_epi=`fastgrid whoami | tail --lines 2 | head --lines 1 | grep Cert | awk '{ print $2 }' | sed 's/\"//g'` echo $my_cert_epi echo "waiting for " $1 " to complete" declare -i completed=0 declare -i count=0 while (( $completed<$1 )) do sleep 30 # First get a list of completed jobs fastgrid qlist $QUEUE > $$.qlist cat $$.qlist | grep $my_cert_epi | grep "FINISHED" | awk '{print $1 }' > $$.tmp cat $$.tmp if [ -s $$.tmp ] then # now clean them up xargs -a $$.tmp grid qcomplete $QUEUE # add the count to the number completed let count=0 count=`cat $$.tmp | wc -l` let completed=completed+count fi echo "There are " $completed " completed" done } #if [ $# -ne 2 ] #then # echo "USAGE: $0 " # exit 1 #fi while getopts i:o:j:n:m: option do case "${option}" in i) inputfile=${OPTARG};; o) outputfile=${OPTARG};; j) jsdlfile=${OPTARG};; n) numberofjobs=${OPTARG};; m) monitor=$OPTARG;; esac done if [ -z "$jsdlfile" ] then echo "No JSDL file provided. So exiting" exit 1 fi # Do any pre-processig here if [ ! -z "$inputfile" ] || [ ! -z "$outputfile" ] then preprocessing $inputfile $outputfile fi if [ -z "$numberofjobs" ] then echo "Since # of Jobs not specified, submitting 1 job" numberofjobs=1 fi run_jobs test $numberofjobs $jsdlfile if [ -z "$monitor" ] || [ ! $monitor -eq "True" ] then echo "monitor flag not set. So exiting" exit 0 fi monitor $numberofjobs # Do the post-processing here