#!/bin/bash

# Run a command in parallel
#
# Call a process for each configured argument line in the arglist file.
# (provide help here, please)

# Give the command line argument "-q" to suppress all but error output, "-Q" to suppress process' output to stdout.

# (c) 2010 under GPL v2 by Adrian Zaugg.



# if you run on a non-GNU system, you can provide 
# the path to GNU date command here
GNUDATE="date"


# ----don't edit below this line-----
TIMESTAMP="$("$GNUDATE" --utc --date now "+%s")"

# provide some defaults if the users setup() isn't found
RUN_BIN=""
ARGLIST_FILE=""
ARGS_LOG="my_args"
NICENESS=19
MAX_PROCESSES=2
RETRY=2
STARTMSG="Starting..."

# Help message
function usage()
{
USAGE="
Usage: $PROGNAME [-d|--rundir-basename] [-i|--rundir-infix] [-o|--runid-offset] [-q|--quiet] [-Q|--quiet-run] [-h|-?|--help] 

Start any process in parallel. See /usr/local/lib/parallelstarter/README for more info.

  -d|--rundir-basename    Base name for your result directories
  -i|--rundir-infix       Addition to the result directories' base name
  -o|--runid-offset       Run count start number
  -Q|--quiet		  Suppress all but error messages
  -q|--quiet-run	  Suppress output messages from started processes
  -h|-?|--help		  This message

To use parallelstarter create a new directory, change into it and call setup_parallelstarter.
"
# Print help
$SPEAK && echo "$USAGE"
}

# get command line arguments
function get_commandlineswitches()
{
	# read the users' wish
	while [ "$#" -gt 0 ]; do
	
	   case "$1" in
	
	      -Q|--quiet)
	         # suppress all output but errors
		 SPEAK=false
	         shift
	         ;;

	      -q|--quiet-run)
	         SILENT_PROC_OUT=true
	         shift
	         ;;
	      
	      -d|--rundir-basename)
	      	 shift
		 RUNDIR_BASENAME="$1"
		 shift
		 ;;

	      -i|--rundir-infix)
	      	 shift
		 RUN_DIR_PREFIX="$1"
		 shift
		 ;;
	      
	      -o|--runid-offset)
	      	 shift
		 RUN_COUNT_OFFSET="$1"
		 shift
		 ;;

	      -h|-?|--help)
		 shift
		 PSHELP=true
		 ;;

	      *)
		echo "Error: Undefined argument to `basename $0`. Use --help to get a list of valid arguments."
	        exit 1
		;;
 
	   esac
	done

	# check for help request
	if [ ! -z "$PSHELP" ]; then
		usage
		exit 0
	fi
}	 
		 
function count_proc() {
	
	# count the number of running processes
	
	if [ ! -z "$PID_LIST" ]; then
		if [ "$(uname -s | egrep -wc "Linux|Darwin")" -eq 1 ]; then
			# Linux or Darwin
			if $SPEAK; then
				psproc='/bin/sh ./parallelstarter.start_run_[0-9]+'
			else
				psproc="$RUN_BIN $COMMON_LEADING_ARGS"
				psproc="$(echo $psproc) .*"
			fi
			COUNT_PROC=$(ps -p "$PID_LIST" -o command | egrep -cwe '^'"$psproc"'$')
		elif [ "$(uname -o | grep -c "Solaris")" -eq 1 ]; then
			# Solaris
			if $SPEAK; then
				psproc='parallel'
			else
				# This might fail, if RUN_BIN gets longer than 8 chars
				# -> would start all processes at once...
				psproc="$RUN_BIN"
			fi
			COUNT_PROC=$(ps -p "$PID_LIST" -o fname | grep -cwe '^'"$psproc"'$')
		else
			echo "Unknown OS. Only tested on Linux and Darwin."
			echo "You need to test the function count_proc() on your system, prior to use this script."
			exit 1
		fi
	else
		COUNT_PROC=0
	fi
}

function execute_user_code() {
	# source the file parallelstarter.setup
	if [ -e "$USERCODE_FILE" ]; then
		. "$USERCODE_FILE"
	fi
}

function setup_run() {
	USERCODE_FILE="parallelstarter.setup"
	if [ ! -e "$USERCODE_FILE" ]; then
		echo -e "You need to provide a file called $USERCODE_FILE in the current\n"\
			"directory. The file will be executed by the shell unconditionally!"
		exit 1
	fi
	execute_user_code
}

function prepare_process_start() {
	USERCODE_FILE="parallelstarter.prepare_process_start"
	execute_user_code
	RUN_DIR="$(pwd)"
}

function start_process() {
	# start new process
	RUN_COUNT=$(($RUN_COUNT+1))
	RUN_ID=$(($RUN_COUNT+$RUN_COUNT_OFFSET))
	cd "$RUN_DIR"
	
	# log arguments
	echo -e "Common arguments for run $RUN_ID:\n$COMMON_LEADING_ARGS\t$COMMON_POST_ARGS\n" > "$ARGS_LOG"
	echo -e "Changing arguments for run $RUN_ID:\n$RUN_ARGS" >> "$ARGS_LOG"
	
	# start process
	if $SPEAK; then
		# write starter script...
		RUN_FILE="parallelstarter.start_run_$RUN_ID"
		echo '#!/bin/sh' > "$RUN_FILE"
		echo 'TIMESTAMP="$('"$GNUDATE"' --utc --date now "+%s")"' >> "$RUN_FILE"
		echo "RUN_ID=\"$RUN_ID\"" >> "$RUN_FILE"
		echo -n "nice -n $NICENESS \"$RUN_BIN\" $COMMON_LEADING_ARGS $RUN_ARGS $COMMON_POST_ARGS" >> "$RUN_FILE"
		$SILENT_PROC_OUT && echo -n " > /dev/null" >> "$RUN_FILE"
		echo >> "$RUN_FILE"
		echo "echo \"\$("$GNUDATE" \"+%Y-%m-%d %H:%M:%S\"): process$RUN_ID ($RUN_COUNT/$TOTAL_RUNS) finished in"\
		     '$('"$GNUDATE"' --utc --date "now -$TIMESTAMP seconds" "+%H:%M:%S"s)."' >> "$RUN_FILE"
		chmod u+x "$RUN_FILE"
		# ...and start it
		"./$RUN_FILE" &
	else
		# print errors only
		nice -n "$NICENESS" "$RUN_BIN" $COMMON_LEADING_ARGS $RUN_ARGS $COMMON_POST_ARGS > /dev/null &
	fi

	PID_LIST="$PID_LIST $!"
	PID_LIST="$(echo "$PID_LIST" | sed -e "s/^ *\(.*\) *$/\1/")"

	cd "$CURRENT_DIR"
}


# initialize vars and user customization, get command line switches
COUNT_PROC=0
CURRENT_DIR="$(pwd)"
RUN_DIR="."
RUN_COUNT=0
RUN_COUNT_OFFSET=0
RUN_ID=0
RUN_DIR_PREFIX=""
# default to babbly, -q suppress output, -Q suppress process output
SPEAK=true
SILENT_PROC_OUT=false

get_commandlineswitches "$@"
setup_run


# check for arglist file
if [ "x$ARGLIST_FILE" = "x" ]; then
	echo "You need to point the variable ARGLIST_FILE to a valid file containing the command line arguments for your processes."
	echo -e "Create a file named \"parallelstarter.setup\" in the current directory and add a line like:\n\n"\
		"\tARGLIST_FILE=/path/to/a/file/containing/the/command/line/arguments/for/each/run\n"
	exit 1
elif [ ! -e "$ARGLIST_FILE" ]; then
	echo "Argument list file not found. You need to create the configuration file named $ARGLIST_FILE, which should contain"
	echo "the command line arguments for each run on a separate line."
	exit 1
fi

# check that run_bin has been set
if [ "x$RUN_BIN" = "x" ]; then
	echo "You need to point the variable RUN_BIN to a valid executable which should get exected in parallel. "\
	      "Set this in the file named \"parallelstarter.setup\"."
	exit 1
fi

# count number of runs to do
TOTAL_RUNS="$(cat "$ARGLIST_FILE" | grep -vce "^ *	*#.*$")"

while read RUN_ARGS; do

        notstarted=true

	# control the number uf running processes
        while [ $notstarted = true ]; do
                count_proc
                if [ $COUNT_PROC -lt $MAX_PROCESSES ]; then
	              	# exclude commented out lines in the conf file (but not empty ones!)
			if [ $( echo "$RUN_ARGS" | grep -ce "^ *	*#.*$") -eq 0 ]; then
				prepare_process_start
				$SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: process$(($RUN_COUNT+1+$RUN_COUNT_OFFSET)) ($(($RUN_COUNT+1))/$TOTAL_RUNS): $STARTMSG (right now running: $COUNT_PROC)."
				start_process
                	fi
                        notstarted=false
			break;
		fi

		sleep "$RETRY"
        done

done < "$ARGLIST_FILE"
cd "$CURRENT_DIR"

# wait for all processes to finish
count_proc
if [ $COUNT_PROC -gt 0 ]; then
        $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: All jobs started - $COUNT_PROC process(es) still runnning. Waiting..."

        # wait for processes to finish
        while [ $COUNT_PROC -gt 0 ]; do
                sleep "$RETRY"
		count_proc
        done
fi

$SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: All processes finished."
$SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: Processed $RUN_COUNT jobs in $("$GNUDATE" --utc --date "now -$TIMESTAMP seconds" "+%H:%M:%S")s."


# run users post processing script
if [ -e "parallelstarter.cleanup" ]; then
	$SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: Starting post processing..."
	. ./parallelstarter.cleanup
	$SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: done."
fi

exit 0