Annotation of badi/public_scripts/parallelstarter/parallelstarter, revision 1.1
1.1 ! adi 1: #!/bin/bash
! 2:
! 3: # Run a command in parallel
! 4: #
! 5: # Call a process for each configured argument line in the arglist file.
! 6: # (provide help here, please)
! 7:
! 8: # Give the command line argument "-q" to suppress all but error output, "-Q" to suppress process' output to stdout.
! 9:
! 10: # (c) 2010 under GPL v2 by Adrian Zaugg.
! 11:
! 12:
! 13:
! 14: # if you run on a non-GNU system, you can provide
! 15: # the path to GNU date command here
! 16: GNUDATE="date"
! 17:
! 18:
! 19: # ----don't edit below this line-----
! 20: TIMESTAMP="$("$GNUDATE" --utc --date now "+%s")"
! 21:
! 22: # provide some defaults if the users setup() isn't found
! 23: RUN_BIN=""
! 24: ARGLIST_FILE=""
! 25: ARGS_LOG="my_args"
! 26: NICENESS=19
! 27: MAX_PROCESSES=2
! 28: RETRY=2
! 29: STARTMSG="Starting..."
! 30:
! 31: # Help message
! 32: function usage()
! 33: {
! 34: USAGE="
! 35: Usage: $PROGNAME [-d|--rundir-basename] [-i|--rundir-infix] [-o|--runid-offset] [-q|--quiet] [-Q|--quiet-run] [-h|-?|--help]
! 36:
! 37: Start any process in parallel. See /usr/local/lib/parallelstarter/README for more info.
! 38:
! 39: -d|--rundir-basename Base name for your result directories
! 40: -i|--rundir-infix Addition to the result directories' base name
! 41: -o|--runid-offset Run count start number
! 42: -Q|--quiet Suppress all but error messages
! 43: -q|--quiet-run Suppress output messages from started processes
! 44: -h|-?|--help This message
! 45:
! 46: To use parallelstarter create a new directory, change into it and call setup_parallelstarter.
! 47: "
! 48: # Print help
! 49: $SPEAK && echo "$USAGE"
! 50: }
! 51:
! 52: # get command line arguments
! 53: function get_commandlineswitches()
! 54: {
! 55: # read the users' wish
! 56: while [ "$#" -gt 0 ]; do
! 57:
! 58: case "$1" in
! 59:
! 60: -Q|--quiet)
! 61: # suppress all output but errors
! 62: SPEAK=false
! 63: shift
! 64: ;;
! 65:
! 66: -q|--quiet-run)
! 67: SILENT_PROC_OUT=true
! 68: shift
! 69: ;;
! 70:
! 71: -d|--rundir-basename)
! 72: shift
! 73: RUNDIR_BASENAME="$1"
! 74: shift
! 75: ;;
! 76:
! 77: -i|--rundir-infix)
! 78: shift
! 79: RUN_DIR_PREFIX="$1"
! 80: shift
! 81: ;;
! 82:
! 83: -o|--runid-offset)
! 84: shift
! 85: RUN_COUNT_OFFSET="$1"
! 86: shift
! 87: ;;
! 88:
! 89: -h|-?|--help)
! 90: shift
! 91: PSHELP=true
! 92: ;;
! 93:
! 94: *)
! 95: echo "Error: Undefined argument to `basename $0`. Use --help to get a list of valid arguments."
! 96: exit 1
! 97: ;;
! 98:
! 99: esac
! 100: done
! 101:
! 102: # check for help request
! 103: if [ ! -z "$PSHELP" ]; then
! 104: usage
! 105: exit 0
! 106: fi
! 107: }
! 108:
! 109: function count_proc() {
! 110:
! 111: # count the number of running processes
! 112:
! 113: if [ ! -z "$PID_LIST" ]; then
! 114: if [ "$(uname -s | egrep -wc "Linux|Darwin")" -eq 1 ]; then
! 115: # Linux or Darwin
! 116: if $SPEAK; then
! 117: psproc='/bin/sh ./parallelstarter.start_run_[0-9]+'
! 118: else
! 119: psproc="$RUN_BIN $COMMON_LEADING_ARGS"
! 120: psproc="$(echo $psproc) .*"
! 121: fi
! 122: COUNT_PROC=$(ps -p "$PID_LIST" -o command | egrep -cwe '^'"$psproc"'$')
! 123: elif [ "$(uname -o | grep -c "Solaris")" -eq 1 ]; then
! 124: # Solaris
! 125: if $SPEAK; then
! 126: psproc='parallel'
! 127: else
! 128: # This might fail, if RUN_BIN gets longer than 8 chars
! 129: # -> would start all processes at once...
! 130: psproc="$RUN_BIN"
! 131: fi
! 132: COUNT_PROC=$(ps -p "$PID_LIST" -o fname | grep -cwe '^'"$psproc"'$')
! 133: else
! 134: echo "Unknown OS. Only tested on Linux and Darwin."
! 135: echo "You need to test the function count_proc() on your system, prior to use this script."
! 136: exit 1
! 137: fi
! 138: else
! 139: COUNT_PROC=0
! 140: fi
! 141: }
! 142:
! 143: function execute_user_code() {
! 144: # source the file parallelstarter.setup
! 145: if [ -e "$USERCODE_FILE" ]; then
! 146: . "$USERCODE_FILE"
! 147: fi
! 148: }
! 149:
! 150: function setup_run() {
! 151: USERCODE_FILE="parallelstarter.setup"
! 152: if [ ! -e "$USERCODE_FILE" ]; then
! 153: echo -e "You need to provide a file called $USERCODE_FILE in the current\n"\
! 154: "directory. The file will be executed by the shell unconditionally!"
! 155: exit 1
! 156: fi
! 157: execute_user_code
! 158: }
! 159:
! 160: function prepare_process_start() {
! 161: USERCODE_FILE="parallelstarter.prepare_process_start"
! 162: execute_user_code
! 163: RUN_DIR="$(pwd)"
! 164: }
! 165:
! 166: function start_process() {
! 167: # start new process
! 168: RUN_COUNT=$(($RUN_COUNT+1))
! 169: RUN_ID=$(($RUN_COUNT+$RUN_COUNT_OFFSET))
! 170: cd "$RUN_DIR"
! 171:
! 172: # log arguments
! 173: echo -e "Common arguments for run $RUN_ID:\n$COMMON_LEADING_ARGS\t$COMMON_POST_ARGS\n" > "$ARGS_LOG"
! 174: echo -e "Changing arguments for run $RUN_ID:\n$RUN_ARGS" >> "$ARGS_LOG"
! 175:
! 176: # start process
! 177: if $SPEAK; then
! 178: # write starter script...
! 179: RUN_FILE="parallelstarter.start_run_$RUN_ID"
! 180: echo '#!/bin/sh' > "$RUN_FILE"
! 181: echo 'TIMESTAMP="$('"$GNUDATE"' --utc --date now "+%s")"' >> "$RUN_FILE"
! 182: echo "RUN_ID=\"$RUN_ID\"" >> "$RUN_FILE"
! 183: echo -n "nice -n $NICENESS \"$RUN_BIN\" $COMMON_LEADING_ARGS $RUN_ARGS $COMMON_POST_ARGS" >> "$RUN_FILE"
! 184: $SILENT_PROC_OUT && echo -n " > /dev/null" >> "$RUN_FILE"
! 185: echo >> "$RUN_FILE"
! 186: echo "echo \"\$("$GNUDATE" \"+%Y-%m-%d %H:%M:%S\"): process$RUN_ID ($RUN_COUNT/$TOTAL_RUNS) finished in"\
! 187: '$('"$GNUDATE"' --utc --date "now -$TIMESTAMP seconds" "+%H:%M:%S"s)."' >> "$RUN_FILE"
! 188: chmod u+x "$RUN_FILE"
! 189: # ...and start it
! 190: "./$RUN_FILE" &
! 191: else
! 192: # print errors only
! 193: nice -n "$NICENESS" "$RUN_BIN" $COMMON_LEADING_ARGS $RUN_ARGS $COMMON_POST_ARGS > /dev/null &
! 194: fi
! 195:
! 196: PID_LIST="$PID_LIST $!"
! 197: PID_LIST="$(echo "$PID_LIST" | sed -e "s/^ *\(.*\) *$/\1/")"
! 198:
! 199: cd "$CURRENT_DIR"
! 200: }
! 201:
! 202:
! 203: # initialize vars and user customization, get command line switches
! 204: COUNT_PROC=0
! 205: CURRENT_DIR="$(pwd)"
! 206: RUN_DIR="."
! 207: RUN_COUNT=0
! 208: RUN_COUNT_OFFSET=0
! 209: RUN_ID=0
! 210: RUN_DIR_PREFIX=""
! 211: # default to babbly, -q suppress output, -Q suppress process output
! 212: SPEAK=true
! 213: SILENT_PROC_OUT=false
! 214:
! 215: get_commandlineswitches "$@"
! 216: setup_run
! 217:
! 218:
! 219: # check for arglist file
! 220: if [ "x$ARGLIST_FILE" = "x" ]; then
! 221: echo "You need to point the variable ARGLIST_FILE to a valid file containing the command line arguments for your processes."
! 222: echo -e "Create a file named \"parallelstarter.setup\" in the current directory and add a line like:\n\n"\
! 223: "\tARGLIST_FILE=/path/to/a/file/containing/the/command/line/arguments/for/each/run\n"
! 224: exit 1
! 225: elif [ ! -e "$ARGLIST_FILE" ]; then
! 226: echo "Argument list file not found. You need to create the configuration file named $ARGLIST_FILE, which should contain"
! 227: echo "the command line arguments for each run on a separate line."
! 228: exit 1
! 229: fi
! 230:
! 231: # check that run_bin has been set
! 232: if [ "x$RUN_BIN" = "x" ]; then
! 233: echo "You need to point the variable RUN_BIN to a valid executable which should get exected in parallel. "\
! 234: "Set this in the file named \"parallelstarter.setup\"."
! 235: exit 1
! 236: fi
! 237:
! 238: # count number of runs to do
! 239: TOTAL_RUNS="$(cat "$ARGLIST_FILE" | grep -vce "^ * *#.*$")"
! 240:
! 241: while read RUN_ARGS; do
! 242:
! 243: notstarted=true
! 244:
! 245: # control the number uf running processes
! 246: while [ $notstarted = true ]; do
! 247: count_proc
! 248: if [ $COUNT_PROC -lt $MAX_PROCESSES ]; then
! 249: # exclude commented out lines in the conf file (but not empty ones!)
! 250: if [ $( echo "$RUN_ARGS" | grep -ce "^ * *#.*$") -eq 0 ]; then
! 251: prepare_process_start
! 252: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: process$(($RUN_COUNT+1+$RUN_COUNT_OFFSET)) ($(($RUN_COUNT+1))/$TOTAL_RUNS): $STARTMSG (right now running: $COUNT_PROC)."
! 253: start_process
! 254: fi
! 255: notstarted=false
! 256: break;
! 257: fi
! 258:
! 259: sleep "$RETRY"
! 260: done
! 261:
! 262: done < "$ARGLIST_FILE"
! 263: cd "$CURRENT_DIR"
! 264:
! 265: # wait for all processes to finish
! 266: count_proc
! 267: if [ $COUNT_PROC -gt 0 ]; then
! 268: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: All jobs started - $COUNT_PROC process(es) still runnning. Waiting..."
! 269:
! 270: # wait for processes to finish
! 271: while [ $COUNT_PROC -gt 0 ]; do
! 272: sleep "$RETRY"
! 273: count_proc
! 274: done
! 275: fi
! 276:
! 277: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: All processes finished."
! 278: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: Processed $RUN_COUNT jobs in $("$GNUDATE" --utc --date "now -$TIMESTAMP seconds" "+%H:%M:%S")s."
! 279:
! 280:
! 281: # run users post processing script
! 282: if [ -e "parallelstarter.cleanup" ]; then
! 283: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: Starting post processing..."
! 284: . ./parallelstarter.cleanup
! 285: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: done."
! 286: fi
! 287:
! 288: exit 0
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to parallelstarter CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Local Repository] / badi / public_scripts / parallelstarter