Annotation of badi/public_scripts/parallelstarter/parallelstarter, revision 1.1.1.1
1.1 adi 1: #!/bin/bash
2:
3: # Run a command in parallel
4: #
5: # Call a process for each configured argument line in the arglist file.
6: # (provide help here, please)
7:
8: # Give the command line argument "-q" to suppress all but error output, "-Q" to suppress process' output to stdout.
9:
10: # (c) 2010 under GPL v2 by Adrian Zaugg.
11:
12:
13:
14: # if you run on a non-GNU system, you can provide
15: # the path to GNU date command here
16: GNUDATE="date"
17:
18:
19: # ----don't edit below this line-----
20: TIMESTAMP="$("$GNUDATE" --utc --date now "+%s")"
21:
22: # provide some defaults if the users setup() isn't found
23: RUN_BIN=""
24: ARGLIST_FILE=""
25: ARGS_LOG="my_args"
26: NICENESS=19
27: MAX_PROCESSES=2
28: RETRY=2
29: STARTMSG="Starting..."
30:
31: # Help message
32: function usage()
33: {
34: USAGE="
35: Usage: $PROGNAME [-d|--rundir-basename] [-i|--rundir-infix] [-o|--runid-offset] [-q|--quiet] [-Q|--quiet-run] [-h|-?|--help]
36:
37: Start any process in parallel. See /usr/local/lib/parallelstarter/README for more info.
38:
39: -d|--rundir-basename Base name for your result directories
40: -i|--rundir-infix Addition to the result directories' base name
41: -o|--runid-offset Run count start number
42: -Q|--quiet Suppress all but error messages
43: -q|--quiet-run Suppress output messages from started processes
44: -h|-?|--help This message
45:
46: To use parallelstarter create a new directory, change into it and call setup_parallelstarter.
47: "
48: # Print help
49: $SPEAK && echo "$USAGE"
50: }
51:
52: # get command line arguments
53: function get_commandlineswitches()
54: {
55: # read the users' wish
56: while [ "$#" -gt 0 ]; do
57:
58: case "$1" in
59:
60: -Q|--quiet)
61: # suppress all output but errors
62: SPEAK=false
63: shift
64: ;;
65:
66: -q|--quiet-run)
67: SILENT_PROC_OUT=true
68: shift
69: ;;
70:
71: -d|--rundir-basename)
72: shift
73: RUNDIR_BASENAME="$1"
74: shift
75: ;;
76:
77: -i|--rundir-infix)
78: shift
79: RUN_DIR_PREFIX="$1"
80: shift
81: ;;
82:
83: -o|--runid-offset)
84: shift
85: RUN_COUNT_OFFSET="$1"
86: shift
87: ;;
88:
89: -h|-?|--help)
90: shift
91: PSHELP=true
92: ;;
93:
94: *)
95: echo "Error: Undefined argument to `basename $0`. Use --help to get a list of valid arguments."
96: exit 1
97: ;;
98:
99: esac
100: done
101:
102: # check for help request
103: if [ ! -z "$PSHELP" ]; then
104: usage
105: exit 0
106: fi
107: }
108:
109: function count_proc() {
110:
111: # count the number of running processes
112:
113: if [ ! -z "$PID_LIST" ]; then
114: if [ "$(uname -s | egrep -wc "Linux|Darwin")" -eq 1 ]; then
115: # Linux or Darwin
116: if $SPEAK; then
117: psproc='/bin/sh ./parallelstarter.start_run_[0-9]+'
118: else
119: psproc="$RUN_BIN $COMMON_LEADING_ARGS"
120: psproc="$(echo $psproc) .*"
121: fi
122: COUNT_PROC=$(ps -p "$PID_LIST" -o command | egrep -cwe '^'"$psproc"'$')
123: elif [ "$(uname -o | grep -c "Solaris")" -eq 1 ]; then
124: # Solaris
125: if $SPEAK; then
126: psproc='parallel'
127: else
128: # This might fail, if RUN_BIN gets longer than 8 chars
129: # -> would start all processes at once...
130: psproc="$RUN_BIN"
131: fi
132: COUNT_PROC=$(ps -p "$PID_LIST" -o fname | grep -cwe '^'"$psproc"'$')
133: else
134: echo "Unknown OS. Only tested on Linux and Darwin."
135: echo "You need to test the function count_proc() on your system, prior to use this script."
136: exit 1
137: fi
138: else
139: COUNT_PROC=0
140: fi
141: }
142:
143: function execute_user_code() {
144: # source the file parallelstarter.setup
145: if [ -e "$USERCODE_FILE" ]; then
146: . "$USERCODE_FILE"
147: fi
148: }
149:
150: function setup_run() {
151: USERCODE_FILE="parallelstarter.setup"
152: if [ ! -e "$USERCODE_FILE" ]; then
153: echo -e "You need to provide a file called $USERCODE_FILE in the current\n"\
154: "directory. The file will be executed by the shell unconditionally!"
155: exit 1
156: fi
157: execute_user_code
158: }
159:
160: function prepare_process_start() {
161: USERCODE_FILE="parallelstarter.prepare_process_start"
162: execute_user_code
163: RUN_DIR="$(pwd)"
164: }
165:
166: function start_process() {
167: # start new process
168: RUN_COUNT=$(($RUN_COUNT+1))
169: RUN_ID=$(($RUN_COUNT+$RUN_COUNT_OFFSET))
170: cd "$RUN_DIR"
171:
172: # log arguments
173: echo -e "Common arguments for run $RUN_ID:\n$COMMON_LEADING_ARGS\t$COMMON_POST_ARGS\n" > "$ARGS_LOG"
174: echo -e "Changing arguments for run $RUN_ID:\n$RUN_ARGS" >> "$ARGS_LOG"
175:
176: # start process
177: if $SPEAK; then
178: # write starter script...
179: RUN_FILE="parallelstarter.start_run_$RUN_ID"
180: echo '#!/bin/sh' > "$RUN_FILE"
181: echo 'TIMESTAMP="$('"$GNUDATE"' --utc --date now "+%s")"' >> "$RUN_FILE"
182: echo "RUN_ID=\"$RUN_ID\"" >> "$RUN_FILE"
183: echo -n "nice -n $NICENESS \"$RUN_BIN\" $COMMON_LEADING_ARGS $RUN_ARGS $COMMON_POST_ARGS" >> "$RUN_FILE"
184: $SILENT_PROC_OUT && echo -n " > /dev/null" >> "$RUN_FILE"
185: echo >> "$RUN_FILE"
186: echo "echo \"\$("$GNUDATE" \"+%Y-%m-%d %H:%M:%S\"): process$RUN_ID ($RUN_COUNT/$TOTAL_RUNS) finished in"\
187: '$('"$GNUDATE"' --utc --date "now -$TIMESTAMP seconds" "+%H:%M:%S"s)."' >> "$RUN_FILE"
188: chmod u+x "$RUN_FILE"
189: # ...and start it
190: "./$RUN_FILE" &
191: else
192: # print errors only
193: nice -n "$NICENESS" "$RUN_BIN" $COMMON_LEADING_ARGS $RUN_ARGS $COMMON_POST_ARGS > /dev/null &
194: fi
195:
196: PID_LIST="$PID_LIST $!"
197: PID_LIST="$(echo "$PID_LIST" | sed -e "s/^ *\(.*\) *$/\1/")"
198:
199: cd "$CURRENT_DIR"
200: }
201:
202:
203: # initialize vars and user customization, get command line switches
204: COUNT_PROC=0
205: CURRENT_DIR="$(pwd)"
206: RUN_DIR="."
207: RUN_COUNT=0
208: RUN_COUNT_OFFSET=0
209: RUN_ID=0
210: RUN_DIR_PREFIX=""
211: # default to babbly, -q suppress output, -Q suppress process output
212: SPEAK=true
213: SILENT_PROC_OUT=false
214:
215: get_commandlineswitches "$@"
216: setup_run
217:
218:
219: # check for arglist file
220: if [ "x$ARGLIST_FILE" = "x" ]; then
221: echo "You need to point the variable ARGLIST_FILE to a valid file containing the command line arguments for your processes."
222: echo -e "Create a file named \"parallelstarter.setup\" in the current directory and add a line like:\n\n"\
223: "\tARGLIST_FILE=/path/to/a/file/containing/the/command/line/arguments/for/each/run\n"
224: exit 1
225: elif [ ! -e "$ARGLIST_FILE" ]; then
226: echo "Argument list file not found. You need to create the configuration file named $ARGLIST_FILE, which should contain"
227: echo "the command line arguments for each run on a separate line."
228: exit 1
229: fi
230:
231: # check that run_bin has been set
232: if [ "x$RUN_BIN" = "x" ]; then
233: echo "You need to point the variable RUN_BIN to a valid executable which should get exected in parallel. "\
234: "Set this in the file named \"parallelstarter.setup\"."
235: exit 1
236: fi
237:
238: # count number of runs to do
239: TOTAL_RUNS="$(cat "$ARGLIST_FILE" | grep -vce "^ * *#.*$")"
240:
241: while read RUN_ARGS; do
242:
243: notstarted=true
244:
245: # control the number uf running processes
246: while [ $notstarted = true ]; do
247: count_proc
248: if [ $COUNT_PROC -lt $MAX_PROCESSES ]; then
249: # exclude commented out lines in the conf file (but not empty ones!)
250: if [ $( echo "$RUN_ARGS" | grep -ce "^ * *#.*$") -eq 0 ]; then
251: prepare_process_start
252: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: process$(($RUN_COUNT+1+$RUN_COUNT_OFFSET)) ($(($RUN_COUNT+1))/$TOTAL_RUNS): $STARTMSG (right now running: $COUNT_PROC)."
253: start_process
254: fi
255: notstarted=false
256: break;
257: fi
258:
259: sleep "$RETRY"
260: done
261:
262: done < "$ARGLIST_FILE"
263: cd "$CURRENT_DIR"
264:
265: # wait for all processes to finish
266: count_proc
267: if [ $COUNT_PROC -gt 0 ]; then
268: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: All jobs started - $COUNT_PROC process(es) still runnning. Waiting..."
269:
270: # wait for processes to finish
271: while [ $COUNT_PROC -gt 0 ]; do
272: sleep "$RETRY"
273: count_proc
274: done
275: fi
276:
277: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: All processes finished."
278: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: Processed $RUN_COUNT jobs in $("$GNUDATE" --utc --date "now -$TIMESTAMP seconds" "+%H:%M:%S")s."
279:
280:
281: # run users post processing script
282: if [ -e "parallelstarter.cleanup" ]; then
283: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: Starting post processing..."
284: . ./parallelstarter.cleanup
285: $SPEAK && echo "`"$GNUDATE" "+%Y-%m-%d %H:%M:%S"`: done."
286: fi
287:
288: exit 0
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to parallelstarter CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Local Repository] / badi / public_scripts / parallelstarter