# HG changeset patch
# User aw148015
# Date 1204054740 28800
# Node ID d5f45b4dae7e241fa099a99a7b5d0db393d67729
# Parent  23e77aa611b1388f10301e3de57b479f2780d381
6581656 invalid data should be friendlier to users
6592825 filebench should allow certain workloads run to full completion w/out requiring runtime specified

diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/filebench.h
--- a/usr/src/cmd/filebench/common/filebench.h	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/filebench.h	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -66,7 +66,8 @@
 extern "C" {
 #endif
 
-extern pid_t pid;
+extern pid_t my_pid;		/* this process' process id */
+extern procflow_t *my_procflow;	/* if slave process, procflow pointer */
 extern int errno;
 extern char *execname;
 extern int noproc;
@@ -102,11 +103,17 @@
 #define	MIN(x, y) ((x) < (y) ? (x) : (y))
 #endif
 
-#define	FILEBENCH_VERSION	"1.1.0"
+#define	FILEBENCH_VERSION	"1.1.1"
 #define	FILEBENCHDIR	"/usr/benchmarks/filebench"
 #define	FILEBENCH_PROMPT	"filebench> "
 #define	MAX_LINE_LEN	1024
 #define	MAX_CMD_HIST	128
+#define	SHUTDOWN_WAIT_SECONDS	5 /* time to wait for proc / thrd to quit */
+
+#define	FILEBENCH_DONE	 1
+#define	FILEBENCH_OK	 0
+#define	FILEBENCH_ERROR -1
+#define	FILEBENCH_NORSC -2
 
 /* For MacOSX */
 #ifndef HAVE_OFF64_T
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/flowop.c
--- a/usr/src/cmd/filebench/common/flowop.c	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/flowop.c	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -87,7 +87,7 @@
 		char procname[128];
 
 		(void) snprintf(procname, sizeof (procname),
-		    "/proc/%d/lwp/%d/lwpusage", pid, _lwp_self());
+		    "/proc/%d/lwp/%d/lwpusage", my_pid, _lwp_self());
 		threadflow->tf_lwpusagefd = open(procname, O_RDONLY);
 	}
 
@@ -192,6 +192,45 @@
 }
 
 /*
+ * Calls the flowop's destruct function, pointed to by
+ * flowop->fo_destruct.
+ */
+static void
+flowop_destructflow(flowop_t *flowop)
+{
+	(*flowop->fo_destruct)(flowop);
+}
+
+/*
+ * call the destruct funtions of all the threadflow's flowops,
+ * if it is still flagged as "running".
+ */
+void
+flowop_destruct_all_flows(threadflow_t *threadflow)
+{
+	flowop_t *flowop;
+
+	(void) ipc_mutex_lock(&threadflow->tf_lock);
+
+	/* prepare to call destruct flow routines, if necessary */
+	if (threadflow->tf_running == 0) {
+
+		/* allready destroyed */
+		(void) ipc_mutex_unlock(&threadflow->tf_lock);
+		return;
+	}
+
+	flowop = threadflow->tf_ops;
+	threadflow->tf_running = 0;
+	(void) ipc_mutex_unlock(&threadflow->tf_lock);
+
+	while (flowop) {
+		flowop_destructflow(flowop);
+		flowop = flowop->fo_threadnext;
+	}
+}
+
+/*
  * The final initialization and main execution loop for the
  * worker threads. Sets threadflow and flowop start times,
  * waits for all process to start, then creates the runtime
@@ -210,8 +249,6 @@
 	size_t memsize;
 	int ret = 0;
 
-	pid = getpid();
-
 #ifdef HAVE_PROCFS
 	if (noproc == 0) {
 		char procname[128];
@@ -219,7 +256,7 @@
 		int pfd;
 
 		(void) snprintf(procname, sizeof (procname),
-		    "/proc/%d/lwp/%d/lwpctl", pid, _lwp_self());
+		    "/proc/%d/lwp/%d/lwpctl", my_pid, _lwp_self());
 		pfd = open(procname, O_WRONLY);
 		(void) pwrite(pfd, &ctl, sizeof (ctl), 0);
 		(void) close(pfd);
@@ -303,12 +340,8 @@
 		int i;
 
 		/* Abort if asked */
-		if (threadflow->tf_abort || filebench_shm->f_abort) {
-			(void) ipc_mutex_lock(&threadflow->tf_lock);
-			threadflow->tf_running = 0;
-			(void) ipc_mutex_unlock(&threadflow->tf_lock);
+		if (threadflow->tf_abort || filebench_shm->f_abort)
 			break;
-		}
 
 		/* Be quiet while stats are gathered */
 		if (filebench_shm->bequiet) {
@@ -317,7 +350,7 @@
 		}
 
 		/* Take it easy until everyone is ready to go */
-		if (!filebench_shm->allrunning)
+		if (!filebench_shm->shm_running)
 			(void) sleep(1);
 
 		if (flowop->fo_stats.fs_stime == 0)
@@ -328,13 +361,6 @@
 			return;
 		}
 
-		if (threadflow->tf_memsize == 0) {
-			filebench_log(LOG_ERROR,
-			    "Zero memory size for thread %s",
-			    threadflow->tf_name);
-			return;
-		}
-
 		filebench_log(LOG_DEBUG_SCRIPT, "%s: executing flowop %s-%d",
 		    threadflow->tf_name, flowop->fo_name, flowop->fo_instance);
 
@@ -348,30 +374,62 @@
 			    "%s-%d", threadflow->tf_name, flowop->fo_name,
 			    flowop->fo_instance);
 
-			/* Return value > 0 means "stop the filebench run" */
-			if (ret > 0) {
-				filebench_log(LOG_VERBOSE,
-				    "%s: exiting flowop %s-%d",
-				    threadflow->tf_name, flowop->fo_name,
+			/*
+			 * Return value FILEBENCH_ERROR means "flowop
+			 * failed, stop the filebench run"
+			 */
+			if (ret == FILEBENCH_ERROR) {
+				filebench_log(LOG_ERROR,
+				    "%s-%d: flowop %s-%d failed",
+				    threadflow->tf_name,
+				    threadflow->tf_instance,
+				    flowop->fo_name,
 				    flowop->fo_instance);
 				(void) ipc_mutex_lock(&threadflow->tf_lock);
 				threadflow->tf_abort = 1;
-				filebench_shm->f_abort = 1;
-				threadflow->tf_running = 0;
+				filebench_shm->f_abort = FILEBENCH_ABORT_ERROR;
 				(void) ipc_mutex_unlock(&threadflow->tf_lock);
 				break;
 			}
+
 			/*
-			 * Return value < 0 means "flowop failed, stop the
-			 * filebench run"
+			 * Return value of FILEBENCH_NORSC means "stop
+			 * the filebench run" if in "end on no work mode",
+			 * otherwise it indicates an error
 			 */
-			if (ret < 0) {
-				filebench_log(LOG_ERROR, "flowop %s failed",
-				    flowop->fo_name);
+			if (ret == FILEBENCH_NORSC) {
 				(void) ipc_mutex_lock(&threadflow->tf_lock);
-				threadflow->tf_abort = 1;
-				filebench_shm->f_abort = 1;
-				threadflow->tf_running = 0;
+				threadflow->tf_abort = FILEBENCH_DONE;
+				if (filebench_shm->shm_rmode ==
+				    FILEBENCH_MODE_Q1STDONE) {
+					filebench_shm->f_abort =
+					    FILEBENCH_ABORT_RSRC;
+				} else if (filebench_shm->shm_rmode !=
+				    FILEBENCH_MODE_QALLDONE) {
+					filebench_log(LOG_ERROR1,
+					    "WARNING! Run stopped early:\n   "
+					    "             flowop %s-%d could "
+					    "not obtain a file. Please\n      "
+					    "          reduce runtime, "
+					    "increase fileset entries "
+					    "($nfiles), or switch modes.",
+					    flowop->fo_name,
+					    flowop->fo_instance);
+					filebench_shm->f_abort =
+					    FILEBENCH_ABORT_ERROR;
+				}
+				(void) ipc_mutex_unlock(&threadflow->tf_lock);
+				break;
+			}
+
+			/*
+			 * Return value of FILEBENCH_DONE means "stop
+			 * the filebench run without error"
+			 */
+			if (ret == FILEBENCH_DONE) {
+				(void) ipc_mutex_lock(&threadflow->tf_lock);
+				threadflow->tf_abort = FILEBENCH_DONE;
+				filebench_shm->f_abort = FILEBENCH_ABORT_DONE;
 				(void) ipc_mutex_unlock(&threadflow->tf_lock);
 				break;
 			}
@@ -392,7 +450,10 @@
 	    _lwp_self());
 #endif
 
-	pthread_exit(&ret);
+	/* Tell flowops to destroy locally acquired state */
+	flowop_destruct_all_flows(threadflow);
+
+	pthread_exit(&threadflow->tf_abort);
 }
 
 void
@@ -402,19 +463,7 @@
 }
 
 /*
- * Calls the flowop's destruct function, pointed to by
- * flowop->fo_destruct.
- */
-static void
-flowop_destructflow(flowop_t *flowop)
-{
-	(*flowop->fo_destruct)(flowop);
-}
-
-/*
  * Delete the designated flowop from the thread's flowop list.
- * After removal from the list, the flowop is destroyed with
- * flowop_destructflow().
  */
 static void
 flowop_delete(flowop_t **flowoplist, flowop_t *flowop)
@@ -457,9 +506,6 @@
 		}
 	}
 
-	/* Call destructor */
-	flowop_destructflow(flowop);
-
 #ifdef HAVE_PROCFS
 	/* Close /proc stats */
 	if (flowop->fo_thread)
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/flowop.h
--- a/usr/src/cmd/filebench/common/flowop.h	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/flowop.h	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -136,6 +136,7 @@
 void flowop_delete_all(flowop_t **threadlist);
 void flowop_endop(threadflow_t *threadflow, flowop_t *flowop, int64_t bytes);
 void flowop_beginop(threadflow_t *threadflow, flowop_t *flowop);
+void flowop_destruct_all_flows(threadflow_t *threadflow);
 
 #ifdef	__cplusplus
 }
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/flowop_library.c
--- a/usr/src/cmd/filebench/common/flowop_library.c	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/flowop_library.c	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -99,6 +99,7 @@
 
 static int flowoplib_init_generic(flowop_t *flowop);
 static void flowoplib_destruct_generic(flowop_t *flowop);
+static void flowoplib_destruct_noop(flowop_t *flowop);
 static int flowoplib_fdnum(threadflow_t *threadflow, flowop_t *flowop);
 static int flowoplib_write(threadflow_t *threadflow, flowop_t *flowop);
 #ifdef HAVE_AIO
@@ -162,7 +163,7 @@
 	FLOW_TYPE_SYNC, 0, "semblock", flowoplib_semblock_init,
 	flowoplib_semblock, flowoplib_semblock_destruct,
 	FLOW_TYPE_SYNC, 0, "sempost", flowoplib_sempost_init,
-	flowoplib_sempost, flowoplib_destruct_generic,
+	flowoplib_sempost, flowoplib_destruct_noop,
 	FLOW_TYPE_OTHER, 0, "hog", flowoplib_init_generic,
 	flowoplib_hog, flowoplib_destruct_generic,
 	FLOW_TYPE_OTHER, 0, "delay", flowoplib_init_generic,
@@ -242,16 +243,31 @@
 flowoplib_init_generic(flowop_t *flowop)
 {
 	(void) ipc_mutex_unlock(&flowop->fo_lock);
-	return (0);
+	return (FILEBENCH_OK);
 }
 
-/* ARGSUSED */
 static void
 flowoplib_destruct_generic(flowop_t *flowop)
 {
-	/* release any resources held by the flowop */
-	if (flowop->fo_buf)
-		free(flowop->fo_buf);
+	char *buf;
+
+	/* release any local resources held by the flowop */
+	(void) ipc_mutex_lock(&flowop->fo_lock);
+	buf = flowop->fo_buf;
+	flowop->fo_buf = NULL;
+	(void) ipc_mutex_unlock(&flowop->fo_lock);
+
+	if (buf)
+		free(buf);
+}
+
+/*
+ * Special total noop destruct
+ */
+/* ARGSUSED */
+static void
+flowoplib_destruct_noop(flowop_t *flowop)
+{
 }
 
 /*
@@ -309,7 +325,7 @@
 		filebench_log(LOG_ERROR, "Out of file descriptors in flowop %s"
 		    " (too many files : %d", flowop->fo_name,
 		    *(flowop->fo_fileset->fs_entries));
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	/* First time around */
@@ -331,7 +347,9 @@
 /*
  * Determines the file descriptor to use, and attempts to open
  * the file if it is not already open. Also determines the wss
- * value. Returns -1 on errors, 0 otherwise.
+ * value. Returns FILEBENCH_ERROR on errors, FILESET_NORSC if
+ * if flowop_openfile_common couldn't obtain an appropriate file
+ * from a the fileset, and FILEBENCH_OK otherwise.
  */
 static int
 flowoplib_filesetup(threadflow_t *threadflow, flowop_t *flowop,
@@ -340,12 +358,14 @@
 	int fd = flowoplib_fdnum(threadflow, flowop);
 
 	if (fd == -1)
-		return (-1);
+		return (FILEBENCH_ERROR);
 
 	if (threadflow->tf_fd[fd] == 0) {
-		if (flowoplib_openfile_common(
-		    threadflow, flowop, fd) == -1)
-			return (-1);
+		int ret;
+
+		if ((ret = flowoplib_openfile_common(
+		    threadflow, flowop, fd)) != FILEBENCH_OK)
+			return (ret);
 
 		if (threadflow->tf_fse[fd]) {
 			filebench_log(LOG_DEBUG_IMPL, "opened file %s",
@@ -369,12 +389,12 @@
 		*wssp = *flowop->fo_wss;
 	}
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Determines the io buffer or random offset into tf_mem for
- * the IO operation. Returns -1 on errors, 0 otherwise.
+ * the IO operation. Returns FILEBENCH_ERROR on errors, FILEBENCH_OK otherwise.
  */
 static int
 flowoplib_iobufsetup(threadflow_t *threadflow, flowop_t *flowop,
@@ -386,7 +406,7 @@
 	if (iosize == 0) {
 		filebench_log(LOG_ERROR, "zero iosize for thread %s",
 		    flowop->fo_name);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	if ((memsize = *threadflow->tf_memsize) != 0) {
@@ -396,7 +416,7 @@
 			filebench_log(LOG_ERROR,
 			    "tf_memsize smaller than IO size for thread %s",
 			    flowop->fo_name);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 		*iobufp = threadflow->tf_mem + memoffset;
 
@@ -409,30 +429,34 @@
 		}
 		if ((flowop->fo_buf == NULL) && ((flowop->fo_buf
 		    = (char *)malloc(iosize)) == NULL))
-				return (-1);
+			return (FILEBENCH_ERROR);
 
 		flowop->fo_buf_size = iosize;
 		*iobufp = flowop->fo_buf;
 	}
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Determines the file descriptor to use, opens it if necessary, the
  * io buffer or random offset into tf_mem for IO operation and the wss
- * value. Returns -1 on errors, 0 otherwise.
+ * value. Returns FILEBENCH_ERROR on errors, FILEBENCH_OK otherwise.
  */
 static int
 flowoplib_iosetup(threadflow_t *threadflow, flowop_t *flowop,
     vinteger_t *wssp, caddr_t *iobufp, int *filedescp, vinteger_t iosize)
 {
-	if (flowoplib_filesetup(threadflow, flowop, wssp, filedescp) == -1)
-		return (-1);
+	int ret;
+
+	if ((ret = flowoplib_filesetup(threadflow, flowop, wssp, filedescp)) !=
+	    FILEBENCH_OK)
+		return (ret);
 
-	if (flowoplib_iobufsetup(threadflow, flowop, iobufp, iosize) == -1)
-		return (-1);
+	if ((ret = flowoplib_iobufsetup(threadflow, flowop, iobufp, iosize)) !=
+	    FILEBENCH_OK)
+		return (ret);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -440,15 +464,16 @@
  * a file descriptor number index is fetched, otherwise a
  * supplied fileobj file is used. In either case the specified
  * file will be opened if not already open. If the flowop has
- * neither a fileset or fileobj, an error is logged and -1
+ * neither a fileset or fileobj, an error is logged and FILEBENCH_ERROR
  * returned.
  *
  * The actual read is done to a random offset in the
  * threadflow's thread memory (tf_mem), with a size set by
  * fo_iosize and at either a random disk offset within the
  * working set size, or at the next sequential location. If
- * any errors are encountered, -1 is returned, if successful,
- * 0 is returned.
+ * any errors are encountered, FILEBENCH_ERROR is returned,
+ * if no appropriate file can be obtained from the fileset then
+ * FILEBENCH_NORSC is returned, otherise FILEBENCH_OK is returned.
  */
 static int
 flowoplib_read(threadflow_t *threadflow, flowop_t *flowop)
@@ -458,9 +483,9 @@
 	int filedesc;
 	int ret;
 
-	if (flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, *flowop->fo_iosize) != 0)
-		return (-1);
+	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
+	    &filedesc, *flowop->fo_iosize)) != FILEBENCH_OK)
+		return (ret);
 
 	if (*flowop->fo_random) {
 		uint64_t fileoffset;
@@ -470,7 +495,7 @@
 			filebench_log(LOG_ERROR,
 			    "file size smaller than IO size for thread %s",
 			    flowop->fo_name);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 
 		(void) flowop_beginop(threadflow, flowop);
@@ -483,7 +508,7 @@
 			    flowop->fo_fileset->fs_name,
 			    fileoffset, iobuf, strerror(errno));
 			flowop_endop(threadflow, flowop, 0);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 		(void) flowop_endop(threadflow, flowop, ret);
 
@@ -499,7 +524,7 @@
 			    flowop->fo_fileset->fs_name,
 			    iobuf, strerror(errno));
 			(void) flowop_endop(threadflow, flowop, 0);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 		(void) flowop_endop(threadflow, flowop, ret);
 
@@ -507,7 +532,7 @@
 			(void) lseek64(filedesc, 0, SEEK_SET);
 	}
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 #ifdef HAVE_AIO
@@ -551,9 +576,9 @@
 
 /*
  * Searches for the aiolist element that has a matching
- * completion block, aiocb. If none found returns -1. If
+ * completion block, aiocb. If none found returns FILEBENCH_ERROR. If
  * found, removes the aiolist element from flowop thread's
- * list and returns 0.
+ * list and returns FILEBENCH_OK.
  */
 static int
 aio_deallocate(flowop_t *flowop, struct aiocb64 *aiocb)
@@ -564,7 +589,7 @@
 
 	if (aiocb == NULL) {
 		filebench_log(LOG_ERROR, "null aiocb deallocate");
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
 	while (aiolist) {
@@ -577,7 +602,7 @@
 	}
 
 	if (match == NULL)
-		return (-1);
+		return (FILEBENCH_ERROR);
 
 	/* Remove from the list */
 	if (previous)
@@ -585,7 +610,7 @@
 	else
 		flowop->fo_thread->tf_aiolist = match->al_next;
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -594,8 +619,10 @@
  * with a fileobj, allocates and fills an aiolist_t element
  * for the write, and issues the asynchronous write. This
  * operation is only valid for random IO, and returns an
- * error if the flowop is set for sequential IO. Returns 0
- * on success, -1 on any encountered error.
+ * error if the flowop is set for sequential IO. Returns
+ * FILEBENCH_OK on success, FILEBENCH_NORSC if iosetup can't
+ * obtain a file to open, and FILEBENCH_ERROR on any
+ * encountered error.
  */
 static int
 flowoplib_aiowrite(threadflow_t *threadflow, flowop_t *flowop)
@@ -603,10 +630,11 @@
 	caddr_t iobuf;
 	vinteger_t wss;
 	int filedesc;
+	int ret;
 
-	if (flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, *flowop->fo_iosize) != 0)
-		return (-1);
+	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
+	    &filedesc, *flowop->fo_iosize)) != FILEBENCH_OK)
+		return (ret);
 
 	if (*flowop->fo_random) {
 		uint64_t fileoffset;
@@ -618,7 +646,7 @@
 			filebench_log(LOG_ERROR,
 			    "file size smaller than IO size for thread %s",
 			    flowop->fo_name);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 
 		aiolist = aio_allocate(flowop);
@@ -643,10 +671,10 @@
 		}
 		flowop_endop(threadflow, flowop, *flowop->fo_iosize);
 	} else {
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 
@@ -720,7 +748,7 @@
 				filebench_log(LOG_ERROR, "Could not remove "
 				    "aio from list ");
 				flowop_endop(threadflow, flowop, 0);
-				return (-1);
+				return (FILEBENCH_ERROR);
 			}
 		}
 
@@ -752,7 +780,7 @@
 				filebench_log(LOG_ERROR, "Could not remove aio "
 				    "from list ");
 				flowop_endop(threadflow, flowop, 0);
-				return (-1);
+				return (FILEBENCH_ERROR);
 			}
 		}
 
@@ -769,7 +797,7 @@
 
 	free(worklist);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 #endif /* HAVE_AIO */
@@ -786,7 +814,7 @@
 	(void) pthread_cond_init(&flowop->fo_cv, ipc_condattr());
 	(void) ipc_mutex_unlock(&flowop->fo_lock);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -809,7 +837,7 @@
 
 	(void) ipc_mutex_unlock(&flowop->fo_lock);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -820,7 +848,7 @@
  * flowops whose name matches this flowop's "fo_targetname"
  * attribute. The target list is generated on the first
  * invocation, and the run will be shutdown if no targets
- * are found. Otherwise the routine always returns 0.
+ * are found. Otherwise the routine always returns FILEBENCH_OK.
  */
 static int
 flowoplib_wakeup(threadflow_t *threadflow, flowop_t *flowop)
@@ -869,7 +897,7 @@
 		target = target->fo_targetnext;
 	}
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -898,7 +926,7 @@
 	}
 	flowop_endop(threadflow, flowop, 0);
 	filebench_log(LOG_DEBUG_IMPL, "hog exit");
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 
@@ -913,7 +941,7 @@
 	flowop_beginop(threadflow, flowop);
 	(void) sleep(value);
 	flowop_endop(threadflow, flowop, 0);
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -938,7 +966,7 @@
 {
 	/* Immediately bail if not set/enabled */
 	if (filebench_shm->eventgen_hz == 0)
-		return (0);
+		return (FILEBENCH_OK);
 
 	if (flowop->fo_initted == 0) {
 		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
@@ -959,14 +987,14 @@
 		(void) ipc_mutex_unlock(&filebench_shm->eventgen_lock);
 	}
 	flowop_endop(threadflow, flowop, 0);
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Blocks the calling thread if the number of issued I/O
  * operations exceeds the number of posted events, thus
  * limiting the average I/O operation rate to the rate
- * specified by eventgen_hz. Always returns 0.
+ * specified by eventgen_hz. Always returns FILEBENCH_OK.
  */
 static int
 flowoplib_iopslimit(threadflow_t *threadflow, flowop_t *flowop)
@@ -977,7 +1005,7 @@
 
 	/* Immediately bail if not set/enabled */
 	if (filebench_shm->eventgen_hz == 0)
-		return (0);
+		return (FILEBENCH_OK);
 
 	if (flowop->fo_initted == 0) {
 		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
@@ -991,7 +1019,7 @@
 	/* Is this the first time around */
 	if (flowop->fo_tputlast == 0) {
 		flowop->fo_tputlast = iops;
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
 	delta = iops - flowop->fo_tputlast;
@@ -1001,7 +1029,7 @@
 	/* No need to block if the q isn't empty */
 	if (flowop->fo_tputbucket >= 0LL) {
 		flowop_endop(threadflow, flowop, 0);
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
 	iops = flowop->fo_tputbucket * -1;
@@ -1023,14 +1051,14 @@
 	}
 	flowop_endop(threadflow, flowop, 0);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Blocks the calling thread if the number of issued filebench
  * operations exceeds the number of posted events, thus limiting
  * the average filebench operation rate to the rate specified by
- * eventgen_hz. Always returns 0.
+ * eventgen_hz. Always returns FILEBENCH_OK.
  */
 static int
 flowoplib_opslimit(threadflow_t *threadflow, flowop_t *flowop)
@@ -1041,7 +1069,7 @@
 
 	/* Immediately bail if not set/enabled */
 	if (filebench_shm->eventgen_hz == 0)
-		return (0);
+		return (FILEBENCH_OK);
 
 	if (flowop->fo_initted == 0) {
 		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
@@ -1054,7 +1082,7 @@
 	/* Is this the first time around */
 	if (flowop->fo_tputlast == 0) {
 		flowop->fo_tputlast = ops;
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
 	delta = ops - flowop->fo_tputlast;
@@ -1064,7 +1092,7 @@
 	/* No need to block if the q isn't empty */
 	if (flowop->fo_tputbucket >= 0LL) {
 		flowop_endop(threadflow, flowop, 0);
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
 	ops = flowop->fo_tputbucket * -1;
@@ -1085,7 +1113,7 @@
 	}
 	flowop_endop(threadflow, flowop, 0);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 
@@ -1094,7 +1122,7 @@
  * issued exceeds one megabyte times the number of posted
  * events, thus limiting the average I/O byte rate to one
  * megabyte times the event rate as set by eventgen_hz.
- * Always retuns 0.
+ * Always retuns FILEBENCH_OK.
  */
 static int
 flowoplib_bwlimit(threadflow_t *threadflow, flowop_t *flowop)
@@ -1105,7 +1133,7 @@
 
 	/* Immediately bail if not set/enabled */
 	if (filebench_shm->eventgen_hz == 0)
-		return (0);
+		return (FILEBENCH_OK);
 
 	if (flowop->fo_initted == 0) {
 		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
@@ -1119,7 +1147,7 @@
 	/* Is this the first time around */
 	if (flowop->fo_tputlast == 0) {
 		flowop->fo_tputlast = bytes;
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
 	delta = bytes - flowop->fo_tputlast;
@@ -1129,7 +1157,7 @@
 	/* No need to block if the q isn't empty */
 	if (flowop->fo_tputbucket >= 0LL) {
 		flowop_endop(threadflow, flowop, 0);
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
 	bytes = flowop->fo_tputbucket * -1;
@@ -1153,7 +1181,7 @@
 	}
 	flowop_endop(threadflow, flowop, 0);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -1180,18 +1208,18 @@
 	flowop_beginop(threadflow, flowop);
 	if (b > bytes) {
 		flowop_endop(threadflow, flowop, 0);
-		return (1);
+		return (FILEBENCH_DONE);
 	}
 	flowop_endop(threadflow, flowop, 0);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Stop filebench run when specified number of I/O operations have
  * been performed. Compares controlstats.fs_count with *flowop->value,
- * and if greater returns 1, stopping the run, if not, returns 0 to
- * continue running.
+ * and if greater returns 1, stopping the run, if not, returns FILEBENCH_OK
+ * to continue running.
  */
 static int
 flowoplib_finishoncount(threadflow_t *threadflow, flowop_t *flowop)
@@ -1202,13 +1230,13 @@
 	ops = controlstats.fs_count;
 
 	flowop_beginop(threadflow, flowop);
-	if (ops > count) {
+	if (ops >= count) {
 		flowop_endop(threadflow, flowop, 0);
-		return (1);
+		return (FILEBENCH_DONE);
 	}
 	flowop_endop(threadflow, flowop, 0);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -1225,8 +1253,8 @@
  * already been initialized, also allocates a pair of semids
  * and initializes the highwater System V semaphore.
  * If no System V semaphores, then does nothing special.
- * Returns -1 if it cannot acquire a set of System V semphores
- * or if the initial post to the semaphore set fails. Returns 0
+ * Returns FILEBENCH_ERROR if it cannot acquire a set of System V semphores
+ * or if the initial post to the semaphore set fails. Returns FILEBENCH_OK
  * on success.
  */
 static int
@@ -1254,7 +1282,7 @@
 		filebench_log(LOG_ERROR, "semblock init lookup %x failed: %s",
 		    filebench_shm->semkey,
 		    strerror(errno));
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	if ((highwater = flowop->fo_semid_hw) == 0)
@@ -1268,7 +1296,7 @@
 	if ((semop(semid, &sbuf[0], 1) == -1) && errno) {
 		filebench_log(LOG_ERROR, "semblock init post failed: %s (%d,"
 		    "%d)", strerror(errno), sbuf[0].sem_num, sbuf[0].sem_op);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 #else
 	filebench_log(LOG_DEBUG_IMPL,
@@ -1281,13 +1309,13 @@
 	if (!(*flowop->fo_blocking))
 		(void) ipc_mutex_unlock(&flowop->fo_lock);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Releases the semids for the System V semaphore allocated
  * to this flowop. If not using System V semaphores, then
- * it is effectively just a no-op. Always returns 0.
+ * it is effectively just a no-op.
  */
 static void
 flowoplib_semblock_destruct(flowop_t *flowop)
@@ -1302,9 +1330,9 @@
 
 /*
  * Attempts to pass a System V or posix semaphore as appropriate,
- * and blocks if necessary. Returns -1 if a set of System V
+ * and blocks if necessary. Returns FILEBENCH_ERROR if a set of System V
  * semphores is not available or cannot be acquired, or if the initial
- * post to the semaphore set fails. Returns 0 on success.
+ * post to the semaphore set fails. Returns FILEBENCH_OK on success.
  */
 static int
 flowoplib_semblock(threadflow_t *threadflow, flowop_t *flowop)
@@ -1320,7 +1348,7 @@
 		filebench_log(LOG_ERROR, "lookup semop %x failed: %s",
 		    filebench_shm->semkey,
 		    strerror(errno));
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	filebench_log(LOG_DEBUG_IMPL,
@@ -1368,7 +1396,7 @@
 	for (i = 0; i < value; i++) {
 		if (sem_wait(&flowop->fo_sem) == -1) {
 			filebench_log(LOG_ERROR, "semop wait failed");
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 	}
 
@@ -1376,13 +1404,11 @@
 	    flowop->fo_name, flowop->fo_instance);
 #endif /* HAVE_SYSV_SEM */
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
- * Calls ipc_seminit(), and does so whether System V semaphores
- * are available or not. Hence it will cause ipc_seminit to log errors
- * if they are not. Always returns 0.
+ * Calls ipc_seminit(). Always returns FILEBENCH_OK.
  */
 /* ARGSUSED */
 static int
@@ -1391,7 +1417,7 @@
 #ifdef HAVE_SYSV_SEM
 	ipc_seminit();
 #endif /* HAVE_SYSV_SEM */
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -1467,7 +1493,7 @@
 			    "lookup semop %x failed: %s",
 			    filebench_shm->semkey,
 			    strerror(errno));
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 
 		sbuf[0].sem_num = target->fo_semid_lw;
@@ -1493,7 +1519,7 @@
 #endif /* HAVE_SEMTIMEDOP */
 			filebench_log(LOG_ERROR, "semop post failed: %s",
 			    strerror(errno));
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 
 		filebench_log(LOG_DEBUG_IMPL,
@@ -1509,7 +1535,7 @@
 		for (i = 0; i < value; i++) {
 			if (sem_post(&target->fo_sem) == -1) {
 				filebench_log(LOG_ERROR, "semop post failed");
-				return (-1);
+				return (FILEBENCH_ERROR);
 			}
 		}
 
@@ -1521,7 +1547,7 @@
 	}
 	flowop_endop(threadflow, flowop, 0);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 
@@ -1547,9 +1573,10 @@
 
 /*
  * Emulates (and actually does) file open. Obtains a file descriptor
- * index, then calls flowoplib_openfile_common() to open. Returns -1
- * if not file descriptor is found or flowoplib_openfile_common
- * encounters an error, otherwise 0.
+ * index, then calls flowoplib_openfile_common() to open. Returns
+ * FILEBENCH_ERROR if no file descriptor is found, and returns the
+ * status from flowoplib_openfile_common otherwise (FILEBENCH_ERROR,
+ * FILEBENCH_NORSC, FILEBENCH_OK).
  */
 static int
 flowoplib_openfile(threadflow_t *threadflow, flowop_t *flowop)
@@ -1557,7 +1584,7 @@
 	int fd = flowoplib_fdnum(threadflow, flowop);
 
 	if (fd == -1)
-		return (-1);
+		return (FILEBENCH_ERROR);
 
 	return (flowoplib_openfile_common(threadflow, flowop, fd));
 }
@@ -1570,7 +1597,9 @@
  * specified in the filesetentry is opened, and the returned
  * operating system file descriptor and a pointer to the
  * filesetentry are stored in tf_fd[fd] and tf_fse[fd],
- * respectively. Returns -1 on error, 0 on success.
+ * respectively. Returns FILEBENCH_ERROR on error,
+ * FILEBENCH_NORSC if no suitable filesetentry can be found,
+ * and FILEBENCH_OK on success.
  */
 static int
 flowoplib_openfile_common(threadflow_t *threadflow, flowop_t *flowop, int fd)
@@ -1589,12 +1618,12 @@
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to open without closing on fd %d",
 		    flowop->fo_name, fd);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	if (flowop->fo_fileset == NULL) {
 		filebench_log(LOG_ERROR, "flowop NULL file");
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 #ifdef HAVE_RAW_SUPPORT
@@ -1624,7 +1653,7 @@
 			filebench_log(LOG_ERROR,
 			    "Failed to open raw device %s: %s",
 			    name, strerror(errno));
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 
 		/* if running on Solaris, use un-buffered io */
@@ -1634,17 +1663,17 @@
 
 		threadflow->tf_fse[fd] = NULL;
 
-		return (0);
+		return (FILEBENCH_OK);
 	}
 #endif /* HAVE_RAW_SUPPORT */
 
 	if ((file = fileset_pick(flowop->fo_fileset,
 	    FILESET_PICKEXISTS, tid)) == NULL) {
-		filebench_log(LOG_ERROR,
+		filebench_log(LOG_DEBUG_SCRIPT,
 		    "flowop %s failed to pick file from %s on fd %d",
 		    flowop->fo_name,
 		    flowop->fo_fileset->fs_name, fd);
-		return (-1);
+		return (FILEBENCH_NORSC);
 	}
 
 	threadflow->tf_fse[fd] = file;
@@ -1657,14 +1686,14 @@
 	if (threadflow->tf_fd[fd] < 0) {
 		filebench_log(LOG_ERROR, "failed to open file %s",
 		    flowop->fo_name);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	filebench_log(LOG_DEBUG_SCRIPT,
 	    "flowop %s: opened %s fd[%d] = %d",
 	    flowop->fo_name, file->fse_path, fd, threadflow->tf_fd[fd]);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -1674,10 +1703,10 @@
  * to select a specific filesetentry whose file does not currently
  * exist for the file create operation. Then calls
  * fileset_openfile() with the O_CREATE flag set to create the
- * file. Returns -1 if the array index specified by fdnumber is
+ * file. Returns FILEBENCH_ERROR if the array index specified by fdnumber is
  * already in use, the flowop has no associated fileset, or
  * the create call fails. Returns 1 if a filesetentry with a
- * nonexistent file cannot be found. Returns 0 on success.
+ * nonexistent file cannot be found. Returns FILEBENCH_OK on success.
  */
 static int
 flowoplib_createfile(threadflow_t *threadflow, flowop_t *flowop)
@@ -1689,12 +1718,12 @@
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to create without closing on fd %d",
 		    flowop->fo_name, fd);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	if (flowop->fo_fileset == NULL) {
 		filebench_log(LOG_ERROR, "flowop NULL file");
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 #ifdef HAVE_RAW_SUPPORT
@@ -1703,15 +1732,16 @@
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to a createfile on RAW device",
 		    flowop->fo_name);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 #endif /* HAVE_RAW_SUPPORT */
 
 	if ((file = fileset_pick(flowop->fo_fileset,
 	    FILESET_PICKNOEXIST, 0)) == NULL) {
-		filebench_log(LOG_DEBUG_SCRIPT, "flowop %s failed to pick file",
-		    flowop->fo_name);
-		return (1);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "flowop %s failed to pick file from fileset %s",
+		    flowop->fo_name, flowop->fo_fileset->fs_name);
+		return (FILEBENCH_NORSC);
 	}
 
 	threadflow->tf_fse[fd] = file;
@@ -1724,22 +1754,22 @@
 	if (threadflow->tf_fd[fd] < 0) {
 		filebench_log(LOG_ERROR, "failed to create file %s",
 		    flowop->fo_name);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	filebench_log(LOG_DEBUG_SCRIPT,
 	    "flowop %s: created %s fd[%d] = %d",
 	    flowop->fo_name, file->fse_path, fd, threadflow->tf_fd[fd]);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Emulates delete of a file. Picks an arbitrary filesetentry
  * whose file exists and uses unlink() to delete it. Clears
- * the FSE_EXISTS flag for the filesetentry. Returns -1 if the
- * flowop has no associated fileset. Returns 1 if an appropriate
- * filesetentry cannot be found, and 0 on success.
+ * the FSE_EXISTS flag for the filesetentry. Returns FILEBENCH_ERROR if the
+ * flowop has no associated fileset. Returns FILEBENCH_NORSC if an appropriate
+ * filesetentry cannot be found, and FILEBENCH_OK on success.
  */
 static int
 flowoplib_deletefile(threadflow_t *threadflow, flowop_t *flowop)
@@ -1751,7 +1781,7 @@
 
 	if (flowop->fo_fileset == NULL) {
 		filebench_log(LOG_ERROR, "flowop NULL file");
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	fileset = flowop->fo_fileset;
@@ -1762,7 +1792,7 @@
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted a deletefile on RAW device",
 		    flowop->fo_name);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 #endif /* HAVE_RAW_SUPPORT */
 
@@ -1770,7 +1800,7 @@
 	    FILESET_PICKEXISTS, 0)) == NULL) {
 		filebench_log(LOG_DEBUG_SCRIPT, "flowop %s failed to pick file",
 		    flowop->fo_name);
-		return (1);
+		return (FILEBENCH_NORSC);
 	}
 
 	*path = 0;
@@ -1789,15 +1819,15 @@
 
 	filebench_log(LOG_DEBUG_SCRIPT, "deleted file %s", file->fse_path);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Emulates fsync of a file. Obtains the file descriptor index
  * from the flowop, obtains the actual file descriptor from
  * the threadflow's table, checks to be sure it is still an
- * open file, then does an fsync operation on it. Returns -1
- * if the file no longer is open, 0 otherwise.
+ * open file, then does an fsync operation on it. Returns FILEBENCH_ERROR
+ * if the file no longer is open, FILEBENCH_OK otherwise.
  */
 static int
 flowoplib_fsync(threadflow_t *threadflow, flowop_t *flowop)
@@ -1809,7 +1839,7 @@
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to fsync a closed fd %d",
 		    flowop->fo_name, fd);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	file = threadflow->tf_fse[fd];
@@ -1819,7 +1849,7 @@
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to a fsync a RAW device",
 		    flowop->fo_name);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	/* Measure time to fsync */
@@ -1829,14 +1859,14 @@
 
 	filebench_log(LOG_DEBUG_SCRIPT, "fsync file %s", file->fse_path);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Emulate fsync of an entire fileset. Search through the
  * threadflow's file descriptor array, doing fsync() on each
  * open file that belongs to the flowop's fileset. Always
- * returns 0.
+ * returns FILEBENCH_OK.
  */
 static int
 flowoplib_fsyncset(threadflow_t *threadflow, flowop_t *flowop)
@@ -1862,7 +1892,7 @@
 		    file->fse_path);
 	}
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -1871,8 +1901,8 @@
  * threadflow's table, checks to be sure it is still an open
  * file, then does a close operation on it. Then sets the
  * threadflow file descriptor table entry to 0, and the file set
- * entry pointer to NULL. Returns -1 if the file was not open,
- * 0 otherwise.
+ * entry pointer to NULL. Returns FILEBENCH_ERROR if the file was not open,
+ * FILEBENCH_OK otherwise.
  */
 static int
 flowoplib_closefile(threadflow_t *threadflow, flowop_t *flowop)
@@ -1884,7 +1914,7 @@
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to close an already closed fd %d",
 		    flowop->fo_name, fd);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	/* Measure time to close */
@@ -1899,15 +1929,15 @@
 
 	filebench_log(LOG_DEBUG_SCRIPT, "closed file %s", file->fse_path);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
  * Emulate stat of a file. Picks an arbitrary filesetentry with
  * an existing file from the flowop's fileset, then performs a
- * stat() operation on it. Returns -1 if the flowop has no
- * associated fileset. Returns 1 if an appropriate filesetentry
- * cannot be found, and 0 on success.
+ * stat() operation on it. Returns FILEBENCH_ERROR if the flowop has no
+ * associated fileset. Returns FILEBENCH_NORSC if an appropriate filesetentry
+ * cannot be found, and FILEBENCH_OK on success.
  */
 static int
 flowoplib_statfile(threadflow_t *threadflow, flowop_t *flowop)
@@ -1919,7 +1949,7 @@
 
 	if (flowop->fo_fileset == NULL) {
 		filebench_log(LOG_ERROR, "flowop NULL file");
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	fileset = flowop->fo_fileset;
@@ -1928,7 +1958,7 @@
 	    FILESET_PICKEXISTS, 0)) == NULL) {
 		filebench_log(LOG_DEBUG_SCRIPT, "flowop %s failed to pick file",
 		    flowop->fo_name);
-		return (1);
+		return (FILEBENCH_NORSC);
 	}
 
 	*path = 0;
@@ -1944,7 +1974,7 @@
 
 	(void) ipc_mutex_unlock(&file->fse_lock);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 
@@ -1963,7 +1993,8 @@
  * file descriptor and filesetentry stored at the locations indexed
  * by the flowop's fdnumber. It then seeks to the beginning of the
  * associated file, and reads fs_iosize bytes at a time until the end
- * of the file. Returns -1 on error, 0 on success.
+ * of the file. Returns FILEBENCH_ERROR on error, FILEBENCH_NORSC if
+ * out of files, and FILEBENCH_OK on success.
  */
 static int
 flowoplib_readwholefile(threadflow_t *threadflow, flowop_t *flowop)
@@ -1977,15 +2008,16 @@
 	vinteger_t iosize = *flowop->fo_iosize;
 
 	/* get the file to use */
-	if (flowoplib_filesetup(threadflow, flowop, &wss, &filedesc) != 0)
-		return (-1);
+	if ((ret = flowoplib_filesetup(threadflow, flowop, &wss,
+	    &filedesc)) != FILEBENCH_OK)
+		return (ret);
 
 	/* an I/O size of zero means read entire working set with one I/O */
 	if (iosize == 0)
 		iosize = wss;
 
 	if (flowoplib_iobufsetup(threadflow, flowop, &iobuf, iosize) != 0)
-		return (-1);
+		return (FILEBENCH_ERROR);
 
 	/* Measure time to read bytes */
 	flowop_beginop(threadflow, flowop);
@@ -1999,10 +2031,10 @@
 		filebench_log(LOG_ERROR,
 		    "Failed to read fd %d: %s",
 		    fd, strerror(errno));
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -2016,8 +2048,9 @@
  * filesetentry's  fse_size will be used. A random memory
  * buffer offset is calculated, and, if fo_random is TRUE,
  * a random file offset is used for the write. Otherwise the
- * write is to the next sequential location. Returns 1 on
- * errors, 0 on success.
+ * write is to the next sequential location. Returns
+ * FILEBENCH_ERROR on errors, FILEBENCH_NORSC if iosetup can't
+ * obtain a file, or FILEBENCH_OK on success.
  */
 static int
 flowoplib_write(threadflow_t *threadflow, flowop_t *flowop)
@@ -2025,10 +2058,11 @@
 	caddr_t iobuf;
 	vinteger_t wss;
 	int filedesc;
+	int ret;
 
-	if (flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, *flowop->fo_iosize) != 0)
-		return (-1);
+	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
+	    &filedesc, *flowop->fo_iosize)) != FILEBENCH_OK)
+		return (ret);
 
 	if (*flowop->fo_random) {
 		uint64_t fileoffset;
@@ -2038,7 +2072,7 @@
 			filebench_log(LOG_ERROR,
 			    "file size smaller than IO size for thread %s",
 			    flowop->fo_name);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 		flowop_beginop(threadflow, flowop);
 		if (pwrite64(filedesc, iobuf,
@@ -2047,7 +2081,7 @@
 			    "offset %lld io buffer %zd: %s",
 			    fileoffset, iobuf, strerror(errno));
 			flowop_endop(threadflow, flowop, 0);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 		flowop_endop(threadflow, flowop, *flowop->fo_iosize);
 	} else {
@@ -2058,12 +2092,12 @@
 			    "write failed, io buffer %zd: %s",
 			    iobuf, strerror(errno));
 			flowop_endop(threadflow, flowop, 0);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 		flowop_endop(threadflow, flowop, *flowop->fo_iosize);
 	}
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -2071,8 +2105,8 @@
  * is taken from a filesetentry identified by fo_srcfdnumber or
  * from the working set size, while the file descriptor used is
  * identified by fo_fdnumber. Does multiple writes of fo_iosize
- * length length until full file has been written. Returns -1 on
- * error, 0 on success.
+ * length length until full file has been written. Returns FILEBENCH_ERROR on
+ * error, FILEBENCH_NORSC if out of files, FILEBENCH_OK on success.
  */
 static int
 flowoplib_writewholefile(threadflow_t *threadflow, flowop_t *flowop)
@@ -2089,21 +2123,22 @@
 	vinteger_t iosize = *flowop->fo_iosize;
 
 	/* get the file to use */
-	if (flowoplib_filesetup(threadflow, flowop, &wss, &filedesc) != 0)
-		return (-1);
+	if ((ret = flowoplib_filesetup(threadflow, flowop, &wss,
+	    &filedesc)) != FILEBENCH_OK)
+		return (ret);
 
 	/* an I/O size of zero means read entire working set with one I/O */
 	if (iosize == 0)
 		iosize = wss;
 
 	if (flowoplib_iobufsetup(threadflow, flowop, &iobuf, iosize) != 0)
-		return (-1);
+		return (FILEBENCH_ERROR);
 
 	file = threadflow->tf_fse[srcfd];
 	if ((srcfd != 0) && (file == NULL)) {
 		filebench_log(LOG_ERROR, "flowop %s: NULL src file",
 		    flowop->fo_name);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	if (file)
@@ -2120,14 +2155,14 @@
 			    "Failed to write %d bytes on fd %d: %s",
 			    wsize, filedesc, strerror(errno));
 			flowop_endop(threadflow, flowop, 0);
-			return (-1);
+			return (FILEBENCH_ERROR);
 		}
 		wsize = (int)MIN(wss - seek, iosize);
 		bytes += ret;
 	}
 	flowop_endop(threadflow, flowop, bytes);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 
@@ -2144,7 +2179,8 @@
  * bytes. Writes are actually done from fo_buf, rather than
  * tf_mem as is done with flowoplib_write(), and no check
  * is made to see if fo_iosize exceeds the size of fo_buf.
- * Returns -1 on error, 0 on success.
+ * Returns FILEBENCH_ERROR on error, FILEBENCH_NORSC if out of
+ * files in the fileset, FILEBENCH_OK on success.
  */
 static int
 flowoplib_appendfile(threadflow_t *threadflow, flowop_t *flowop)
@@ -2155,9 +2191,9 @@
 	vinteger_t iosize = *flowop->fo_iosize;
 	int ret;
 
-	if (flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, iosize) != 0)
-		return (-1);
+	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
+	    &filedesc, iosize)) != FILEBENCH_OK)
+		return (ret);
 
 	/* XXX wss is not being used */
 
@@ -2170,11 +2206,11 @@
 		    "Failed to write %d bytes on fd %d: %s",
 		    iosize, filedesc, strerror(errno));
 		flowop_endop(threadflow, flowop, 0);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 	flowop_endop(threadflow, flowop, iosize);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -2190,7 +2226,8 @@
  * FILE_ALLOC_BLOCK in size are done until the full transfer
  * size has been written. Writes are actually done from fo_buf,
  * rather than tf_mem as is done with flowoplib_write().
- * Returns -1 on error, 0 on success.
+ * Returns FILEBENCH_ERROR on error, FILEBENCH_NORSC if out of
+ * files in the fileset, FILEBENCH_OK on success.
  */
 static int
 flowoplib_appendfilerand(threadflow_t *threadflow, flowop_t *flowop)
@@ -2199,21 +2236,21 @@
 	uint64_t appendsize;
 	int filedesc;
 	vinteger_t wss;
-	int ret = 0;
+	int ret;
 
 	if (filebench_randomno64(&appendsize, *flowop->fo_iosize, 1LL) != 0)
-		return (-1);
+		return (FILEBENCH_ERROR);
 
 	/* skip if attempting zero length append */
 	if (appendsize == 0) {
 		flowop_beginop(threadflow, flowop);
 		flowop_endop(threadflow, flowop, 0LL);
-		return (0);
+		return (FILEBENCH_OK);
 	}
 
-	if (flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, appendsize) != 0)
-		return (-1);
+	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
+	    &filedesc, appendsize)) != FILEBENCH_OK)
+		return (ret);
 
 	/* XXX wss is not being used */
 
@@ -2227,12 +2264,12 @@
 		    "Failed to write %d bytes on fd %d: %s",
 		    appendsize, filedesc, strerror(errno));
 		flowop_endop(threadflow, flowop, 0);
-		return (-1);
+		return (FILEBENCH_ERROR);
 	}
 
 	flowop_endop(threadflow, flowop, appendsize);
 
-	return (0);
+	return (FILEBENCH_OK);
 }
 
 
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/ipc.c
--- a/usr/src/cmd/filebench/common/ipc.c	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/ipc.c	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -322,7 +322,8 @@
 
 	(void) memset(filebench_shm, 0, c2 - c1);
 	filebench_shm->epoch = gethrtime();
-	filebench_shm->debug_level = 2;
+	filebench_shm->debug_level = LOG_VERBOSE;
+	filebench_shm->shm_rmode = FILEBENCH_MODE_TIMEOUT;
 	filebench_shm->string_ptr = &filebench_shm->strings[0];
 	filebench_shm->shm_ptr = (char *)filebench_shm->shm_addr;
 	filebench_shm->path_ptr = &filebench_shm->filesetpaths[0];
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/ipc.h
--- a/usr/src/cmd/filebench/common/ipc.h	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/ipc.h	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -69,6 +69,14 @@
 
 #define	FILEBENCH_NSEMS 128
 
+#define	FILEBENCH_ABORT_ERROR  	1
+#define	FILEBENCH_ABORT_DONE   	2
+#define	FILEBENCH_ABORT_RSRC	3
+
+#define	FILEBENCH_MODE_TIMEOUT	0x0
+#define	FILEBENCH_MODE_Q1STDONE	0x1
+#define	FILEBENCH_MODE_QALLDONE	0x2
+
 typedef struct filebench_shm {
 	pthread_mutex_t fileset_lock;
 	pthread_mutex_t procflow_lock;
@@ -108,8 +116,10 @@
 	size_t		shm_allocated;
 	caddr_t		shm_addr;
 	char		*shm_ptr;
-	int		allrunning;
+	int		shm_running;
 	int		f_abort;
+	int		shm_rmode;
+	int		shm_1st_err;
 
 	int		marker;
 
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/misc.c
--- a/usr/src/cmd/filebench/common/misc.c	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/misc.c	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -331,6 +331,19 @@
 		level = LOG_ERROR;
 	}
 
+	/* Quit if this is a LOG_ERROR messages and they are disabled */
+	if ((filebench_shm->shm_1st_err) && (level == LOG_ERROR))
+		return;
+
+	if (level == LOG_ERROR1) {
+		if (filebench_shm->shm_1st_err)
+			return;
+
+		/* A LOG_ERROR1 temporarily disables LOG_ERROR messages */
+		filebench_shm->shm_1st_err = 1;
+		level = LOG_ERROR;
+	}
+
 	/* Only log greater than debug setting */
 	if ((level != LOG_DUMP) && (level != LOG_LOG) &&
 	    (level > filebench_shm->debug_level))
@@ -376,7 +389,7 @@
 
 	} else if (filebench_shm->debug_level > LOG_INFO) {
 		(void) fprintf(stdout, "%5d: %4.3f: %s",
-		    (int)pid, (now - filebench_shm->epoch) / FSECS,
+		    (int)my_pid, (now - filebench_shm->epoch) / FSECS,
 		    line);
 	} else {
 		(void) fprintf(stdout, "%4.3f: %s",
@@ -385,7 +398,8 @@
 	}
 
 	if (level == LOG_ERROR) {
-		(void) fprintf(stdout, " on line %d", lex_lineno);
+		if (my_procflow == NULL)
+			(void) fprintf(stdout, " on line %d", lex_lineno);
 	}
 
 	if ((level != LOG_LOG) && (level != LOG_DUMP)) {
@@ -405,7 +419,7 @@
 filebench_shutdown(int error) {
 	filebench_log(LOG_DEBUG_IMPL, "Shutdown");
 	(void) unlink("/tmp/filebench_shm");
-	if (filebench_shm->allrunning)
+	if (filebench_shm->shm_running)
 		procflow_shutdown();
 	filebench_shm->f_abort = 1;
 	ipc_ismdelete();
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/misc.h
--- a/usr/src/cmd/filebench/common/misc.h	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/misc.h	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -51,15 +51,16 @@
 #endif
 #define	FSECS (double)1000000000.0
 
-#define	LOG_INFO 1
-#define	LOG_VERBOSE 2
-#define	LOG_DEBUG_SCRIPT 3
-#define	LOG_DEBUG_IMPL 5
+#define	LOG_INFO 2
+#define	LOG_VERBOSE 3
+#define	LOG_DEBUG_SCRIPT 4
+#define	LOG_DEBUG_IMPL 6
 #define	LOG_DEBUG_NEVER 10
 #define	LOG_LOG 1000
 #define	LOG_DUMP 1001
 #define	LOG_FATAL 999
 #define	LOG_ERROR 0
+#define	LOG_ERROR1 1
 
 var_t *date_var(var_t *var);
 var_t *script_var(var_t *var);
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/parser_gram.y
--- a/usr/src/cmd/filebench/common/parser_gram.y	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/parser_gram.y	Tue Feb 26 11:39:00 2008 -0800
@@ -20,7 +20,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -157,7 +157,7 @@
 %token FSV_STRING FSV_VAL_INT FSV_VAL_BOOLEAN FSV_VARIABLE FSV_WHITESTRING
 %token FST_INT FST_BOOLEAN
 %token FSE_FILE FSE_PROC FSE_THREAD FSE_CLEAR FSE_ALL FSE_SNAP FSE_DUMP
-%token FSE_DIRECTORY FSE_COMMAND FSE_FILESET FSE_XMLDUMP
+%token FSE_DIRECTORY FSE_COMMAND FSE_FILESET FSE_XMLDUMP FSE_MODE
 %token FSK_SEPLST FSK_OPENLST FSK_CLOSELST FSK_ASSIGN FSK_IN FSK_QUOTE
 %token FSK_DIRSEPLST
 %token FSA_SIZE FSA_PREALLOC FSA_PARALLOC FSA_PATH FSA_REUSE
@@ -166,7 +166,7 @@
 %token FSA_DSYNC FSA_TARGET FSA_ITERS FSA_NICE FSA_VALUE FSA_BLOCKING
 %token FSA_HIGHWATER FSA_DIRECTIO FSA_DIRWIDTH FSA_FD FSA_SRCFD FSA_ROTATEFD
 %token FSA_NAMELENGTH FSA_FILESIZE FSA_ENTRIES FSA_FILESIZEGAMMA
-%token FSA_DIRGAMMA FSA_USEISM
+%token FSA_DIRGAMMA FSA_USEISM FSA_ALLDONE FSA_FIRSTDONE FSA_TIMEOUT
 
 %type <ival> FSV_VAL_INT
 %type <bval> FSV_VAL_BOOLEAN
@@ -684,6 +684,24 @@
 		parser_vars($$);
 	}
 	$$->cmd = NULL;
+} | FSC_SET FSE_MODE FSC_QUIT FSA_TIMEOUT
+{
+	filebench_shm->shm_rmode = FILEBENCH_MODE_TIMEOUT;
+	if (($$ = alloc_cmd()) == NULL)
+		YYERROR;
+	$$->cmd = NULL;
+} | FSC_SET FSE_MODE FSC_QUIT FSA_ALLDONE
+{
+	filebench_shm->shm_rmode = FILEBENCH_MODE_QALLDONE;
+	if (($$ = alloc_cmd()) == NULL)
+		YYERROR;
+	$$->cmd = NULL;
+} | FSC_SET FSE_MODE FSC_QUIT FSA_FIRSTDONE
+{
+	filebench_shm->shm_rmode = FILEBENCH_MODE_Q1STDONE;
+	if (($$ = alloc_cmd()) == NULL)
+		YYERROR;
+	$$->cmd = NULL;
 };
 
 stats_command: FSC_STATS FSE_SNAP
@@ -1180,26 +1198,26 @@
 	printf("FileBench Version %s\n", FILEBENCH_VERSION);
 	filebench_init();
 
+	/* get process pid for use with message logging */
+	my_pid = getpid();
+
 #ifdef USE_PROCESS_MODEL
 	if (*procname) {
-		pid = getpid();
-
+		/* A child FileBench instance */
 		if (ipc_attach(shmaddr) < 0) {
 			filebench_log(LOG_ERROR, "Cannot attach shm for %s",
 			    procname);
 			exit(1);
 		}
 
-		if (procflow_exec(procname, instance) < 0) {
-			filebench_log(LOG_ERROR, "Cannot startup process %s",
-			    procname);
+		if (procflow_exec(procname, instance) < 0)
 			exit(1);
-		}
-		exit(1);
+
+		exit(0);
 	}
 #endif
 
-	pid = getpid();
+	/* master (or only) process */
 	ipc_init();
 
 	if (fscriptname)
@@ -1917,6 +1935,7 @@
 static void
 parser_proc_create(cmd_t *cmd)
 {
+	filebench_shm->shm_1st_err = 0;
 	if (procflow_init() != 0) {
 		filebench_log(LOG_ERROR, "Failed to create processes\n");
 		filebench_shutdown(1);
@@ -1993,27 +2012,36 @@
 }
 
 /*
- * Sleeps for cmd->cmd_qty seconds, one second at a time.
+ * This is Used for timing runs.Pauses the master thread in one second
+ * intervals until the supplied ptime runs out or the f_abort flag
+ * is raised. If given a time of zero or less, or the mode is stop on
+ * lack of resources, it will pause until f_abort is raised.
  */
 static void
-parser_sleep(cmd_t *cmd)
+parser_pause(int ptime)
 {
-	int sleeptime;
-
-	/* check for startup errors */
-	if (filebench_shm->f_abort)
-		return;
-
-	sleeptime = cmd->cmd_qty;
-	filebench_log(LOG_INFO, "Running...");
-	while (sleeptime) {
-		(void) sleep(1);
-		sleeptime--;
-		if (filebench_shm->f_abort)
-			break;
+	int timeslept = 0;
+
+	if ((filebench_shm->shm_rmode == FILEBENCH_MODE_TIMEOUT) &&
+	    (ptime > 0)) {
+		while (timeslept < ptime) {
+			(void) sleep(1);
+			timeslept++;
+			if (filebench_shm->f_abort)
+				break;
+		}
+	} else {
+		/* initial runtime of 0 means run till abort */
+		/* CONSTCOND */
+		while (1) {
+			(void) sleep(1);
+			timeslept++;
+			if (filebench_shm->f_abort)
+				break;
+		}
 	}
-	filebench_log(LOG_INFO, "Run took %lld seconds...",
-	    cmd->cmd_qty - sleeptime);
+
+	filebench_log(LOG_INFO, "Run took %lld seconds...", timeslept);
 }
 
 /*
@@ -2029,6 +2057,7 @@
 	int runtime;
 
 	runtime = cmd->cmd_qty;
+
 	parser_fileset_create(cmd);
 	parser_proc_create(cmd);
 
@@ -2038,14 +2067,9 @@
 
 	filebench_log(LOG_INFO, "Running...");
 	stats_clear();
-	while (runtime) {
-		(void) sleep(1);
-		runtime--;
-		if (filebench_shm->f_abort)
-			break;
-	}
-	filebench_log(LOG_INFO, "Run took %lld seconds...",
-	    cmd->cmd_qty - runtime);
+
+	parser_pause(runtime);
+
 	parser_statssnap(cmd);
 	parser_proc_shutdown(cmd);
 }
@@ -2074,15 +2098,11 @@
 
 	filebench_log(LOG_INFO, "Running...");
 	stats_clear();
-	while (runtime) {
-		(void) sleep(1);
-		runtime--;
-		if (filebench_shm->f_abort)
-			break;
-	}
-	filebench_log(LOG_INFO, "Run took %lld seconds...",
-	    *integer - runtime);
+
+	parser_pause(runtime);
+
 	parser_statssnap(cmd);
+	parser_proc_shutdown(cmd);
 }
 
 char *usagestr = NULL;
@@ -2161,6 +2181,24 @@
 }
 
 /*
+ * Sleeps for cmd->cmd_qty seconds, one second at a time.
+ */
+static void
+parser_sleep(cmd_t *cmd)
+{
+	int sleeptime;
+
+	/* check for startup errors */
+	if (filebench_shm->f_abort)
+		return;
+
+	sleeptime = cmd->cmd_qty;
+	filebench_log(LOG_INFO, "Running...");
+
+	parser_pause(sleeptime);
+}
+
+/*
  * Same as parser_sleep, except the sleep time is obtained from a variable
  * whose name is passed to it as an argument on the command line.
  */
@@ -2183,14 +2221,8 @@
 		return;
 
 	filebench_log(LOG_INFO, "Running...");
-	while (sleeptime) {
-		(void) sleep(1);
-		sleeptime--;
-		if (filebench_shm->f_abort)
-			break;
-	}
-	filebench_log(LOG_INFO, "Run took %lld seconds...",
-	    *integer - sleeptime);
+
+	parser_pause(sleeptime);
 }
 
 /*
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/parser_lex.l
--- a/usr/src/cmd/filebench/common/parser_lex.l	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/parser_lex.l	Tue Feb 26 11:39:00 2008 -0800
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -97,6 +97,7 @@
 dump                    { return FSE_DUMP; }
 xmldump                 { return FSE_XMLDUMP; }
 all                     { return FSE_ALL; }
+mode                    { return FSE_MODE; }
 
 cached                  { return FSA_CACHED; }
 dirwidth                { return FSA_DIRWIDTH; }
@@ -132,6 +133,9 @@
 workingset              { return FSA_WSS; }
 blocking                { return FSA_BLOCKING; }
 highwater               { return FSA_HIGHWATER; }
+alldone                 { return FSA_ALLDONE; }
+firstdone               { return FSA_FIRSTDONE; }
+timeout                 { return FSA_TIMEOUT; }
 
 <INITIAL>\"			{ 
                                 BEGIN WHITESTRINGSTATE;
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/procflow.c
--- a/usr/src/cmd/filebench/common/procflow.c	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/procflow.c	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -35,8 +35,9 @@
 #include "flowop.h"
 #include "ipc.h"
 
-pid_t pid;
-static int procflow_delete_wait_cnt = 0;
+/* pid and procflow pointer for this process */
+pid_t my_pid;
+procflow_t *my_procflow = NULL;
 
 static procflow_t *procflow_define_common(procflow_t **list, char *name,
     procflow_t *inherit, int instance);
@@ -135,6 +136,7 @@
 	}
 #endif /* HAVE_FORK1 */
 
+	/* if child, start up new copy of filebench */
 	if (pid == 0) {
 #ifdef USE_SYSTEM
 		char syscmd[1024];
@@ -170,6 +172,7 @@
 #endif
 		exit(1);
 	} else {
+		/* if parent, save pid and return */
 		procflow->pf_pid = pid;
 	}
 #else
@@ -277,6 +280,7 @@
 #ifdef HAVE_SETRLIMIT
 	struct rlimit rlp;
 #endif
+	int ret;
 
 	filebench_log(LOG_DEBUG_IMPL,
 	    "procflow_execproc %s-%d",
@@ -288,10 +292,15 @@
 		    name, instance);
 		return (-1);
 	}
-	procflow->pf_pid = pid;
+
+	/* set the slave process' procflow pointer */
+	my_procflow = procflow;
+
+	/* set its pid from value stored by main() */
+	procflow->pf_pid = my_pid;
 
 	filebench_log(LOG_DEBUG_IMPL,
-	    "Started up %s pid %d", procflow->pf_name, pid);
+	    "Started up %s pid %d", procflow->pf_name, my_pid);
 
 	filebench_log(LOG_DEBUG_IMPL,
 	    "nice = %llx", procflow->pf_nice);
@@ -308,18 +317,23 @@
 	filebench_log(LOG_DEBUG_SCRIPT, "%d file descriptors", rlp.rlim_cur);
 #endif
 
-	if (threadflow_init(procflow) < 0) {
-		filebench_log(LOG_ERROR,
-		    "Failed to start threads for %s pid %d",
-		    procflow->pf_name, pid);
-		procflow->pf_running = 0;
-		exit(1);
+	if ((ret = threadflow_init(procflow)) != FILEBENCH_OK) {
+		if (ret < 0) {
+			filebench_log(LOG_ERROR,
+			    "Failed to start threads for %s pid %d",
+			    procflow->pf_name, my_pid);
+		}
+	} else {
+		filebench_log(LOG_DEBUG_IMPL,
+		    "procflow_createproc exiting...");
 	}
-	filebench_log(LOG_DEBUG_IMPL, "procflow_createproc exiting...");
+
 	procflow->pf_running = 0;
-	exit(0);
+	(void) ipc_mutex_lock(&filebench_shm->procflow_lock);
+	filebench_shm->shm_running --;
+	(void) ipc_mutex_unlock(&filebench_shm->procflow_lock);
 
-	return (0);
+	return (ret);
 }
 
 
@@ -373,8 +387,11 @@
 			filebench_shutdown(1);
 		}
 
-		if (filebench_shm->allrunning == 0)
+		/* nothing running, exit */
+		if (filebench_shm->shm_running == 0) {
+			filebench_shm->f_abort = FILEBENCH_ABORT_RSRC;
 			pthread_exit(0);
+		}
 	}
 	/* NOTREACHED */
 	return (NULL);
@@ -454,11 +471,11 @@
  * the procflow is not deleted. Otherwise it returns 0.
  */
 static int
-procflow_delete(procflow_t *procflow)
+procflow_delete(procflow_t *procflow, int wait_cnt)
 {
 	procflow_t *entry;
 
-	threadflow_delete_all(&procflow->pf_threads);
+	threadflow_delete_all(&procflow->pf_threads, wait_cnt);
 
 	filebench_log(LOG_DEBUG_SCRIPT,
 	    "Deleted proc: (%s-%d) pid %d",
@@ -473,11 +490,11 @@
 		    procflow->pf_instance,
 		    procflow->pf_pid);
 
-		if (procflow_delete_wait_cnt < 10) {
+		if (wait_cnt) {
 			(void) ipc_mutex_unlock(&filebench_shm->procflow_lock);
 			(void) sleep(1);
 			(void) ipc_mutex_lock(&filebench_shm->procflow_lock);
-			procflow_delete_wait_cnt++;
+			wait_cnt--;
 			continue;
 		}
 #ifdef USE_PROCESS_MODEL
@@ -534,6 +551,7 @@
 procflow_allstarted()
 {
 	procflow_t *procflow = filebench_shm->proclist;
+	int running_procs = 0;
 	int ret = 0;
 
 	(void) ipc_mutex_lock(&filebench_shm->procflow_lock);
@@ -568,16 +586,18 @@
 		}
 
 		if (waits == 0)
-			filebench_log(LOG_INFO, "Failed to start process %s-%d",
+			filebench_log(LOG_INFO,
+			    "Failed to start process %s-%d",
 			    procflow->pf_name,
 			    procflow->pf_instance);
 
+		running_procs++;
 		threadflow_allstarted(procflow->pf_pid, procflow->pf_threads);
 
 		procflow = procflow->pf_next;
 	}
+	filebench_shm->shm_running = running_procs;
 
-	filebench_shm->allrunning = 1;
 	(void) ipc_mutex_unlock(&filebench_shm->procflow_lock);
 
 
@@ -586,7 +606,7 @@
 
 
 /*
- * Sets the f_abort flag and clears the allrunning flag to stop
+ * Sets the f_abort flag and clears the running count to stop
  * all the flowop execution threads from running. Iterates
  * through the procflow list and deletes all procflows except
  * for the FLOW_MASTER procflow. Resets the f_abort flag when
@@ -596,11 +616,12 @@
 procflow_shutdown(void)
 {
 	procflow_t *procflow = filebench_shm->proclist;
+	int wait_cnt;
 
 	(void) ipc_mutex_lock(&filebench_shm->procflow_lock);
-	filebench_shm->allrunning = 0;
+	filebench_shm->shm_running = 0;
 	filebench_shm->f_abort = 1;
-	procflow_delete_wait_cnt = 0;
+	wait_cnt = SHUTDOWN_WAIT_SECONDS;
 
 	while (procflow) {
 		if (procflow->pf_instance &&
@@ -612,8 +633,11 @@
 		    procflow->pf_name,
 		    procflow->pf_instance,
 		    procflow->pf_pid);
-		(void) procflow_delete(procflow);
+		(void) procflow_delete(procflow, wait_cnt);
 		procflow = procflow->pf_next;
+		/* grow more impatient */
+		if (wait_cnt)
+			wait_cnt--;
 	}
 
 	filebench_shm->f_abort = 0;
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/stats.c
--- a/usr/src/cmd/filebench/common/stats.c	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/stats.c	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -402,6 +402,13 @@
 		return;
 	}
 
+	/* don't print out if run ended in error */
+	if (filebench_shm->f_abort == FILEBENCH_ABORT_ERROR) {
+		filebench_log(LOG_ERROR,
+		    "NO VALID RESULTS! FileBench run terminated prematurely");
+		return;
+	}
+
 	globalstats->fs_etime = gethrtime();
 
 	filebench_log(LOG_DEBUG_SCRIPT, "Stats period = %ds",
@@ -544,6 +551,10 @@
 	flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO];
 	flowop_t *flowop;
 
+	/* don't dump stats if run ended in error */
+	if (filebench_shm->f_abort == FILEBENCH_ABORT_ERROR)
+		return;
+
 	(void) strcpy(filebench_shm->dump_filename, filename);
 
 	filebench_log(LOG_INFO, "in statsdump %s", filename);
@@ -616,6 +627,10 @@
 	flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO];
 	flowop_t *flowop;
 
+	/* don't dump stats if run ended in error */
+	if (filebench_shm->f_abort == FILEBENCH_ABORT_ERROR)
+		return;
+
 	(void) strcpy(filebench_shm->dump_filename, filename);
 
 	if (filebench_shm->dump_fd > 0) {
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/threadflow.c
--- a/usr/src/cmd/filebench/common/threadflow.c	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/threadflow.c	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -82,8 +82,6 @@
 static int
 threadflow_createthread(threadflow_t *threadflow)
 {
-	int fp = 0;
-
 	filebench_log(LOG_DEBUG_SCRIPT, "Creating thread %s, memory = %ld",
 	    threadflow->tf_name,
 	    *threadflow->tf_memsize);
@@ -95,15 +93,12 @@
 	    (void *(*)(void*))flowop_start, threadflow) != 0) {
 		filebench_log(LOG_ERROR, "thread create failed");
 		filebench_shutdown(1);
+		return (FILEBENCH_ERROR);
 	}
 
-	/* XXX */
-	return (fp < 0);
+	return (FILEBENCH_OK);
 }
 
-#ifndef USE_PROCESS_MODEL
-static procflow_t *my_procflow;
-
 /*
  * Terminates (exits) all the threads of the procflow (process).
  * The procflow is determined from a process private pointer
@@ -113,26 +108,30 @@
 static void
 threadflow_cancel(int arg1)
 {
-	threadflow_t *threadflow = my_procflow->pf_threads;
+	threadflow_t *threadflow;
 
 #ifdef HAVE_LWPS
 	filebench_log(LOG_DEBUG_IMPL, "Thread signal handler on tid %d",
 	    _lwp_self());
 #endif
 
+	threadflow = my_procflow->pf_threads;
 	my_procflow->pf_running = 0;
-	exit(0);
 
 	while (threadflow) {
 		if (threadflow->tf_tid) {
+			/* make sure thread has been cleaned up */
+			flowop_destruct_all_flows(threadflow);
+
 			(void) pthread_cancel(threadflow->tf_tid);
 			filebench_log(LOG_DEBUG_IMPL, "Thread %d cancelled...",
 			    threadflow->tf_tid);
 		}
 		threadflow = threadflow->tf_next;
 	}
+
+	exit(0);
 }
-#endif /* USE_PROCESS_MODEL */
 
 /*
  * Creates threads for the threadflows associated with a procflow.
@@ -157,11 +156,9 @@
 	int ret = 0;
 
 	(void) ipc_mutex_lock(&filebench_shm->threadflow_lock);
-#ifndef USE_PROCESS_MODEL
-	my_procflow = procflow;
 
 	(void) signal(SIGUSR1, threadflow_cancel);
-#endif
+
 	while (threadflow) {
 		threadflow_t *newthread;
 		int i;
@@ -178,7 +175,7 @@
 			    threadflow->tf_name, threadflow, i + 1);
 			if (newthread == NULL)
 				return (-1);
-			ret += threadflow_createthread(newthread);
+			ret |= threadflow_createthread(newthread);
 		}
 
 		newthread = threadflow_define_common(procflow,
@@ -188,8 +185,8 @@
 		if (newthread == NULL)
 			return (-1);
 
-		/* Create threads */
-		ret += threadflow_createthread(newthread);
+		/* Create each thread */
+		ret |= threadflow_createthread(newthread);
 
 		threadflow = threadflow->tf_next;
 	}
@@ -201,10 +198,11 @@
 	while (threadflow) {
 		void *status;
 
+		/* wait for all threads to finish */
 		if (threadflow->tf_tid)
 			(void) pthread_join(threadflow->tf_tid, &status);
 
-		ret += *(int *)status;
+		ret |= *(int *)status;
 		threadflow = threadflow->tf_next;
 	}
 
@@ -218,11 +216,17 @@
  * its associated process to end the thread.
  */
 static void
-threadflow_kill(threadflow_t *threadflow)
+threadflow_kill(threadflow_t *threadflow, int wait_cnt)
 {
 	/* Tell thread to finish */
 	threadflow->tf_abort = 1;
 
+	/* wait a bit for threadflow to stop */
+	while (wait_cnt && threadflow->tf_running) {
+		(void) sleep(1);
+		wait_cnt--;
+	}
+
 #ifdef USE_PROCESS_MODEL
 #ifdef HAVE_SIGSEND
 	(void) sigsend(P_PID, threadflow->tf_process->pf_pid, SIGUSR1);
@@ -244,7 +248,8 @@
  * returns -1.
  */
 static int
-threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow)
+threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
+    int wait_cnt)
 {
 	threadflow_t *entry = *threadlist;
 
@@ -262,7 +267,7 @@
 		    threadflow->tf_name,
 		    threadflow->tf_instance);
 
-		threadflow_kill(threadflow);
+		threadflow_kill(threadflow, wait_cnt);
 		flowop_delete_all(&threadflow->tf_ops);
 		*threadlist = threadflow->tf_next;
 		ipc_free(FILEBENCH_THREADFLOW, (char *)threadflow);
@@ -283,7 +288,7 @@
 			    "Deleted thread: (%s-%d)",
 			    entry->tf_next->tf_name,
 			    entry->tf_next->tf_instance);
-			threadflow_kill(entry->tf_next);
+			threadflow_kill(entry->tf_next, wait_cnt);
 			flowop_delete_all(&entry->tf_next->tf_ops);
 			ipc_free(FILEBENCH_THREADFLOW, (char *)threadflow);
 			entry->tf_next = entry->tf_next->tf_next;
@@ -301,7 +306,7 @@
  * except the FLOW_MASTER.
  */
 void
-threadflow_delete_all(threadflow_t **threadlist)
+threadflow_delete_all(threadflow_t **threadlist, int wait_cnt)
 {
 	threadflow_t *threadflow = *threadlist;
 
@@ -315,8 +320,11 @@
 			threadflow = threadflow->tf_next;
 			continue;
 		}
-		(void) threadflow_delete(threadlist, threadflow);
+		(void) threadflow_delete(threadlist, threadflow, wait_cnt);
 		threadflow = threadflow->tf_next;
+		/* grow more impatient */
+		if (wait_cnt > 0)
+			wait_cnt--;
 	}
 
 	(void) ipc_mutex_unlock(&filebench_shm->threadflow_lock);
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/common/threadflow.h
--- a/usr/src/cmd/filebench/common/threadflow.h	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/common/threadflow.h	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -111,7 +111,7 @@
 void flowop_start(threadflow_t *threadflow);
 void threadflow_usage(void);
 void threadflow_allstarted(pid_t pid, threadflow_t *threadflow);
-void threadflow_delete_all(threadflow_t **threadlist);
+void threadflow_delete_all(threadflow_t **threadlist, int wait_cnt);
 
 #ifdef	__cplusplus
 }
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/workloads/bringover.f
--- a/usr/src/cmd/filebench/workloads/bringover.f	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/workloads/bringover.f	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 # ident	"%Z%%M%	%I%	%E% SMI"
@@ -32,6 +32,8 @@
 set $iosize=1m
 set $nthreads=1
 
+set mode quit alldone
+
 define fileset name=srcfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc
 define fileset name=destfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth
 
@@ -48,11 +50,12 @@
   }
 }
 
-echo  "Bringover Version 2.1 personality successfully loaded"
+echo  "Bringover Version 2.2 personality successfully loaded"
 usage "Usage: set \$dir=<dir>"
 usage "       set \$filesize=<size>   defaults to $filesize"
 usage "       set \$nfiles=<value>    defaults to $nfiles"
 usage "       set \$iosize=<size>     defaults to $iosize"
 usage "       set \$dirwidth=<value>  defaults to $dirwidth"
 usage "       set \$nthreads=<value>  defaults to $nthreads"
-usage "       run runtime (e.g. run 60)"
+usage " "
+usage "       run 0"
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/workloads/copyfiles.f
--- a/usr/src/cmd/filebench/workloads/copyfiles.f	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/workloads/copyfiles.f	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 # ident	"%Z%%M%	%I%	%E% SMI"
@@ -31,6 +31,8 @@
 set $iosize=1m
 set $nthreads=1
 
+set mode quit alldone
+
 define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc=100
 define fileset name=destfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth
 
@@ -47,11 +49,12 @@
   }
 }
 
-echo  "CopyFiles Version 2.1 personality successfully loaded"
+echo  "CopyFiles Version 2.2 personality successfully loaded"
 usage "Usage: set \$dir=<dir>"
 usage "       set \$filesize=<size>   defaults to $filesize"
 usage "       set \$nfiles=<value>    defaults to $nfiles"
 usage "       set \$iosize=<size>     defaults to $iosize"
 usage "       set \$dirwidth=<value>  defaults to $dirwidth"
 usage "       set \$nthreads=<value>  defaults to $nthreads"
-usage "       run runtime (e.g. run 60)"
+usage " "
+usage "       run 0"
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/workloads/createfiles.f
--- a/usr/src/cmd/filebench/workloads/createfiles.f	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/workloads/createfiles.f	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 # ident	"%Z%%M%	%I%	%E% SMI"
@@ -31,6 +31,8 @@
 set $iosize=1m
 set $nthreads=16
 
+set mode quit alldone
+
 define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth
 
 define process name=filecreate,instances=1
@@ -44,7 +46,7 @@
   }
 }
 
-echo  "Createfiles Version 2.1 personality successfully loaded"
+echo  "Createfiles Version 2.2 personality successfully loaded"
 usage "Usage: set \$dir=<dir>"
 usage "       set \$filesize=<size>    defaults to $filesize"
 usage "       set \$iosize=<size>      defaults to $iosize"
@@ -53,4 +55,4 @@
 usage "       set \$meandirwidth=<size> defaults to $meandirwidth"
 usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)"
 usage " "
-usage "       run runtime (e.g. run 60)"
+usage "       run 0"
diff -r 23e77aa611b1 -r d5f45b4dae7e usr/src/cmd/filebench/workloads/deletefiles.f
--- a/usr/src/cmd/filebench/workloads/deletefiles.f	Tue Feb 26 11:31:49 2008 -0800
+++ b/usr/src/cmd/filebench/workloads/deletefiles.f	Tue Feb 26 11:39:00 2008 -0800
@@ -19,7 +19,7 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 # ident	"%Z%%M%	%I%	%E% SMI"
@@ -30,6 +30,8 @@
 set $filesize=16k
 set $nthreads=16
 
+set mode quit alldone
+
 define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=100
 
 define process name=filedelete,instances=1
@@ -41,7 +43,7 @@
   }
 }
 
-echo  "Deletefiles Version 2.0 personality successfully loaded"
+echo  "Deletefiles Version 2.1 personality successfully loaded"
 usage "Usage: set \$dir=<dir>"
 usage "       set \$filesize=<size>    defaults to $filesize"
 usage "       set \$nfiles=<value>     defaults to $nfiles"
@@ -49,4 +51,4 @@
 usage "       set \$meandirwidth=<size> defaults to $meandirwidth"
 usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)"
 usage " "
-usage "       run runtime (e.g. run 60)"
+usage "       run 0"