diff -r -u /tmp/rds-tools-2.0.4/rds-stress.c rds-tools-2.0.7/rds-stress.c

--- /tmp/rds-tools-2.0.4/rds-stress.c	Wed Aug  4 15:25:10 2010

+++ rds-tools-2.0.7/rds-stress.c	Thu Feb 24 13:27:52 2011

@@ -15,7 +15,13 @@

 #include <sys/time.h>

 #include <time.h>

 #include <inttypes.h>

+#if defined(__SVR4) && defined(__sun)

+#include <sys/syscall.h>

+#include <signal.h>

+#include <sys/lgrp_user.h>

+#else

 #include <syscall.h>

+#endif

 #include <sys/stat.h>

 #include <sys/poll.h>

 #include <ctype.h>

 #include <fcntl.h>

 #include <sched.h>

 #include <getopt.h>

+#include <sys/ioctl.h>

+#if !(defined(__SVR4) && defined(__sun))

 #include <byteswap.h>

 #include "rds.h"

-

+#else

+#include <sys/rds.h>

+#endif

 #include "pfhack.h"

+#if defined(__SVR4) && defined(__sun)

+#include <infiniband/ofa_solaris.h>

+#endif

 /*

+ */

+#endif

+

+/*

  *

  * TODO

  *  - checksum the data some day.

 	uint32_t	connect_retries;

 } __attribute__((packed));

+#define MAX_BUCKETS 16

+

 static struct options	opt;

 static int		control_fd;

+static uint64_t         rtt_threshold;

+static int              show_histogram;

+static int get_bucket(uint64_t rtt_time)

+{

+  int i;

+  uint64_t l_rtt_time = rtt_time;

+

+  if (!l_rtt_time)

+    i = 0;

+  else

+  {

+    i = -1;

+    while (l_rtt_time)

+    {

+      i++;

+      l_rtt_time = (l_rtt_time >> 1);

+    }

+  }

+

+  return i;

+}

+

 struct counter {

 	uint64_t	nr;

 	uint64_t	sum;

 #define NR_STATS S__LAST

+#if defined(__SVR4) && defined(__sun)

+int sol_ioctl(int, int, struct rds_info_arg *, socklen_t *, int *);

+#endif

+

 /*

  * Parents share a mapped array of these with their children.  Each child

  * gets one.  It's used to communicate between the child and the parent

 struct child_control {

 	pid_t pid;

 	int ready;

+	int stopping;

 	struct timeval start;

 	struct counter cur[NR_STATS];

 	struct counter last[NR_STATS];

+        uint64_t       latency_histogram[MAX_BUCKETS];

 } __attribute__((aligned (256))); /* arbitrary */

 struct soak_control {

  */

 #define OP_REQ		1

 #define OP_ACK		2

+#define OP_DUMP		3

 #define RDMA_OP_READ	1

 #define RDMA_OP_WRITE	2

 	uint16_t	from_port;

 	uint16_t	to_port;

 	uint16_t	index;

-	uint8_t		op;

 	/* RDMA related.

 	 * rdma_op must be the first field, because we

 	uint32_t        rdma_vector;

+	uint8_t         pending;

 } __attribute__((packed));

 #define MIN_MSG_BYTES		(sizeof(struct header))

 #define BASIC_HEADER_SIZE	(size_t)(&((struct header *) 0)->rdma_op)

+#define print_outlier(...) do {         \

+        fprintf(stderr, __VA_ARGS__);   \

+} while (0)

+

 #define die(fmt...) do {		\

 	fprintf(stderr, fmt);		\

 	exit(1);			\

 	die("invalid host name or dotted quad '%s'\n", ptr);

 }

+#if defined(__SVR4) && defined(__sun)

+static lgrp_id_t lgrp_id = -1;

+static void

+set_my_lgrp(void)

+{

+	if (lgrp_id != -1) {

+		lgrp_affinity_set(P_LWPID, P_MYID, lgrp_id,

+			LGRP_AFF_STRONG);

+		yield(); /* force a context switch */

+	}

+}

+#endif

+

 static void usage(void)

 {

         fprintf(stderr, "rds-stress version %s\n", RDS_VERSION);

 	" -d [depth, 1]     request pipeline depth, nr outstanding\n"

 	" -t [nr, 1]        number of child tasks\n"

 	" -T [seconds, 0]   runtime of test, 0 means infinite\n"

+	" -Q [tos, 0]       Type of Service\n"

 	" -D [bytes]        RDMA: size\n"

 	" -I [iovecs, 1]    RDMA: number of user buffers to target (max 512)\n"

         " -M [nr, 0]        RDMA: mode (0=readwrite,1=readonly,2=writeonly)\n"

 	" -c                measure cpu use with per-cpu soak processes\n"

 	" -V                trace execution\n"

 	" -z                print a summary at end of test only\n"

+#if defined(__SVR4) && defined(__sun)

+	" -g [lgrpid]       bind the process to the specified lgrp\n"

+#endif

 	"\n"

 	"Example:\n"

 	"  recv$ rds-stress\n"

 static void check_parent(pid_t pid)

 {

 	if (pid != getppid())

-		die("parent %u exited\n", pid);

+		die("parent %u exited\n", (int)pid);

 }

 /*

 		msg_pattern[i] = k;

 }

+#if !(defined(__SVR4) && defined(__sun))

 #if __BYTE_ORDER == __LITTLE_ENDIAN

 #define htonll(x)	bswap_64(x)

 #define ntohll(x)	bswap_64(x)

 #define htonll(x)	(x)

 #define ntohll(x)	(x)

 #endif

+#endif /* Not sun */

 static void encode_hdr(struct header *dst, const struct header *hdr)

 {

 	dst->rdma_key = htonll(hdr->rdma_key);

 	dst->rdma_size = htonl(hdr->rdma_size);

 	dst->rdma_vector = htonl(hdr->rdma_vector);

+	dst->retry = hdr->retry;

 }

 static void decode_hdr(struct header *dst, const struct header *hdr)

 	dst->rdma_key = ntohll(hdr->rdma_key);

 	dst->rdma_size = ntohl(hdr->rdma_size);

 	dst->rdma_vector = ntohl(hdr->rdma_vector);

+	dst->retry = hdr->retry;

 }

 static void fill_hdr(void *message, uint32_t bytes, struct header *hdr)

  * Compare incoming message header with expected header. All header fields

  * are in host byte order except for address and port fields.

  */

-static int check_hdr(void *message, uint32_t bytes, const struct header *hdr)

+static int check_hdr(void *message, uint32_t bytes, struct header *hdr, struct options *opts)

 {

 	struct header msghdr;

+	uint32_t	inc_seq;

+	uint32_t	my_seq;

 	decode_hdr(&msghdr, message);

+	inc_seq = msghdr.seq;

+	my_seq = hdr->seq;

+

+	if (msghdr.retry && (inc_seq < my_seq))

+		return -1;

+

 	if (memcmp(&msghdr, hdr, BASIC_HEADER_SIZE)) {

 #define bleh(var, disp)					\

 		disp(hdr->var),				\

 		 * with stdout() and we don't get things stomping on each

 		 * other

 		 */

-		printf( "An incoming message had a header which\n"

+		printf( "An incoming message had a %s header which\n"

 			"didn't contain the fields we expected:\n"

 			"    member        expected eq             got\n"

 			"       seq %15u %s %15u\n"

 			"   to_port %15u %s %15u\n"

 			"     index %15u %s %15u\n"

 			"        op %15u %s %15u\n",

+			(msghdr.retry) ? "RETRY" : "",

 			bleh(seq, /**/),

 			bleh(from_addr, inet_ntoa_32),

 			bleh(from_port, ntohs),

 	fcntl(fd, F_SETFL, O_NONBLOCK);

+	if (opts->tos && ioctl(fd, SIOCRDSSETTOS, &opts->tos)) 

+		die_errno("ERROR: failed to set TOS\n");

+

 	return fd;

 }

 	if (opts->receive_addr == 0)

 		return 1;

+#if defined(__SVR4) && defined(__sun)

+	sin.sin_family = AF_INET_OFFLOAD;

+#else

 	sin.sin_family = AF_INET;

+#endif

 	sin.sin_port = htons(opts->starting_port);

 	sin.sin_addr.s_addr = htonl(opts->receive_addr);

 	mr_args.flags = RDS_FREE_MR_ARGS_INVALIDATE;

 #endif

 	if (setsockopt(fd, sol, RDS_FREE_MR, &mr_args, sizeof(mr_args)))

-		die_errno("setsockopt(RDS_FREE_MR) failed");

+		return;

 	mrs_allocated--;

 }

 	size = sizeof(struct rdma_key_o_meter)

 			+ 2 * nr_tasks * sizeof(*kt)

 			+ 2 * RDMA_MAX_TRACKED_KEYS * sizeof(*ks);

+#if defined(__SVR4) && defined(__sun)

+	base = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_SHARED, -1, 0);

+#else

 	base = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_SHARED, 0, 0);

+#endif

 	if (base == MAP_FAILED)

 		die_errno("alloc_rdma_buffers: mmap failed");

 	}

 	if (!failed)

-		trace("compare pass pattern %Lx addr %p\n",

+		trace("compare pass pattern 0x%Lx addr %p\n",

 			(unsigned long long) pattern, addr);

 }

+struct retry_entry {

+	uint32_t	retries;

+	uint32_t	seq;

+	int		status;

+};

+

 struct task {

 	unsigned int		nr;

 	unsigned int		pending;

+	int			trace;

 	unsigned int		unacked;

 	struct sockaddr_in	src_addr;	/* same for all tasks */

 	struct sockaddr_in	dst_addr;

 	uint16_t		recv_index;

 	struct timeval *	send_time;

 	struct header *		ack_header;

+	struct header *         ack2_header;

+	struct header *         req_header;

+	uint64_t *		retry_token;

+	uint32_t		retries;

+	uint32_t            	last_retry_seq;

+	uint32_t		retry_index;

+

 	/* RDMA related stuff */

 	uint64_t **		local_buf;

 	uint64_t **		rdma_buf;

 	/* We use mmap here rather than malloc, because it is always

 	 * page aligned. */

 	len = 2 * opts->nr_tasks * opts->req_depth * (opts->rdma_vector * opts->rdma_size) + sys_page_size;

+#if defined(__SVR4) && defined(__sun)

+	base = mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);

+#else	

 	base = mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);

+#endif

 	if (base == MAP_FAILED)

 		die_errno("alloc_rdma_buffers: mmap failed");

 	memset(base, 0x2f, len);

 	if (RDMA_OP_READ == hdr->rdma_op) {

 		if (opt.verify)

 			rds_fill_buffer(rdma_addr, rdma_size, hdr->rdma_pattern);

-		trace("Requesting RDMA read for pattern %Lx "

-				"local addr to rdma read %p\n",

-				(unsigned long long) hdr->rdma_pattern,

+		trace("Requesting RDMA read for pattern 0x%Lx"

+				"local addr to rdma read 0x%p\n",

+				hdr->rdma_pattern,

 				rdma_addr);

 	} else {

 		if (opt.verify)

 			rds_fill_buffer(rdma_addr, rdma_size, 0);

-		trace("Requesting RDMA write for pattern %Lx "

-				"local addr to rdma write %p\n",

-				(unsigned long long) hdr->rdma_pattern,

-				rdma_addr);

+

+		trace("Requesting RDMA write for pattern 0x%Lx",

+				hdr->rdma_pattern);

 	}

 }

 		die("Unexpected RDMA op %u in request\n", in_hdr->rdma_op);

-	trace("RDS received request to issue rdma %s len %lu rva %Lx key %Lx pattern %Lx\n",

+	trace("RDS received request to issue rdma %s len %lu rva 0x%Lx key 0x%Lx pattern 0x%Lx\n",

 		in_hdr->rdma_op == RDMA_OP_WRITE? "write to" : "read from",

 		rdma_size,

 		(unsigned long long) in_hdr->rdma_addr,

 	hdr->rdma_vector = in_hdr->rdma_vector;

 }

-static inline unsigned int rdma_user_token(struct task *t, unsigned int qindex)

+static inline uint64_t rdma_user_token(struct task *t, unsigned int qindex,  unsigned int type, uint32_t seq)

 {

-	return t->nr * opt.req_depth + qindex;

+	uint64_t tmp = seq;

+	return (tmp << 32) | ((t->nr * opt.req_depth + qindex) << 2 | type);

 }

-static void rdma_mark_completed(struct task *tasks, unsigned int token, int status)

+static void rdma_mark_completed(struct task *tasks, uint64_t token, int status, struct options *opts)

 {

 	struct task *t;

 	unsigned int i;

+	struct header *hdr = NULL;

+	uint32_t seq = token >> 32;

+	unsigned int type = token & 0x03;

+	unsigned int index = (token & 0xFFFFFFFF) >> 2;

-	trace("RDS rdma completion for token %x\n", token);

+	trace("RDS rdma completion for token 0x%lx\n", token);

-	t = &tasks[token / opt.req_depth];

-	i = token % opt.req_depth;

+	t = &tasks[index / opt.req_depth];

+	i = index % opt.req_depth;

+	if (opts->async) {

+		if (type == OP_REQ)

+			hdr = &t->req_header[i];

+		else

+			hdr = &t->ack2_header[i];

+	}

+

 	if (status) {

 		const char *errmsg;

 		switch (status) {

 		case RDS_RDMA_REMOTE_ERROR:

 			errmsg = "remote error"; break;

-		case RDS_RDMA_CANCELED:

-			errmsg = "operation was cancelled"; break;

 			errmsg = "operation was dropped"; break;

+			errmsg = "operation was cancelled"; break;

 			errmsg = "other error"; break;

 		default:

 			errmsg = "unknown error"; break;

 		}

-		printf("%s:%u: RDMA op %u failed: %s\n",

+		trace("%s:%u: %s failed: %s\n",

 				inet_ntoa(t->dst_addr.sin_addr),

 				ntohs(t->dst_addr.sin_port),

-				i, errmsg);

+				type ? "SEND" : "RDMA",

+				errmsg);

+

+		if (hdr &&