6926434 ib_read_bw, ib_read_lat: OFED utilities sometimes hang when using "-e" (event) flag
6996726 "rds-stress --show-perfdata" option is broken.
7003185 rds-stress man page needs cleanup
7005654 qperf: 32bit only: qperf fails in all RC/UD streaming tests
7024095 set_nodedesc.sh: heading whitespace of HCA specific desc string is ignored if '-N' not specified
7043392 OFED 1.5.3: test_verbs: 'resize CQ' test failed on tavor
7043758 OFED 1.5.3: test_verbs: core dump while during async test on tavor with snv_166
7044543 ibsysstat server process fails to get cpu info
7046730 ibstatus needs to clean up after itself
7050802 OFED 1.5.3: ib_send_bw/ib_send_lat doesn't work with '-g' option
7061241 OFED 1.5.3 ib_read_lat/ib_read_bw don't work between tavor and hermon
7087339 modify solaris changes to libmlx4 to use returned inline size from hermon driver
7090343 solaris_set_nodedesc: the -N option does not work
7091277 /usr/man/man3/ibnd_debug.3 and ibnd_destroy_fabric.3 refer to non-existence ibnd_discover_fabric.3
7091649 OFED 1.5.3: ibdiagnet: "-vlr -r" shows file open failure messages on Solaris
7093499 ib_rdma_lat, ib_read_lat, ib_write_lat and other IB verb latency tools should use gethrtime
7095000 mem leak in libibvers ibv_read_sysfs_file()
7095879 resize cq in libmlx4 incorrect
7095943 rdma_lat & rdma_bw core dump on non ib system
7099692 Add man pages for OFUV perftest utilities
7108873 definitions in sol_uverbs_ioctl.h & sol_umad_ioctl.h are duplicated in solaris_compatibility.c
7119924 ibportstate: operations enable, disable, and reset should only be allowed on switch ports
7120891 ibv_devinfo should report a valid active_mtu instead of 'Unknown'
7141996 sol_uverbs should provide ioctl calls to get GIDs and PKEYs for libibverbs (userland changes)
7144445 setnodedesc.sh -v white space issue
7146479 qperf --cpu_affinity doesn't match with solaris cpu no.
7146482 qperf -cm1 sometimes failed with "rdma_listen failed" message
--- a/components/open-fabrics/ibutils/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/ibutils/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -489,6 +489,17 @@
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
+diff -r -u /tmp/ibutils-1.5.7/ibdm/ibdm/Fabric.cpp ibutils-1.5.7/ibdm/ibdm/Fabric.cpp
+--- /tmp/ibutils-1.5.7/ibdm/ibdm/Fabric.cpp Thu Oct 7 07:29:56 2010
++++ ibutils-1.5.7/ibdm/ibdm/Fabric.cpp Mon Sep 19 12:03:07 2011
+@@ -1954,6 +1954,7 @@
+ }*/
+ }
+ f.close();
++ f.clear();
+
+ // Make second pass and build the tables
+ f.open(fn.c_str(),ifstream::in);
diff -r -u /tmp/ibutils-1.5.7/ibdm/ibdm/Fabric.h ibutils-1.5.7/ibdm/ibdm/Fabric.h
--- /tmp/ibutils-1.5.7/ibdm/ibdm/Fabric.h Thu Oct 7 07:29:56 2010
+++ ibutils-1.5.7/ibdm/ibdm/Fabric.h Thu Feb 24 16:51:17 2011
--- a/components/open-fabrics/infiniband-diags/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/infiniband-diags/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -259,7 +259,7 @@
+# Call solaris_set_nodedesc, if operating system is SunOS
+os=`uname -s`
+if [ $os == "SunOS" ]; then
-+ solaris_set_nodedesc $*
++ solaris_set_nodedesc "$@"
+ rc=$?
+ exit $rc
+fi
@@ -436,12 +436,12 @@
-infiniband_base="/sys/class/infiniband"
-def_ibdev="mthca0"
+ibvdevinfo="/usr/bin/ibv_devinfo"
-+ibvdevinfo_results="/tmp/ibvdevinfo.output"
-+tmpout="/tmp/tmpout"
++ibvdevinfo_results="/tmp/ibvdevinfo.output.$$"
++tmpout="/tmp/tmpout.$$"
usage() {
prog=`basename $0`
-@@ -10,58 +11,119 @@
+@@ -10,58 +11,128 @@
echo "Usage: " $prog " [-h] [devname[:portnum]]"
echo " -h: this help screen"
echo " Examples:"
@@ -528,10 +528,14 @@
+ }
+ }
+ END {
-+ if (hcaid_in != "+" && port_in == "+" && hcaid_hit == 0)
++ if (hcaid_in != "+" && port_in == "+" && hcaid_hit == 0) {
+ printf("Fatal error: device '\''%s'\'' not found\n\n", hcaid_in)
-+ if (hacid_in != "+" && port_in != "+" && port_hit == 0)
++ exit 1;
++ }
++ if (hacid_in != "+" && port_in != "+" && port_hit == 0) {
+ printf("Fatal error: port '\''%s:%d'\'' not found\n\n", hcaid_in, port_in)
++ exit 1;
++ }
+ }' $tmpout
}
@@ -554,24 +558,23 @@
+# Check to see if /usr/bin/ibv_devinfo exists.
+if [ ! -x $ibvdevinfo ]; then
+ echo "$ibvdevinfo doesn't exist!"
-+ exit 0
++ exit 1
+fi
-ib_status() {
- ports_dir="$infiniband_base/$1/ports"
-+rm -rf $ibvdevinfo_results
++# Run ibv_devinfo and direct the output to $ibvdevinfo_results.
++$ibvdevinfo -v > $ibvdevinfo_results
++if [ $? != 0 ]; then
++ echo "$ibvdevinfo failed!"
++ exit 1
++fi
++#
- if ! [ -d "$ports_dir" ]; then
- fatal "device '$1': sys files not found ($ports_dir)"
- fi
-+# Run ibv_devinfo and direct the output to $ibvdevinfo_results.
-+$ibvdevinfo -v > $ibvdevinfo_results
-+if [ $? != 0 ]; then
-+ echo "$ibvdevinfo failed!"
-+ exit 0
-+fi
-+#
-
+-
- if [ "$2" = "+" ]; then
- ports=`(cd "$infiniband_base/$1/ports" 2>/dev/null || fatal No devices; echo *)`
- else
@@ -588,10 +591,12 @@
+nhcas=`$ibvdevinfo -l | awk '/HCA/{print $1}'`
+if [ -z $nhcas ]; then
+ echo "No HCAs!"
-+ exit 0
++ rm -f $ibvdevinfo_results
++ exit 1
fi
+egrep "port:|hca_id:|_lid|GID|state:|active_|link_layer:" $ibvdevinfo_results |grep -v _mtu | sed -e 's/(//' -e 's/)//' > $tmpout
++rm -f $ibvdevinfo_results
+
if [ -z "$1" ]; then
- cd $infiniband_base 2>/dev/null || fatal No devices
@@ -599,18 +604,28 @@
- ib_status $dev "+";
- done
+ get_status_ports "+" "+"
++ if [ $? != 0 ]; then
++ rm -f $tmpout
++ exit 1
++ fi
++ rm -f $tmpout
exit 0
fi
-@@ -73,6 +135,7 @@
+@@ -73,6 +144,12 @@
port="+"
fi
- ib_status $dev $port
+ get_status_ports $dev $port
++ if [ $? != 0 ]; then
++ rm -f $tmpout
++ exit 1
++ fi
+
shift
done
++rm -f $tmpout
diff -r -u /tmp/infiniband-diags-1.5.8/scripts/ibcheckwidth.in infiniband-diags-1.5.8/scripts/ibcheckwidth.in
--- /tmp/infiniband-diags-1.5.8/scripts/ibcheckwidth.in Wed Feb 16 02:13:21 2011
+++ infiniband-diags-1.5.8/scripts/ibcheckwidth.in Thu Feb 24 11:26:54 2011
@@ -1577,6 +1592,133 @@
%{_mandir}/man3/*
%doc README COPYING ChangeLog
+diff -r -u /tmp/infiniband-diags-1.5.8/src/ibsysstat.c infiniband-diags-1.5.8/src/ibsysstat.c
+--- /tmp/infiniband-diags-1.5.8/src/ibsysstat.c Wed Feb 16 02:13:21 2011
++++ infiniband-diags-1.5.8/src/ibsysstat.c Wed Oct 5 09:27:26 2011
+@@ -40,6 +40,7 @@
+ #include <unistd.h>
+ #include <string.h>
+ #include <getopt.h>
++#include <signal.h>
+
+ #include <infiniband/umad.h>
+ #include <infiniband/mad.h>
+@@ -62,7 +63,7 @@
+ } cpu_info;
+
+ static cpu_info cpus[MAX_CPUS];
+-static int host_ncpu;
++static int host_ncpu = 0;
+
+ static int server_respond(void *umad, int size)
+ {
+@@ -142,7 +143,8 @@
+ break;
+ case IB_CPUINFO_ATTR:
+ s[0] = '\0';
+- for (i = 0; i < host_ncpu && sz > 0; i++) {
++ for (i = 0; i < host_ncpu && sz > 0 && ret <
++ IB_VENDOR_RANGE2_DATA_SIZE; i++) {
+ n = snprintf(s, sz, "cpu %d: model %s MHZ %s\n",
+ i, cpus[i].model, cpus[i].mhz);
+ if (n >= sz) {
+@@ -154,6 +156,8 @@
+ s += n;
+ ret += n;
+ }
++ if (i < host_ncpu)
++ IBWARN("cpuinfo truncated");
+ ret++;
+ break;
+ default:
+@@ -162,6 +166,20 @@
+ return ret;
+ }
+
++void cleanup(int sig)
++{
++ int i;
++ for (i = 0; i < host_ncpu ; i++) {
++ if (cpus[i].model)
++ free(cpus[i].model);
++ if (cpus[i].mhz)
++ free(cpus[i].mhz);
++ }
++ mad_rpc_close_port(srcport);
++ (void) signal(SIGINT, SIG_DFL);
++ exit (0);
++}
++
+ static uint8_t buf[2048];
+
+ static char *ibsystat_serv(void)
+@@ -171,6 +189,7 @@
+ int attr, mod, size;
+
+ DEBUG("starting to serve...");
++ (void) signal(SIGINT, cleanup);
+
+ while ((umad = mad_receive_via(buf, -1, srcport))) {
+ if (umad_status(buf)) {
+@@ -261,7 +280,25 @@
+ char line[1024] = { 0 }, *s, *e;
+ FILE *f;
+ int ncpu = 0;
++#if defined(__SVR4) && defined(__sun)
++ int i;
++ char mhz[8];
++ sol_cpu_info_t *cpu_info;
+
++ ncpu = sol_get_cpu_info(&cpu_info);
++ ncpu = ncpu < MAX_CPUS ? ncpu : MAX_CPUS;
++ if (ncpu <= 0) {
++ IBWARN("couldn't get cpu info");
++ return 0;
++ }
++
++ for (i = 0; i < ncpu; i++) {
++ cpus[i].model = strdup(cpu_info[i].cpu_name);
++ snprintf(mhz, 8, "%d", cpu_info[i].cpu_mhz);
++ cpus[i].mhz = strdup(mhz);
++ }
++ free(cpu_info);
++#else
+ if (!(f = fopen("/proc/cpuinfo", "r"))) {
+ IBWARN("couldn't open /proc/cpuinfo");
+ return 0;
+@@ -287,6 +324,7 @@
+ }
+
+ fclose(f);
++#endif
+
+ DEBUG("ncpu %d", ncpu);
+
+@@ -314,7 +352,7 @@
+ {
+ int mgmt_classes[3] =
+ { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS, IB_SA_CLASS };
+- int sysstat_class = IB_VENDOR_OPENIB_SYSSTAT_CLASS;
++ int sysstat_class = IB_VENDOR_OPENIB_SYSSTAT_CLASS, i;
+ ib_portid_t portid = { 0 };
+ int attr = IB_PING_ATTR;
+ char *err;
+@@ -342,6 +380,8 @@
+ if (!srcport)
+ IBERROR("Failed to open '%s' port '%d'", ibd_ca, ibd_ca_port);
+
++ bzero((void *)cpus, MAX_CPUS * sizeof (cpu_info));
++
+ if (server) {
+ if (mad_register_server_via(sysstat_class, 1, 0, oui, srcport) <
+ 0)
+@@ -366,5 +406,6 @@
+ IBERROR("ibsystat to %s: %s", portid2str(&portid), err);
+
+ mad_rpc_close_port(srcport);
++
+ exit(0);
+ }
diff -r -u /tmp/infiniband-diags-1.5.8/src/ibnetdiscover.c infiniband-diags-1.5.8/src/ibnetdiscover.c
--- /tmp/infiniband-diags-1.5.8/src/ibnetdiscover.c Wed Feb 16 02:13:21 2011
+++ infiniband-diags-1.5.8/src/ibnetdiscover.c Thu Feb 24 11:27:00 2011
@@ -1630,7 +1772,19 @@
int changed = 0;
int i;
long val;
-@@ -431,6 +432,12 @@
+@@ -351,6 +352,11 @@
+ port_op = QUERY;
+
+ is_switch = get_node_info(&portid, data);
++ if (!is_switch && (port_op == RESET || port_op == DISABLE ||
++ port_op == ENABLE)) {
++ mad_rpc_close_port(srcport);
++ exit(-1);
++ }
+
+ if (port_op != QUERY || changed)
+ printf("Initial %s PortInfo:\n", is_switch ? "Switch" : "CA");
+@@ -431,6 +437,12 @@
mad_decode_field(data, IB_PORT_LINK_SPEED_ENABLED_F,
&lse);
@@ -1643,7 +1797,7 @@
/* Setup portid for peer port */
memcpy(&peerportid, &portid, sizeof(peerportid));
peerportid.drpath.cnt = 1;
-@@ -440,6 +447,7 @@
+@@ -440,6 +452,7 @@
if (ib_resolve_self_via(&selfportid,
&selfport, 0, srcport) < 0)
IBERROR("could not resolve self");
--- a/components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c Wed Feb 29 22:39:04 2012 +0000
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -435,7 +435,8 @@
struct utsname uts_name;
uint64_t hca_guid;
boolean_t guid_inited = B_FALSE;
- extern int ibdebug;
+ extern int ibdebug;
+ char nodename[64];
static char const str_opts[] = "N:H:G:vd";
static const struct option long_opts[] = {
@@ -539,10 +540,23 @@
rc = -1;
goto free_and_ret;
}
- }
- rc = update_nodedesc((char *)uts_name.nodename, NULL,
- 0, NODEDESC_UPDATE_STRING);
+ /*
+ * The common nodedesc string can have max 64 chars.
+ * We can accomodate 63 chars from uname and alike
+ * option -N, we append a space to the nodename.
+ */
+ (void) strncpy(nodename, uts_name.nodename, 63);
+ if (nodename[strlen(nodename)] != ' ')
+ (void) strncat(nodename, " ", 1);
+
+ rc = update_nodedesc(nodename, NULL, 0,
+ NODEDESC_UPDATE_STRING);
+ if (rc) {
+ IBERROR("write common node descriptor failed");
+ rc = -1;
+ }
+ }
free_and_ret:
if (nodedesc)
--- a/components/open-fabrics/libibumad/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/libibumad/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -104,7 +104,7 @@
#define TRACE if (umaddebug) IBWARN
#define DEBUG if (umaddebug) IBWARN
-@@ -172,6 +176,82 @@
+@@ -172,6 +176,73 @@
memcpy(&port->port_guid, gid + 8, sizeof port->port_guid);
snprintf(port_dir + len, sizeof(port_dir) - len, "/pkeys");
@@ -166,20 +166,11 @@
+ num_pkeys = ibv_query_pkey(ctx, portnum, port_attr.pkey_tbl_len,
+ port->pkeys);
+ if (num_pkeys != 0) {
-+ /*
-+ * have to do one at a time.
-+ */
-+ portnum &= 0x7f;
-+ for (i = 0; i < port_attr.pkey_tbl_len; i++) {
-+ if (ibv_query_pkey(ctx, portnum, i,
-+ &(port->pkeys[i]))) {
-+ IBWARN("Could not retrieve pkeys for "
-+ "\"%s\"", ca_name);
-+ ibv_close_device(ctx);
-+ ibv_free_device_list(root_dev_list);
-+ goto clean;
-+ }
-+ }
++ IBWARN("get_port: ibv_query_pkey() failed for \"%s\"",
++ ca_name);
++ ibv_close_device(ctx);
++ ibv_free_device_list(root_dev_list);
++ goto clean;
+ }
+ port->pkeys_size = port_attr.pkey_tbl_len;
+ }
@@ -187,7 +178,7 @@
num_pkeys = scandir(port_dir, &namelist, check_for_digit_name, NULL);
if (num_pkeys <= 0) {
IBWARN("no pkeys found for %s:%u (at dir %s)...",
-@@ -193,6 +273,8 @@
+@@ -193,6 +264,8 @@
port->pkeys_size = num_pkeys;
free(namelist);
namelist = NULL;
@@ -196,7 +187,7 @@
port_dir[len] = '\0';
/* FIXME: handle gids */
-@@ -384,6 +466,7 @@
+@@ -384,6 +457,7 @@
snprintf(dir_name, sizeof(dir_name), "%s/%s/%s",
SYS_INFINIBAND, ca->ca_name, SYS_CA_PORTS_DIR);
@@ -204,7 +195,7 @@
if (!(dir = opendir(dir_name)))
return -ENOENT;
-@@ -425,15 +508,41 @@
+@@ -425,15 +499,41 @@
free(namelist);
closedir(dir);
@@ -246,7 +237,7 @@
release_ca(ca);
return ret;
-@@ -484,6 +593,11 @@
+@@ -484,6 +584,11 @@
int umad_init(void)
{
TRACE("umad_init");
@@ -258,7 +249,7 @@
if (sys_read_uint(IB_UMAD_ABI_DIR, IB_UMAD_ABI_FILE, &abi_version) < 0) {
IBWARN
("can't read ABI version from %s/%s (%m): is ib_umad module loaded?",
-@@ -522,6 +636,28 @@
+@@ -522,6 +627,28 @@
int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
{
@@ -287,7 +278,7 @@
struct dirent **namelist;
int n, i, j = 0;
-@@ -547,6 +683,7 @@
+@@ -547,6 +674,7 @@
}
if (n >= 0)
free(namelist);
--- a/components/open-fabrics/libibverbs/ofa_solaris.h Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/libibverbs/ofa_solaris.h Wed Feb 29 22:39:04 2012 +0000
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -35,7 +35,7 @@
* DESC: OFED Solaris wrapper
*/
#ifndef _OFA_SOLARIS_H
-#define _OFA_SOLARIS_H
+#define _OFA_SOLARIS_H
#include <sys/types.h>
#include <sys/socket.h>
@@ -72,29 +72,29 @@
typedef __s64 int64_t;
#endif
-typedef uint8_t __be8;
-typedef uint16_t __be16;
-typedef uint32_t __be32;
-typedef uint64_t __be64;
+typedef uint8_t __be8;
+typedef uint16_t __be16;
+typedef uint32_t __be32;
+typedef uint64_t __be64;
/* Size of a pointer */
#if defined(_LP64) || defined(_I32LPx)
-#define __WORDSIZE 64
+#define __WORDSIZE 64
#else
-#define __WORDSIZE 32
+#define __WORDSIZE 32
#endif
/*
* Endian definitions
*/
-#define __LITTLE_ENDIAN 1234
-#define __BIG_ENDIAN 4321
+#define __LITTLE_ENDIAN 1234
+#define __BIG_ENDIAN 4321
#ifdef _BIG_ENDIAN
-#define __BYTE_ORDER __BIG_ENDIAN
+#define __BYTE_ORDER __BIG_ENDIAN
#elif defined _LITTLE_ENDIAN
-#define __BYTE_ORDER __LITTLE_ENDIAN
+#define __BYTE_ORDER __LITTLE_ENDIAN
#else
#error unknown endianness
#endif
@@ -105,40 +105,53 @@
#define bswap_8(x) ((x) & 0xff)
#if !defined(__i386) && !defined(__amd64)
-#define bswap_16(x) ((bswap_8(x) << 8) | bswap_8((x) >> 8))
-#define bswap_32(x) (((uint32_t)(x) << 24) | \
- (((uint32_t)(x) << 8) & 0xff0000) | \
- (((uint32_t)(x) >> 8) & 0xff00) | \
- ((uint32_t)(x) >> 24))
+#define bswap_16(x) ((bswap_8(x) << 8) | bswap_8((x) >> 8))
+#define bswap_32(x) (((uint32_t)(x) << 24) | \
+ (((uint32_t)(x) << 8) & 0xff0000) | \
+ (((uint32_t)(x) >> 8) & 0xff00) | \
+ ((uint32_t)(x) >> 24))
#else /* x86 */
-#define bswap_16(x) htons(x)
-#define bswap_32(x) htonl(x)
+#define bswap_16(x) htons(x)
+#define bswap_32(x) htonl(x)
#endif /* !__i386 && !__amd64 */
#if defined(_LP64) || defined(_LONGLONG_TYPE)
#if (!defined(__i386) && !defined(__amd64))
-#define bswap_64(x) (((uint64_t)(x) << 56) | \
- (((uint64_t)(x) << 40) & 0xff000000000000ULL) | \
- (((uint64_t)(x) << 24) & 0xff0000000000ULL) | \
- (((uint64_t)(x) << 8) & 0xff00000000ULL) | \
- (((uint64_t)(x) >> 8) & 0xff000000ULL) | \
- (((uint64_t)(x) >> 24) & 0xff0000ULL) | \
- (((uint64_t)(x) >> 40) & 0xff00ULL) | \
- ((uint64_t)(x) >> 56))
+#define bswap_64(x) (((uint64_t)(x) << 56) | \
+ (((uint64_t)(x) << 40) & 0xff000000000000ULL) | \
+ (((uint64_t)(x) << 24) & 0xff0000000000ULL) | \
+ (((uint64_t)(x) << 8) & 0xff00000000ULL) | \
+ (((uint64_t)(x) >> 8) & 0xff000000ULL) | \
+ (((uint64_t)(x) >> 24) & 0xff0000ULL) | \
+ (((uint64_t)(x) >> 40) & 0xff00ULL) | \
+ ((uint64_t)(x) >> 56))
#else /* x86 */
-#define bswap_64(x) htonll(x)
+#define bswap_64(x) htonll(x)
#endif /* !__i386 && !__amd64 */
-#else /* no uint64_t */
-#define bswap_64(x) ((bswap_32(x) << 32) | bswap_32((x) >> 32))
+#else /* no uint64_t */
+#define bswap_64(x) ((bswap_32(x) << 32) | bswap_32((x) >> 32))
#endif
typedef struct sol_cpu_info_s {
char cpu_name[64];
uint_t cpu_mhz;
- uint_t cpu_num;
+ uint_t cpu_number;
} sol_cpu_info_t;
-int sol_get_cpu_info(sol_cpu_info_t *);
+int sol_get_cpu_info(sol_cpu_info_t **);
+
+typedef struct sol_cpu_stats_s {
+ uint64_t t_user;
+ uint64_t t_kernel;
+ uint64_t t_idle;
+ uint64_t t_iowait;
+ uint64_t t_intr;
+} sol_cpu_stats_t;
+
+int sol_get_cpu_stats(sol_cpu_stats_t *);
+
+void solaris_init(void);
+void solaris_fini(void);
#ifdef __cplusplus
}
--- a/components/open-fabrics/libibverbs/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/libibverbs/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -371,52 +371,20 @@
#include "ibverbs.h"
-@@ -93,6 +98,44 @@
+@@ -93,6 +98,12 @@
int __ibv_query_gid(struct ibv_context *context, uint8_t port_num,
int index, union ibv_gid *gid)
{
+#if defined(__SVR4) && defined(__sun)
-+ struct ibv_query_gid cmd;
-+ struct ibv_query_gid_resp resp;
-+
-+ /*
-+ * Not exported via sysfs, use uverbs command.
-+ */
-+ if (!context || !gid || (index < 0))
-+ return -1;
-+
-+ if (port_num & 0x80) {
-+ if (!index)
-+ return -1;
-+
-+ IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_GID, gid,
-+ sizeof (union ibv_gid) * index);
++ extern int sol_ibv_query_gid();
+
-+ cmd.gid_index = index;
-+ cmd.port_num = port_num;
++ return sol_ibv_query_gid(context, port_num, index, gid);
+
-+ if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
-+ return -1;
-+ else
-+ return 0;
-+ } else {
-+ IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_GID, &resp,
-+ sizeof resp);
-+
-+ cmd.gid_index = index;
-+ cmd.port_num = port_num;
-+
-+ if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
-+ return -1;
-+
-+ memcpy(gid, resp.gid, sizeof resp.gid);
-+ return 0;
-+ }
+#else
char name[24];
char attr[41];
uint16_t val;
-@@ -112,6 +155,7 @@
+@@ -112,6 +123,7 @@
}
return 0;
@@ -424,52 +392,20 @@
}
default_symver(__ibv_query_gid, ibv_query_gid);
-@@ -118,6 +162,44 @@
+@@ -118,6 +130,12 @@
int __ibv_query_pkey(struct ibv_context *context, uint8_t port_num,
int index, uint16_t *pkey)
{
+#if defined(__SVR4) && defined(__sun)
-+ struct ibv_query_pkey cmd;
-+ struct ibv_query_pkey_resp resp;
-+
-+ /*
-+ * Not exported via sysfs, use uverbs command.
-+ */
-+ if (!context || !pkey || (index < 0))
-+ return -1;
-+
-+ if (port_num & 0x80) {
-+ IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_PKEY, pkey,
-+ sizeof (uint16_t) * index);
-+
-+ if (!index)
-+ return -1;
-+ cmd.pkey_index = index;
-+ cmd.port_num = port_num;
++ extern int sol_ibv_query_pkey();
+
-+ if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
-+ return -1;
-+ else
-+ return 0;
-+ } else {
-+
-+ IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_PKEY, &resp,
-+ sizeof resp);
++ return sol_ibv_query_pkey(context, port_num, index, pkey);
+
-+ cmd.pkey_index = index;
-+ cmd.port_num = port_num;
-+
-+ if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
-+ return -1;
-+
-+ *pkey = resp.pkey;
-+ return 0;
-+ }
+#else
char name[24];
char attr[8];
uint16_t val;
-@@ -133,6 +215,7 @@
+@@ -133,6 +151,7 @@
*pkey = htons(val);
return 0;
@@ -477,7 +413,7 @@
}
default_symver(__ibv_query_pkey, ibv_query_pkey);
-@@ -212,6 +295,10 @@
+@@ -212,6 +231,10 @@
struct ibv_comp_channel *channel;
struct ibv_create_comp_channel cmd;
struct ibv_create_comp_channel_resp resp;
@@ -488,7 +424,7 @@
if (abi_ver <= 2)
return ibv_create_comp_channel_v2(context);
-@@ -221,7 +308,23 @@
+@@ -221,7 +244,23 @@
return NULL;
IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_COMP_CHANNEL, &resp, sizeof resp);
@@ -512,7 +448,7 @@
free(channel);
return NULL;
}
-@@ -228,6 +331,9 @@
+@@ -228,6 +267,9 @@
VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
@@ -845,7 +781,7 @@
static char *sysfs_path;
const char *ibv_get_sysfs_path(void)
-@@ -79,12 +83,20 @@
+@@ -79,12 +83,18 @@
char *buf, size_t size)
{
char *path;
@@ -859,14 +795,12 @@
+#if defined(__SVR4) && defined(__sun)
+ len = sol_read_sysfs_file(path, buf, size);
-+
-+ if (len < 0)
-+ free(path);
++ free(path);
+#else
fd = open(path, O_RDONLY);
if (fd < 0) {
free(path);
-@@ -98,6 +110,7 @@
+@@ -98,6 +108,7 @@
if (len > 0 && buf[len - 1] == '\n')
buf[--len] = '\0';
@@ -911,6 +845,22 @@
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }
+@@ -3509,13 +3511,13 @@
+ CFLAGS=$ac_save_CFLAGS
+ elif test $ac_cv_prog_cc_g = yes; then
+ if test "$GCC" = yes; then
+- CFLAGS="-g -O2"
++ CFLAGS="-g -O3"
+ else
+ CFLAGS="-g"
+ fi
+ else
+ if test "$GCC" = yes; then
+- CFLAGS="-O2"
++ CFLAGS="-O3"
+ else
+ CFLAGS=
+ fi
@@ -8721,6 +8723,7 @@
;;
esac
@@ -919,6 +869,22 @@
;;
sunos4*)
+@@ -10945,13 +10948,13 @@
+ CFLAGS=$ac_save_CFLAGS
+ elif test $ac_cv_prog_cc_g = yes; then
+ if test "$GCC" = yes; then
+- CFLAGS="-g -O2"
++ CFLAGS="-g -O3"
+ else
+ CFLAGS="-g"
+ fi
+ else
+ if test "$GCC" = yes; then
+- CFLAGS="-O2"
++ CFLAGS="-O3"
+ else
+ CFLAGS=
+ fi
@@ -11198,7 +11201,7 @@
$as_echo_n "(cached) " >&6
else
@@ -1419,16 +1385,3 @@
#include <infiniband/verbs.h>
#include <infiniband/driver.h>
-@@ -292,8 +294,12 @@
- port_state_str(port_attr.state), port_attr.state);
- printf("\t\t\tmax_mtu:\t\t%s (%d)\n",
- mtu_str(port_attr.max_mtu), port_attr.max_mtu);
-+#if !(defined(__SVR4) && defined(__sun))
- printf("\t\t\tactive_mtu:\t\t%s (%d)\n",
- mtu_str(port_attr.active_mtu), port_attr.active_mtu);
-+#else
-+ printf("\t\t\tactive_mtu:\t\t%s\n", "Unknown");
-+#endif
- printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid);
- printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid);
- printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc);
--- a/components/open-fabrics/libibverbs/solaris_compatibility.c Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/libibverbs/solaris_compatibility.c Wed Feb 29 22:39:04 2012 +0000
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,6 +49,8 @@
#include <sys/param.h>
#include <sys/ib/adapters/hermon/hermon_ioctl.h>
#include <sys/ib/adapters/tavor/tavor_ioctl.h>
+#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_ioctl.h>
+#include <sys/ib/clients/of/sol_umad/sol_umad_ioctl.h>
#include <alloca.h>
#include "../include/infiniband/arch.h"
@@ -57,67 +59,41 @@
#include <pthread.h>
#include <kstat.h>
+
/*
- * The followings will be removed when sol_uverbs_ioctl.h and sol_umad_ioctl.h
+ * The followings will be removed when changes in sol_uverbs_ioctl.h
* are delivered through ON.
*/
-
-#define UVERBS_IOCTL ('v' << 8)
-
-#define IB_USER_VERBS_SOLARIS_ABI_VERSION 1
-
-typedef enum {
- UVERBS_IOCTL_GET_HCA_INFO = UVERBS_IOCTL | 0x01
-} uverbs_ioctl_enum_t;
+#if !defined(UVERBS_IOCTL_NO_CMDS) || (UVERBS_IOCTL_NO_CMDS < 3)
-typedef struct sol_uverbs_hca_info_s {
- char uverbs_hca_psid_string[MAXNAMELEN];
- char uverbs_hca_ibdev_name[MAXNAMELEN];
- char uverbs_hca_driver_name[MAXNAMELEN];
- uint32_t uverbs_hca_driver_instance;
- uint32_t uverbs_hca_vendorid;
- uint16_t uverbs_hca_deviceid;
- uint8_t uverbs_hca_devidx;
- uint8_t uverbs_hca_pad1[5];
-} sol_uverbs_hca_info_t;
+#define UVERBS_IOCTL_GET_PKEYS UVERBS_IOCTL | 0x02
+#define UVERBS_IOCTL_GET_GIDS UVERBS_IOCTL | 0x03
-typedef struct sol_uverbs_info_s {
- int32_t uverbs_abi_version;
- int32_t uverbs_solaris_abi_version;
- int16_t uverbs_hca_cnt;
- int8_t uverbs_pad1[6]; /* Padding for alignment */
- sol_uverbs_hca_info_t uverbs_hca_info[];
-} sol_uverbs_info_t;
-
-#define UMAD_IOCTL ('m' << 8)
-
-#define IB_USER_MAD_SOLARIS_ABI_VERSION 1
-
-typedef enum {
- IB_USER_MAD_GET_PORT_INFO = UMAD_IOCTL | 0x01
-} umad_ioctl_enum_t;
+typedef struct sol_uverbs_pkey_s {
+ int32_t uverbs_solaris_abi_version;
+ int16_t uverbs_port_num;
+ int16_t uverbs_pkey_cnt;
+ int16_t uverbs_pkey_start_index;
+ int8_t uverbs_pad1[6]; /* Padding for alignment */
+ uint16_t uverbs_pkey[];
+} sol_uverbs_pkey_t;
-typedef struct sol_umad_ioctl_port_info_s {
- char umad_port_ibdev_name[MAXNAMELEN];
- uint32_t umad_port_num;
- uint16_t umad_port_idx;
- uint8_t umad_port_pad1[2];
-} sol_umad_ioctl_port_info_t;
+typedef struct sol_uverbs_gid_s {
+ int32_t uverbs_solaris_abi_version;
+ int16_t uverbs_port_num;
+ int16_t uverbs_gid_cnt;
+ int16_t uverbs_gid_start_index;
+ int8_t uverbs_pad1[6]; /* Padding for alignment */
+ uint8_t uverbs_gids[][16];
+} sol_uverbs_gid_t;
-typedef struct sol_umad_ioctl_info_s {
- int32_t umad_abi_version;
- int32_t umad_solaris_abi_version;
- int16_t umad_port_cnt;
- int8_t umad_pad1[6]; /* alignment padding */
- sol_umad_ioctl_port_info_t umad_port_info[];
-} sol_umad_ioctl_info_t;
-
-/* end of sol_uverbs_ioctl.h and sol_umad_ioctl.h contents */
+#endif /* !defined(UVERBS_IOCTL_NO_CMDS) || (UVERBS_IOCTL_NO_CMDS < 3) */
+/* end of sol_uverbs_ioctl.h contents */
/*
* duplicate ABI definitions for HCAs as the HCA abi headers are not
* installed in proto.
- */
+ */
#define MLX4_UVERBS_MAX_ABI_VERSION 3 /* mlx4-abi.h */
#define RDMA_USER_CM_MIN_ABI_VERSION 3 /* rdma_cma_abi.h */
#define RDMA_USER_CM_MAX_ABI_VERSION 4 /* rdma_cma_abi.h */
@@ -427,6 +403,8 @@
static pthread_mutex_t ibdev_cache_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t uverbs_cache_mutex = PTHREAD_MUTEX_INITIALIZER;
+int sol_ibv_query_gid(struct ibv_context *, uint8_t, int, union ibv_gid *);
+int sol_ibv_query_pkey(struct ibv_context *, uint8_t, int, uint16_t *);
void __attribute__((constructor))solaris_init(void);
void __attribute__((destructor))solaris_fini(void);
@@ -588,7 +566,7 @@
snprintf(uverbs_devpath, MAXPATHLEN, "%s/%s%d",
IB_OFS_DEVPATH_PREFIX, UVERBS_KERNEL_SYSFS_NAME_BASE,
- sol_uverbs_minor_dev);
+ sol_uverbs_minor_dev);
/*
* using the first sol_uverbs minor node that can be opened to get
@@ -599,10 +577,10 @@
strerror(errno));
goto error_exit1;
}
-
- bufsize = sizeof(sol_uverbs_info_t) + sizeof(sol_uverbs_hca_info_t) *
+
+ bufsize = sizeof (sol_uverbs_info_t) + sizeof (sol_uverbs_hca_info_t) *
MAX_HCAS;
-
+
buf = malloc(bufsize);
memset(buf, 0, bufsize);
uverbs_infop = (sol_uverbs_info_t *)buf;
@@ -611,7 +589,7 @@
if (ioctl(fd, UVERBS_IOCTL_GET_HCA_INFO, uverbs_infop) != 0) {
fprintf(stderr, "sol_uverbs ioctl failed: %s\n",
strerror(errno));
-
+
goto error_exit2;
}
@@ -635,7 +613,8 @@
info.uvc_hca_instance =
hca_infop->uverbs_hca_driver_instance;
- snprintf(info.uvc_ibdev_hca_path, sizeof (info.uvc_ibdev_hca_path),
+ snprintf(info.uvc_ibdev_hca_path,
+ sizeof (info.uvc_ibdev_hca_path),
"%s/%s%d", IB_HCA_DEVPATH_PREFIX,
hca_infop->uverbs_hca_driver_name,
hca_infop->uverbs_hca_driver_instance);
@@ -668,7 +647,7 @@
close(fd);
error_exit1:
- return(0);
+ return (0);
}
static int
@@ -701,9 +680,9 @@
return (0);
}
- bufsize = sizeof(sol_umad_ioctl_info_t) +
- (sizeof(sol_umad_ioctl_port_info_t) * MAX_HCAS * MAX_HCA_PORTS);
-
+ bufsize = sizeof (sol_umad_ioctl_info_t) +
+ (sizeof (sol_umad_ioctl_port_info_t) * MAX_HCAS * MAX_HCA_PORTS);
+
buf = malloc(bufsize);
memset(buf, 0, bufsize);
umad_infop = (sol_umad_ioctl_info_t *)buf;
@@ -712,7 +691,7 @@
if (ioctl(fd, IB_USER_MAD_GET_PORT_INFO, umad_infop) != 0) {
fprintf(stderr, "sol_umad ioctl failed: %s\n",
strerror(errno));
-
+
goto error_exit;
}
@@ -976,22 +955,14 @@
if (!gids)
goto error_exit3;
/*
- * set high bit of port_num, and try get all gids in one go.
+ * set high bit of port_num to get all gids in one shot.
*/
port_num |= 0x80;
- rv = ibv_query_gid(ctx, port_num, port_attr->gid_tbl_len, gids);
+ rv = sol_ibv_query_gid(ctx, port_num, port_attr->gid_tbl_len,
+ gids);
+ if (rv != 0)
+ goto error_exit4;
- if (rv != 0) {
- /*
- * Quering all gids didn't work try one at a time.
- */
- port_num &= 0x7f;
-
- for (i = 0; i < port_attr->gid_tbl_len; i++) {
- if (ibv_query_gid(ctx, port_num, i, &gids[i]))
- goto error_exit4;
- }
- }
*gid_table = gids;
gids = NULL;
}
@@ -1002,22 +973,15 @@
if (!pkeys)
goto error_exit4;
+ /*
+ * set high bit of port_num to get all pkeys in one shot.
+ */
port_num |= 0x80;
-
- rv = ibv_query_pkey(ctx, port_num, port_attr->pkey_tbl_len,
+ rv = sol_ibv_query_pkey(ctx, port_num, port_attr->pkey_tbl_len,
pkeys);
+ if (rv != 0)
+ goto error_exit5;
- if (rv != 0) {
- /*
- * Quering all gids didn't work try one at a time.
- */
- port_num &= 0x7f;
-
- for (i = 0; i < port_attr->pkey_tbl_len; i++) {
- if (ibv_query_pkey(ctx, port_num, i, &pkeys[i]))
- goto error_exit5;
- }
- }
*pkey_table = pkeys;
pkeys = NULL;
}
@@ -1058,8 +1022,8 @@
int uverbs_indx;
/*
- * Map the user verbs device (uverbs) to the associated
- * hca device.
+ * Map the user verbs device (uverbs) to the associated
+ * hca device.
*/
uverbs_indx = strtol(dev_name + strlen(UVERBS_KERNEL_SYSFS_NAME_BASE),
NULL, 0);
@@ -1164,7 +1128,7 @@
goto exit;
}
- len = 1 + sprintf(buf, "%d", uverbs_abi_version);
+ len = 1 + sprintf(buf, "%d", uverbs_abi_version);
} else {
fprintf(stderr, "Unsupported read: %s\n", path);
}
@@ -1586,35 +1550,280 @@
int
-sol_get_cpu_info(sol_cpu_info_t *info)
+sol_get_cpu_info(sol_cpu_info_t **info_p)
{
kstat_t *ksp;
kstat_named_t *knp;
+ uint_t ncpus = 0, i;
+ sol_cpu_info_t *info;
+
+ ncpus = sysconf(_SC_NPROCESSORS_ONLN);
+
+ if (ncpus <= 0)
+ return (0);
+
+ if (!(*info_p = malloc(ncpus * sizeof (sol_cpu_info_t))))
+ return (-1);
+
+ info = *info_p;
+ bzero((void *)info, ncpus * sizeof (sol_cpu_info_t));
+
+ for (i = 0; i < ncpus; i++) {
+ if ((ksp = kstat_lookup(kc, "cpu_info", i, NULL)) == NULL) {
+ if (i >= ncpus)
+ goto err_exit;
+ else
+ continue;
+ }
+
+ if ((kstat_read(kc, ksp, NULL) == -1)) {
+ if (i >= ncpus)
+ goto err_exit;
+ else
+ continue;
+ }
+
+ if ((knp = (kstat_named_t *)kstat_data_lookup(ksp, "brand"))
+ == NULL) {
+ if (i >= ncpus)
+ goto err_exit;
+ else
+ continue;
+ }
+
+ (void) strlcpy(info[i].cpu_name, knp->value.str.addr.ptr,
+ knp->value.str.len);
+
+ if ((knp = (kstat_named_t *)kstat_data_lookup(ksp, "clock_MHz"))
+ == NULL) {
+ if (i >= ncpus)
+ goto err_exit;
+ else
+ continue;
+ }
+
+ info[i].cpu_mhz = knp->value.ui64;
+ info[i].cpu_number = i;
+ }
+ return (ncpus);
+err_exit:
+ free(info);
+ return (-1);
+}
+
+int
+sol_get_cpu_stats(sol_cpu_stats_t *stats)
+{
+ size_t i, nr_cpus;
+ kstat_t *ksp;
+ kstat_named_t *knp;
+ memset(stats, 0, sizeof (stats));
+ nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
+
+ /* Aggregate the value of all CPUs */
+ for (i = 0; i < nr_cpus; i++) {
+ /*
+ * In case of some cpu_id doesn't have kstat info.,
+ * skip it and continue if it isn't the last one.
+ */
+ if ((ksp = kstat_lookup(kc, "cpu", i, "sys")) == NULL) {
+ if (i >= nr_cpus)
+ return (-1);
+ else
+ continue;
+ }
+
+ if (kstat_read(kc, ksp, NULL) == -1) {
+ if (i >= nr_cpus)
+ return (-1);
+ else
+ continue;
+ }
+
+ if ((knp = (kstat_named_t *)
+ kstat_data_lookup(ksp, "cpu_ticks_user")) == NULL) {
+ if (i >= nr_cpus)
+ return (-1);
+ else
+ continue;
+ }
+
+ stats->t_user += knp->value.ui64;
+
+ if ((knp = (kstat_named_t *)
+ kstat_data_lookup(ksp, "cpu_ticks_kernel")) == NULL) {
+ if ((knp == NULL) && i >= nr_cpus)
+ return (-1);
+ else
+ continue;
+ }
+ stats->t_kernel += knp->value.ui64;
+
+ if ((knp = (kstat_named_t *)
+ kstat_data_lookup(ksp, "cpu_ticks_idle")) == NULL) {
+ if (i >= nr_cpus)
+ return (-1);
+ else
+ continue;
+ }
+ stats->t_idle += knp->value.ui64;
+
+ if ((knp = (kstat_named_t *)
+ kstat_data_lookup(ksp, "cpu_ticks_wait")) == NULL) {
+ if (i >= nr_cpus)
+ return (-1);
+ else
+ continue;
+ }
+ stats->t_iowait += knp->value.ui64;
+
+ if ((knp = (kstat_named_t *)
+ kstat_data_lookup(ksp, "cpu_nsec_intr")) == NULL) {
+ if (i >= nr_cpus)
+ return (-1);
+ else
+ continue;
+ }
+ stats->t_intr += knp->value.ui64; /* This is in NSEC */
+ }
+ return (0);
+}
+
+int
+sol_ibv_query_gid(struct ibv_context *context, uint8_t port_num, int index,
+ union ibv_gid *gid)
+{
+ char uverbs_devpath[MAXPATHLEN];
+ int uverbs_fd;
+ int count, start;
+ sol_uverbs_gid_t *uverbs_gidp;
+
/*
- * We should check all CPUS, and make sure they
- * are all the same or return an array of structs.
+ * Not exported via sysfs, use ioctl.
*/
- ksp = kstat_lookup(kc, "cpu_info", 0, NULL);
- if (ksp == NULL)
+ if (!context || !gid || (index < 0) ||
+ ((port_num & 0x80) && (index == 0)))
return (-1);
- if (kstat_read(kc, ksp, NULL) == -1)
+ snprintf(uverbs_devpath, MAXPATHLEN, "%s/%s", IB_OFS_DEVPATH_PREFIX,
+ context->device->dev_name);
+
+ if ((uverbs_fd = open(uverbs_devpath, O_RDWR)) < 0)
return (-1);
- knp = (kstat_named_t *)kstat_data_lookup(ksp, "brand");
- if (knp == NULL)
+ if (port_num & 0x80) {
+ start = 0;
+ count = index;
+ } else {
+ start = index;
+ count = 1;
+ }
+
+ uverbs_gidp = (sol_uverbs_gid_t *)malloc(count *
+ sizeof (union ibv_gid) + sizeof (sol_uverbs_gid_t));
+ if (uverbs_gidp == NULL) {
+ close(uverbs_fd);
+ return (-1);
+ }
+
+ uverbs_gidp->uverbs_port_num = port_num & 0x7F;
+ uverbs_gidp->uverbs_gid_cnt = count;
+ uverbs_gidp->uverbs_gid_start_index = start;
+
+ if (ioctl(uverbs_fd, UVERBS_IOCTL_GET_GIDS, uverbs_gidp) != 0) {
+#ifdef DEBUG
+ fprintf(stderr, "UVERBS_IOCTL_GET_GIDS failed: %s\n",
+ strerror(errno));
+#endif
+ goto gid_error_exit;
+ }
+
+ if (uverbs_gidp->uverbs_solaris_abi_version !=
+ IB_USER_VERBS_SOLARIS_ABI_VERSION) {
+#ifdef DEBUG
+ fprintf(stderr, "sol_uverbs solaris_abi_version != "
+ "IB_USER_VERBS_SOLARIS_ABI_VERSION : %d\n",
+ uverbs_gidp->uverbs_solaris_abi_version);
+#endif
+ goto gid_error_exit;
+ }
+ memcpy(gid, uverbs_gidp->uverbs_gids, sizeof (union ibv_gid) * count);
+ free(uverbs_gidp);
+ close(uverbs_fd);
+ return (0);
+
+gid_error_exit:
+ free(uverbs_gidp);
+ close(uverbs_fd);
+ return (-1);
+}
+
+int
+sol_ibv_query_pkey(struct ibv_context *context, uint8_t port_num,
+ int index, uint16_t *pkey)
+{
+ char uverbs_devpath[MAXPATHLEN];
+ int uverbs_fd;
+ int count, start;
+ sol_uverbs_pkey_t *uverbs_pkeyp;
+
+ /*
+ * Not exported via sysfs, use ioctl.
+ */
+ if (!context || !pkey || (index < 0) ||
+ ((port_num & 0x80) && (index == 0)))
return (-1);
- (void) strlcpy(info->cpu_name, knp->value.str.addr.ptr,
- knp->value.str.len);
+ snprintf(uverbs_devpath, MAXPATHLEN, "%s/%s", IB_OFS_DEVPATH_PREFIX,
+ context->device->dev_name);
+ if ((uverbs_fd = open(uverbs_devpath, O_RDWR)) < 0)
+ return (-1);
+
+ if (port_num & 0x80) {
+ start = 0;
+ count = index;
+ } else {
+ start = index;
+ count = 1;
+ }
+
+ uverbs_pkeyp = (sol_uverbs_pkey_t *)malloc(count *
+ sizeof (uint16_t) + sizeof (sol_uverbs_pkey_t));
+ if (uverbs_pkeyp == NULL) {
+ close(uverbs_fd);
+ return (-1);
+ }
+
+ uverbs_pkeyp->uverbs_port_num = port_num & 0x7F;
+ uverbs_pkeyp->uverbs_pkey_cnt = count;
+ uverbs_pkeyp->uverbs_pkey_start_index = start;
- knp = (kstat_named_t *)kstat_data_lookup(ksp, "clock_MHz");
- if (knp == NULL)
- return -1;
+ if (ioctl(uverbs_fd, UVERBS_IOCTL_GET_PKEYS, uverbs_pkeyp) != 0) {
+#ifdef DEBUG
+ fprintf(stderr, "UVERBS_IOCTL_GET_PKEYS failed: %s\n",
+ strerror(errno));
+#endif
+ goto pkey_error_exit;
+ }
- info->cpu_mhz = knp->value.ui64;
- info->cpu_num = sysconf(_SC_NPROCESSORS_ONLN);
+ if (uverbs_pkeyp->uverbs_solaris_abi_version !=
+ IB_USER_VERBS_SOLARIS_ABI_VERSION) {
+#ifdef DEBUG
+ fprintf(stderr, "sol_uverbs solaris_abi_version != "
+ "IB_USER_VERBS_SOLARIS_ABI_VERSION : %d\n",
+ uverbs_pkeyp->uverbs_solaris_abi_version);
+#endif
+ goto pkey_error_exit;
+ }
+ memcpy(pkey, uverbs_pkeyp->uverbs_pkey, sizeof (uint16_t) * count);
+ free(uverbs_pkeyp);
+ close(uverbs_fd);
return (0);
+
+pkey_error_exit:
+ free(uverbs_pkeyp);
+ close(uverbs_fd);
+ return (-1);
}
#endif
--- a/components/open-fabrics/libmlx4/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/libmlx4/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -27,7 +27,7 @@
diff -r -u /tmp/839450/libmlx4-1.0.1/src/verbs.c libmlx4-1.0.1/src/verbs.c
--- /tmp/839450/libmlx4-1.0.1/src/verbs.c Thu Mar 10 04:48:34 2011
+++ libmlx4-1.0.1/src/verbs.c Fri Mar 11 14:40:18 2011
-@@ -56,6 +56,15 @@
+@@ -56,6 +56,14 @@
if (ret)
return ret;
@@ -39,11 +39,10 @@
+ */
+ attr->max_srq_wr -=1;
+#endif
-+
major = (raw_fw_ver >> 32) & 0xffff;
minor = (raw_fw_ver >> 16) & 0xffff;
sub_minor = raw_fw_ver & 0xffff;
-@@ -79,6 +88,9 @@
+@@ -79,6 +87,9 @@
struct ibv_alloc_pd cmd;
struct mlx4_alloc_pd_resp resp;
struct mlx4_pd *pd;
@@ -53,14 +52,14 @@
pd = malloc(sizeof *pd);
if (!pd)
-@@ -90,7 +102,16 @@
+@@ -90,7 +101,16 @@
return NULL;
}
+#if defined(__SVR4) && defined(__sun)
+ /*
-+ * The kernel driver passes back the PD table index as opaque data. This
-+ * index is required for specifying the PD in user space address vectors.
++ * kernel driver passes back the PD table index as opaque data. This
++ * is required for specifying the PD in user space address vectors.
+ */
+ mdd = (mlnx_umap_pd_data_out_t *) &resp.ibv_resp.drv_out;
+ pd->pdn = mdd->mpd_pdnum;
@@ -70,7 +69,7 @@
return &pd->ibv_pd;
}
-@@ -168,6 +189,10 @@
+@@ -168,6 +188,10 @@
struct mlx4_create_cq_resp resp;
struct mlx4_cq *cq;
int ret;
@@ -81,7 +80,7 @@
/* Sanity check CQ size before proceeding */
if (cqe > 0x3fffff)
-@@ -184,7 +209,8 @@
+@@ -184,7 +208,8 @@
cqe = align_queue_size(cqe + 1);
@@ -91,7 +90,7 @@
goto err;
cq->set_ci_db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_CQ);
-@@ -198,15 +224,84 @@
+@@ -198,15 +223,78 @@
cmd.buf_addr = (uintptr_t) cq->buf.buf;
cmd.db_addr = (uintptr_t) cq->set_ci_db;
@@ -105,13 +104,11 @@
ret = ibv_cmd_create_cq(context, cqe - 1, channel, comp_vector,
&cq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
&resp.ibv_resp, sizeof resp);
-+
+#if defined(__SVR4) && defined(__sun)
-+ if (ret) {
+ if (ret)
+ goto err;
-+ }
+#else
- if (ret)
++ if (ret)
goto err_db;
+#endif
@@ -132,18 +129,17 @@
+ cqbuf = mmap64((void *)0, mdd->mcq_maplen, (PROT_READ | PROT_WRITE),
+ MAP_SHARED, context->mmap_fd, mdd->mcq_mapoffset);
+
-+ if (cqbuf == MAP_FAILED) {
++ if (cqbuf == MAP_FAILED)
+ goto err_destroy;
-+ }
+
+ /*
+ * Extract hardware driver values for the number of CQEs and the
+ * hardware CQ number to use (needed for user space doorbells).
+ */
-+ cqe = mdd->mcq_numcqe;
-+ cq->cqn = mdd->mcq_cqnum;
-+ cq->buf.buf = cqbuf;
-+ cq->buf.length = mdd->mcq_maplen;
++ cqe = mdd->mcq_numcqe;
++ cq->cqn = mdd->mcq_cqnum;
++ cq->buf.buf = cqbuf;
++ cq->buf.length = mdd->mcq_maplen;
+ cq->ibv_cq.cqe = cqe-1;
+
+ /*
@@ -156,27 +152,24 @@
+ mdd->mcq_polldbr_mapoffset,
+ mdd->mcq_polldbr_maplen,
+ mdd->mcq_polldbr_offset);
-+ if (cq->set_ci_db == NULL) {
++ if (cq->set_ci_db == NULL)
+ goto err_buf;
-+ }
+
+ cq->arm_db = mlx4_alloc_db(to_mctx(context),
+ mdd->mcq_armdbr_mapoffset,
+ mdd->mcq_armdbr_maplen,
+ mdd->mcq_armdbr_offset);
-+ if (cq->arm_db == NULL) {
++ if (cq->arm_db == NULL)
+ goto err_db;
-+ }
+
+ *cq->arm_db = 0;
+ cq->arm_sn = 1;
+ *cq->set_ci_db = 0;
+#endif
-+
return &cq->ibv_cq;
err_db:
-@@ -215,6 +310,22 @@
+@@ -215,6 +303,21 @@
err_buf:
mlx4_free_buf(&cq->buf);
@@ -195,11 +188,10 @@
+
+ ibv_cmd_destroy_cq(&cq->ibv_cq);
+#endif
-+
err:
free(cq);
-@@ -225,12 +336,17 @@
+@@ -225,12 +328,16 @@
{
struct mlx4_cq *cq = to_mcq(ibcq);
struct mlx4_resize_cq cmd;
@@ -211,7 +203,6 @@
+ void *cqbuf;
+ mlnx_umap_cq_data_out_t *mdd;
+#endif
-+
/* Sanity check CQ size before proceeding */
if (cqe > 0x3fffff)
- return EINVAL;
@@ -219,7 +210,7 @@
pthread_spin_lock(&cq->lock);
-@@ -247,32 +363,65 @@
+@@ -247,32 +354,76 @@
goto out;
}
@@ -227,10 +218,9 @@
ret = mlx4_alloc_cq_buf(to_mdev(ibcq->context->device), &buf, cqe);
if (ret)
goto out;
-
+-
- old_cqe = ibcq->cqe;
-- cmd.buf_addr = (uintptr_t) buf.buf;
-+ cmd.buf_addr = (uintptr_t) buf.buf;
+ cmd.buf_addr = (uintptr_t) buf.buf;
+#endif
+ old_cqe = ibcq->cqe;
@@ -248,14 +238,35 @@
- mlx4_free_buf(&buf);
+
+ if (ret) {
-+#if ! (defined(__SVR4) && defined(__sun))
++#if !(defined(__SVR4) && defined(__sun))
+ mlx4_free_buf(&buf);
-+#endif
goto out;
}
- mlx4_cq_resize_copy_cqes(cq, buf.buf, old_cqe);
-+#if defined(__SVR4) && defined(__sun)
++ mlx4_cq_resize_copy_cqes(cq, buf.buf, old_cqe);
++ mlx4_free_buf(&cq->buf);
++ cq->buf = buf;
++#else
++ goto out;
++ }
++ if (cq->buf.buf != NULL) {
++ buf.buf = malloc(cq->buf.length);
++ if (!buf.buf) {
++ ret = ENOMEM;
++ goto out;
++ }
+
+- mlx4_free_buf(&cq->buf);
+- cq->buf = buf;
++ memcpy(buf.buf, cq->buf.buf, cq->buf.length);
++ buf.length = cq->buf.length;
++ ret = munmap((char *)cq->buf.buf, cq->buf.length);
++ if (ret) {
++ free(buf.buf);
++ goto out;
++ }
++ }
+ /*
+ * For Solaris the kernel driver passes back mmap information for
+ * mapping the CQ memory it allocated.
@@ -268,35 +279,26 @@
+ goto out;
+ }
-- mlx4_free_buf(&cq->buf);
-- cq->buf = buf;
+ cqbuf = mmap64((void *)0, mdd->mcq_maplen, (PROT_READ | PROT_WRITE),
+ MAP_SHARED, ibcq->context->mmap_fd, mdd->mcq_mapoffset);
-
++
+ if (cqbuf == MAP_FAILED) {
+ ret = EINVAL;
+ goto out;
+ }
-+#endif
-+
-+ mlx4_cq_resize_copy_cqes(cq, buf.buf, old_cqe);
-+
-+#if !(defined(__SVR4) && defined(__sun))
-+ mlx4_free_buf(&cq->buf);
-+#endif
-+ cq->buf = buf;
-+
-+#if defined(__SVR4) && defined(__sun)
-+ cqe = mdd->mcq_numcqe;
-+ cq->cqn = mdd->mcq_cqnum;
++ cq->buf.buf = buf.buf;
++ cq->buf.length = buf.length;
++ mlx4_cq_resize_copy_cqes(cq, cqbuf, old_cqe);
+ cq->buf.buf = cqbuf;
+ cq->buf.length = mdd->mcq_maplen;
-+ cq->ibv_cq.cqe = cqe-1;
++ free(buf.buf);
++ cq->ibv_cq.cqe = mdd->mcq_numcqe - 1;
++ cq->cqn = mdd->mcq_cqnum;
+#endif
out:
pthread_spin_unlock(&cq->lock);
return ret;
-@@ -287,6 +436,9 @@
+@@ -287,6 +438,9 @@
return ret;
mlx4_free_db(to_mctx(cq->context), MLX4_DB_TYPE_CQ, to_mcq(cq)->set_ci_db);
@@ -306,7 +308,7 @@
mlx4_free_buf(&to_mcq(cq)->buf);
free(to_mcq(cq));
-@@ -300,6 +452,10 @@
+@@ -300,6 +454,10 @@
struct mlx4_create_srq_resp resp;
struct mlx4_srq *srq;
int ret;
@@ -317,7 +319,7 @@
/* Sanity check SRQ size before proceeding */
if (attr->attr.max_wr > 1 << 16 || attr->attr.max_sge > 64)
-@@ -312,6 +468,7 @@
+@@ -312,6 +470,7 @@
if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
goto err;
@@ -325,7 +327,7 @@
srq->max = align_queue_size(attr->attr.max_wr + 1);
srq->max_gs = attr->attr.max_sge;
srq->counter = 0;
-@@ -324,7 +481,23 @@
+@@ -324,7 +483,23 @@
goto err_free;
*srq->db = 0;
@@ -349,7 +351,7 @@
cmd.buf_addr = (uintptr_t) srq->buf.buf;
cmd.db_addr = (uintptr_t) srq->db;
-@@ -331,19 +504,97 @@
+@@ -331,19 +506,97 @@
ret = ibv_cmd_create_srq(pd, &srq->ibv_srq, attr,
&cmd.ibv_cmd, sizeof cmd,
&resp.ibv_resp, sizeof resp);
@@ -447,7 +449,7 @@
err:
free(srq);
-@@ -357,7 +608,16 @@
+@@ -357,7 +610,16 @@
{
struct ibv_modify_srq cmd;
@@ -464,7 +466,7 @@
}
int mlx4_query_srq(struct ibv_srq *srq,
-@@ -365,7 +625,17 @@
+@@ -365,7 +627,17 @@
{
struct ibv_query_srq cmd;
@@ -482,32 +484,18 @@
}
int mlx4_destroy_srq(struct ibv_srq *ibsrq)
-@@ -414,7 +684,11 @@
- return -1;
-
- if (attr->cap.max_inline_data) {
-+#if !(defined(__SVR4) && defined(__sun))
- nsegs = num_inline_segs(attr->cap.max_inline_data, attr->qp_type);
-+#else
-+ nsegs = mlx4_num_inline_segs(attr->cap.max_inline_data, attr->qp_type);
-+#endif
- size = MLX4_MAX_WQE_SIZE - nsegs * sizeof (struct mlx4_wqe_inline_seg);
- switch (attr->qp_type) {
- case IBV_QPT_UD:
-@@ -447,6 +721,12 @@
+@@ -447,6 +719,10 @@
struct mlx4_qp *qp;
int ret;
struct mlx4_context *context = to_mctx(pd->context);
+#if defined(__SVR4) && defined(__sun)
+ mlnx_umap_qp_data_out_t *mdd;
+ void *qpbuf;
-+ int max_send_sge;
-+ int max_inline_data;
+#endif
/* Sanity check QP size before proceeding */
-@@ -457,6 +737,7 @@
+@@ -457,6 +733,7 @@
if (!qp)
return NULL;
@@ -515,7 +503,7 @@
mlx4_calc_sq_wqe_size(&attr->cap, attr->qp_type, qp);
/*
-@@ -466,6 +747,7 @@
+@@ -466,6 +743,7 @@
qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
qp->sq.wqe_cnt = align_queue_size(attr->cap.max_send_wr + qp->sq_spare_wqes);
qp->rq.wqe_cnt = align_queue_size(attr->cap.max_recv_wr);
@@ -523,38 +511,14 @@
if (attr->srq || attr->qp_type == IBV_QPT_XRC)
attr->cap.max_recv_wr = qp->rq.wqe_cnt = 0;
-@@ -476,6 +758,46 @@
+@@ -476,6 +754,22 @@
attr->cap.max_recv_wr = 1;
}
+#if defined(__SVR4) && defined(__sun)
+ if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) ||
-+ pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE)) {
++ pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE))
+ goto err;
-+ }
-+
-+ /*
-+ * We adjust the number of send SGL entries to force the kernel to
-+ * allocate a larger WQE that will fit the inline data requested.
-+ * The Solaris Hermon driver does not look at inline data size when
-+ * calculating the send WQE size, so this allows us to get closer
-+ * to what the user has requested.
-+ */
-+ max_send_sge = align(attr->cap.max_inline_data +
-+ mlx4_num_inline_segs(attr->cap.max_inline_data,
-+ attr->qp_type) * sizeof (struct mlx4_wqe_inline_seg),
-+ sizeof( struct mlx4_wqe_data_seg)) /
-+ sizeof(struct mlx4_wqe_data_seg);
-+
-+
-+ if (max_send_sge > attr->cap.max_send_sge)
-+ attr->cap.max_send_sge = max_send_sge;
-+
-+ if (attr->cap.max_send_sge > context->max_sge) {
-+ free(qp);
-+ return (NULL);
-+ };
-+
+
+ /*
+ * Solaris QP work queue memory is supplied by the kernel, so
@@ -570,7 +534,7 @@
if (mlx4_alloc_qp_buf(pd, &attr->cap, attr->qp_type, qp))
goto err;
-@@ -505,11 +827,120 @@
+@@ -505,17 +799,84 @@
; /* nothing */
cmd.sq_no_prefetch = 0; /* OK for ABI 2: just a reserved field */
memset(cmd.reserved, 0, sizeof cmd.reserved);
@@ -580,11 +544,9 @@
ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
&resp, sizeof resp);
-+
+#if defined(__SVR4) && defined(__sun)
-+ if (ret) {
+ if (ret)
+ goto err_free;
-+ }
+
+ /*
+ * The kernel driver passes back mmap information for mapping the
@@ -599,9 +561,8 @@
+ qpbuf = mmap64((void *)0, mdd->mqp_maplen, (PROT_READ | PROT_WRITE),
+ MAP_SHARED, pd->context->mmap_fd, mdd->mqp_mapoffset);
+
-+ if (qpbuf == MAP_FAILED) {
++ if (qpbuf == MAP_FAILED)
+ goto err_destroy;
-+ }
+
+ /*
+ * Need to set qp->buf here in case alloc_db fails then
@@ -615,100 +576,57 @@
+ mdd->mqp_rdbr_mapoffset,
+ mdd->mqp_rdbr_maplen,
+ mdd->mqp_rdbr_offset);
-+ if (qp->db == NULL) {
++ if (qp->db == NULL)
+ goto err_buf;
-+ }
++
+ *qp->db = 0;
+ }
+
+ /*
-+ * Calculate the official maximum inline data size, this is not done
-+ * by the kernel driver, so we do it here and update the qp struct.
-+ */
-+ max_inline_data =
-+ mdd->mqp_sq_wqesz - sizeof(struct mlx4_wqe_inline_seg);
-+ max_inline_data -= sizeof(struct mlx4_wqe_ctrl_seg);
-+
-+ switch (attr->qp_type) {
-+ case IBV_QPT_UD:
-+ max_inline_data -= sizeof(struct mlx4_wqe_datagram_seg);
-+ break;
-+
-+ case IBV_QPT_UC:
-+ max_inline_data -= sizeof(struct mlx4_wqe_raddr_seg);
-+ break;
-+
-+ case IBV_QPT_RC:
-+ max_inline_data -= sizeof(struct mlx4_wqe_raddr_seg);
-+ if (max_inline_data > (sizeof(struct mlx4_wqe_atomic_seg) +
-+ sizeof(struct mlx4_wqe_raddr_seg) +
-+ sizeof(struct mlx4_wqe_data_seg))) {
-+ max_inline_data -= sizeof(struct mlx4_wqe_atomic_seg) +
-+ sizeof(struct mlx4_wqe_raddr_seg) +
-+ sizeof(struct mlx4_wqe_data_seg);
-+ } else {
-+ max_inline_data = 0;
-+ }
-+ break;
-+
-+ default:
-+ break;
-+ }
-+
-+ attr->cap.max_inline_data = max_inline_data;
-+
-+ /*
+ * Retrieve sendqueue actual size, and the number of headroom WQEs
+ * that were required based on kernel setup of prefetch or not for
+ * send queue.
-+ * Note: mqp_sq_numwqe includes the head room wqes.
-+ * The private wqe.cnt also includes headroom wqes,
-+ * the verbs count should reflect the wqe count that
-+ * is usable.
++ * Note: mqp_sq_numwqe includes the head room wqes. The private
++ * wqe.cnt also includes headroom wqes, the verbs count
++ * should reflect the wqe count that is usable.
+ */
+ qp->sq_spare_wqes = mdd->mqp_sq_headroomwqes;
+ qp->sq.wqe_cnt = mdd->mqp_sq_numwqe;
+
-+ if (attr->srq) {
++ if (attr->srq)
+ qp->rq.wqe_cnt = 0;
-+ } else {
++ else
+ qp->rq.wqe_cnt = mdd->mqp_rq_numwqe;
-+ }
+
+ if (mlx4_set_qp_buf(pd, qp, qpbuf, mdd->mqp_maplen,
+ mdd->mqp_rq_wqesz, mdd->mqp_rq_off,
-+ mdd->mqp_sq_wqesz, mdd->mqp_sq_off)) {
++ mdd->mqp_sq_wqesz, mdd->mqp_sq_off))
+ goto err_rq_db;
+
++ mlx4_init_qp_indices(qp);
++
+ ret = mlx4_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp);
+ if (ret)
+ goto err_rq_db;
-+ }
-+
-+ mlx4_init_qp_indices(qp);
++#else
++ if (ret)
++ goto err_rq_db;
+
+ ret = mlx4_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp);
-+ if (ret) {
-+ goto err_rq_db;
-+ }
-+#else
- if (ret)
- goto err_rq_db;
-
-@@ -516,6 +947,7 @@
- ret = mlx4_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp);
- if (ret)
++ if (ret)
goto err_destroy;
+#endif
pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex);
qp->rq.wqe_cnt = attr->cap.max_recv_wr;
-@@ -536,9 +968,42 @@
+@@ -536,9 +897,38 @@
return &qp->ibv_qp;
+#if defined(__SVR4) && defined(__sun)
+err_rq_db:
-+ if (!attr->srq && attr->qp_type != IBV_QPT_XRC) {
++ if (!attr->srq && attr->qp_type != IBV_QPT_XRC)
+ mlx4_free_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ, qp->db);
-+ }
-+
+err_buf:
+ mlx4_free_buf(&qp->buf);
+
@@ -723,16 +641,14 @@
+ pthread_cond_init(&(qp->ibv_qp.cond), NULL);
+ qp->ibv_qp.events_completed = 0;
ibv_cmd_destroy_qp(&qp->ibv_qp);
-
+err_free:
+ pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex);
-+
+
+ if (qp->sq.wrid)
+ free(qp->sq.wrid);
+
+ if (qp->rq.wrid)
+ free(qp->rq.wrid);
-+
+err:
+ free(qp);
+#else
@@ -742,7 +658,7 @@
err_rq_db:
pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex);
if (!attr->srq && attr->qp_type != IBV_QPT_XRC)
-@@ -552,6 +1017,7 @@
+@@ -552,6 +942,7 @@
err:
free(qp);
@@ -750,7 +666,7 @@
return NULL;
}
-@@ -745,6 +1211,13 @@
+@@ -745,6 +1136,13 @@
struct ibv_cq *xrc_cq,
struct ibv_srq_init_attr *attr)
{
@@ -764,7 +680,7 @@
struct mlx4_create_xrc_srq cmd;
struct mlx4_create_srq_resp resp;
struct mlx4_srq *srq;
-@@ -807,6 +1280,7 @@
+@@ -807,6 +1205,7 @@
free(srq);
return NULL;
@@ -772,34 +688,10 @@
}
struct ibv_xrc_domain *mlx4_open_xrc_domain(struct ibv_context *context,
-@@ -893,5 +1367,4 @@
- {
- return ibv_cmd_unreg_xrc_rcv_qp(xrc_domain, xrc_qp_num);
- }
--
- #endif
diff -r -u /tmp/839450/libmlx4-1.0.1/src/qp.c libmlx4-1.0.1/src/qp.c
--- /tmp/839450/libmlx4-1.0.1/src/qp.c Thu Mar 10 04:48:34 2011
+++ libmlx4-1.0.1/src/qp.c Tue Mar 15 07:09:43 2011
-@@ -511,7 +511,17 @@
- return ret;
- }
-
-+#if defined(__SVR4) && defined(__sun)
-+/*
-+ * Create a non-static version that can be called externally;
-+ * default file local calls to now use the name of the non-static
-+ * version.
-+ */
-+#define num_inline_segs mlx4_num_inline_segs
-+int mlx4_num_inline_segs(int data, enum ibv_qp_type type)
-+#else
- int num_inline_segs(int data, enum ibv_qp_type type)
-+#endif
- {
- /*
- * Inline data segments are not allowed to cross 64 byte
-@@ -589,6 +599,58 @@
+@@ -589,6 +589,58 @@
; /* nothing */
}
@@ -900,17 +792,7 @@
void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind);
int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
struct ibv_recv_wr *wr,
-@@ -399,6 +413,9 @@
- struct mlx4_srq *srq);
- void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
-
-+#if defined(__SVR4) && defined(__sun)
-+int mlx4_num_inline_segs(int data, enum ibv_qp_type type);
-+#endif
- struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
- int mlx4_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
- int attr_mask,
-@@ -415,8 +432,14 @@
+@@ -415,8 +429,14 @@
void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
struct mlx4_qp *qp);
int num_inline_segs(int data, enum ibv_qp_type type);
@@ -1176,7 +1058,7 @@
context = calloc(1, sizeof *context);
if (!context)
-@@ -150,11 +155,32 @@
+@@ -150,11 +155,30 @@
return NULL;
context->ibv_ctx.cmd_fd = cmd_fd;
@@ -1193,10 +1075,8 @@
+ * OFED expects power of two, round up here to make user table
+ * large enough.
+ */
-+ for (temp_qp_num = 1; temp_qp_num < resp.qp_tab_size; temp_qp_num <<= 1) {
++ for (temp_qp_num = 1; temp_qp_num < resp.qp_tab_size; temp_qp_num <<= 1)
+ ;
-+ }
-+
+ resp.qp_tab_size = temp_qp_num;
+
+ /*
@@ -1209,7 +1089,7 @@
context->num_qps = resp.qp_tab_size;
context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS;
context->qp_table_mask = (1 << context->qp_table_shift) - 1;
-@@ -172,20 +198,44 @@
+@@ -172,20 +196,44 @@
for (i = 0; i < MLX4_XRC_SRQ_TABLE_SIZE; ++i)
context->xrc_srq_table[i].refcnt = 0;
@@ -1254,7 +1134,7 @@
if (context->bf_page == MAP_FAILED) {
fprintf(stderr, PFX "Warning: BlueFlame available, "
"but failed to mmap() BlueFlame page.\n");
-@@ -214,6 +264,7 @@
+@@ -214,6 +262,7 @@
context->max_qp_wr = dev_attrs.max_qp_wr;
context->max_sge = dev_attrs.max_sge;
context->max_cqe = dev_attrs.max_cqe;
@@ -1262,7 +1142,7 @@
if (!(dev_attrs.device_cap_flags & IBV_DEVICE_XRC)) {
fprintf(stderr, PFX "There is a mismatch between "
"the kernel and the userspace libraries: "
-@@ -220,6 +271,7 @@
+@@ -220,6 +269,7 @@
"Kernel does not support XRC. Exiting.\n");
goto query_free;
}
@@ -1270,15 +1150,7 @@
return &context->ibv_ctx;
-@@ -227,7 +279,6 @@
- munmap(context->uar, to_mdev(ibdev)->page_size);
- if (context->bf_page)
- munmap(context->bf_page, to_mdev(ibdev)->page_size);
--
- err_free:
- free(context);
- return NULL;
-@@ -240,6 +291,7 @@
+@@ -240,6 +290,7 @@
munmap(context->uar, to_mdev(ibctx->device)->page_size);
if (context->bf_page)
munmap(context->bf_page, to_mdev(ibctx->device)->page_size);
@@ -1320,13 +1192,29 @@
License: GPLv2 or BSD
Url: http://openfabrics.org/
-Source: http://openfabrics.org/downloads/libmlx4/libmlx4-1.0.1.tar.gz
-+Source: http://openfabrics.org/downloads/mlx4/libmlx4-1.0.1.tar.gz
++Source: http://openfabrics.org/downloads/libmlx4-1.0.1.tar.gz
BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
BuildRequires: libibverbs-devel >= 1.1-0.1.rc2
diff -r -u /tmp/839450/libmlx4-1.0.1/configure libmlx4-1.0.1/configure
--- /tmp/839450/libmlx4-1.0.1/configure Thu Mar 10 04:48:41 2011
+++ libmlx4-1.0.1/configure Tue Mar 15 07:35:49 2011
+@@ -3899,13 +3899,13 @@
+ CFLAGS=$ac_save_CFLAGS
+ elif test $ac_cv_prog_cc_g = yes; then
+ if test "$GCC" = yes; then
+- CFLAGS="-g -O2"
++ CFLAGS="-g -O3"
+ else
+ CFLAGS="-g"
+ fi
+ else
+ if test "$GCC" = yes; then
+- CFLAGS="-O2"
++ CFLAGS="-O3"
+ else
+ CFLAGS=
+ fi
@@ -8890,6 +8890,7 @@
;;
esac
@@ -1335,22 +1223,23 @@
;;
sunos4*)
-@@ -11616,6 +11617,14 @@
- # This bug is HP SR number 8606223364.
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of long" >&5
- $as_echo_n "checking size of long... " >&6; }
-+
-+echo $CFLAGS | grep 64 > /dev/null
-+if [ $? -eq 0 ]; then
-+ ac_cv_sizeof_long=8
-+else
-+ ac_cv_sizeof_long=4
-+fi
-+
- if test "${ac_cv_sizeof_long+set}" = set; then :
- $as_echo_n "(cached) " >&6
+@@ -11113,13 +11114,13 @@
+ CFLAGS=$ac_save_CFLAGS
+ elif test $ac_cv_prog_cc_g = yes; then
+ if test "$GCC" = yes; then
+- CFLAGS="-g -O2"
++ CFLAGS="-g -O3"
+ else
+ CFLAGS="-g"
+ fi
else
-@@ -11654,11 +11663,11 @@
+ if test "$GCC" = yes; then
+- CFLAGS="-O2"
++ CFLAGS="-O3"
+ else
+ CFLAGS=
+ fi
+@@ -11654,11 +11655,11 @@
ac_fn_c_check_member "$LINENO" "struct ibv_more_ops" "create_xrc_srq" "ac_cv_member_struct_ibv_more_ops_create_xrc_srq" "#include <infiniband/verbs.h>
"
--- a/components/open-fabrics/open-fabrics.p5m Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/open-fabrics.p5m Wed Feb 29 22:39:04 2012 +0000
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
#
<transform file path=usr.*/man/.+ -> default mangler.man.stability volatile>
@@ -241,6 +241,15 @@
file path=usr/sbin/solaris_set_nodedesc variant.opensolaris.zone=global
file path=usr/share/libibverbs.d/mlx4.driver
file path=usr/share/man/man1/ib_clock_test.1
+file path=usr/share/man/man1/ib_send_bw.1
+link path=usr/share/man/man1/ib_send_lat.1 target=ib_send_bw.1
+file path=usr/share/man/man1/ib_read_bw.1
+link path=usr/share/man/man1/ib_read_lat.1 target=ib_read_bw.1
+file path=usr/share/man/man1/ib_write_bw.1
+link path=usr/share/man/man1/ib_write_post_list.1 target=ib_write_bw.1
+link path=usr/share/man/man1/ib_write_lat.1 target=ib_write_bw.1
+file path=usr/share/man/man1/rdma_bw.1
+link path=usr/share/man/man1/rdma_lat.1 target=rdma_bw.1
file path=usr/share/man/man1/ibdiagnet.1 variant.opensolaris.zone=global
file path=usr/share/man/man1/ibis.1 variant.opensolaris.zone=global
file path=usr/share/man/man1/ibv_asyncwatch.1
@@ -296,8 +305,6 @@
file path=usr/share/man/man1m/sminfo.1m variant.opensolaris.zone=global
file path=usr/share/man/man1m/smpdump.1m variant.opensolaris.zone=global
file path=usr/share/man/man1m/smpquery.1m variant.opensolaris.zone=global
-file path=usr/share/man/man3/ibnd_debug.3 variant.opensolaris.zone=global
-file path=usr/share/man/man3/ibnd_destroy_fabric.3 variant.opensolaris.zone=global
file path=usr/share/man/man3/ibv_alloc_pd.3 variant.opensolaris.zone=global
file path=usr/share/man/man3/ibv_attach_mcast.3 variant.opensolaris.zone=global
file path=usr/share/man/man3/ibv_create_ah_from_wc.3 variant.opensolaris.zone=global
--- a/components/open-fabrics/perftest/Makefile Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/perftest/Makefile Wed Feb 29 22:39:04 2012 +0000
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
#
include ../../../make-rules/shared-macros.mk
@@ -38,7 +38,11 @@
include $(WS_TOP)/make-rules/configure.mk
include ../ofed.mk
-MAN1FILES = ib_clock_test.1
+MAN1FILES = ib_clock_test.1 \
+ ib_write_bw.1 \
+ ib_read_bw.1 \
+ ib_send_bw.1 \
+ rdma_bw.1
include $(WS_TOP)/make-rules/shared-targets.mk
--- a/components/open-fabrics/perftest/manpages/ib_clock_test.1 Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/perftest/manpages/ib_clock_test.1 Wed Feb 29 22:39:04 2012 +0000
@@ -1,5 +1,5 @@
'\" t
-.\" Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+.\" Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
.\" Modified for Solaris to add the Solaris stability classification,
.\" and to add a note about source availability.
.\"
@@ -35,6 +35,6 @@
.sp
.SH "SEE ALSO"
.PP
-\fBib_write_bw\fR(1), \fBib_send_bw\fR(1), \fBib_rdma_lat\fR(1),
+\fBib_write_bw\fR(1), \fBib_send_bw\fR(1), \fBrdma_lat\fR(1),
\fBib_read_bw\fR(1), \fBib_read_lat\fR(1), \fBib_write_bw_postlist\fR(1),
-\fBib_rdma_bw\fR(1), \fBib_write_lat\fR(1), \fBib_send_lat\fR(1)
+\fBrdma_bw\fR(1), \fBib_write_lat\fR(1), \fBib_send_lat\fR(1)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/open-fabrics/perftest/manpages/ib_read_bw.1 Wed Feb 29 22:39:04 2012 +0000
@@ -0,0 +1,125 @@
+'\" t
+.\" Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+.\" Modified for Solaris to add the Solaris stability classification,
+.\" and to add a note about source availability.
+.\"
+.TH IB_READ_BW/IB_READ_LAT 1 " Oct 15, 2011"
+.SH "NAME"
+ib_read_bw, ib_read_lat - Bandwith and latency test using RDMA read transactions.
+.PP
+.SH SYNOPSIS
+.HP
+.nf
+ib_read_bw [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-c, --connection=<RC|UD>]
+ [-m, --mtu=<mtu>] [-s, --size=<size>] [-a, --all]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-u, --qp-timeout=<timeout] [-S, --sl=<sl>]
+ [-x, --gid-index=<index>] [-F, --CPU-freq] [-V, --version]
+ [-b, --bidirectional] [-e, --events] [-N, --no peak-bw]
+ [-o, --outs=<num>] [server]
+.fi
+.HP
+
+.nf
+ib_read_lat [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-c, --connection=<RC|UD>]
+ [-m, --mtu=<mtu>] [-s, --size=<size>] [-a, --all]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-u, --qp-timeout=<timeout] [-S, --sl=<sl>]
+ [-x, --gid-index=<index>] [-F, --CPU-freq] [-V, --version]
+ [-e, --events] [-o, --outs=<num>] [-C, --report-cycles]
+ [-H, --report-histogram] [-U, --report-unsorted] [server]
+.fi
+
+
+.SH DESCRIPTION
+.PP
+.Nm
+The ib_read_bw and ib_read_lat tests are part of the "perftest" utilities that are used for InfiniBand related performance testing. The "perftest" utilities are micro bench mark utilities.
+
+.PP
+First a passive receiving instance is started.
+.RS 12
+
+ #ib_read_bw
+.RE
+.PP
+Then an active sending instance is started, giving it the address at which it will find a listening passive receiver.
+.PP
+.RS 12
+ #ib_read_bw server
+.RE
+.PP
+If options are used then the same options must be used for both client and server.
+.PP
+.SH OPTIONS
+.PP
+The following options are available for use on the command line:
+
+.TP 7
+\fB\-p, --port <port>
+Listen on/connect to port <port> (default 18515)
+.TP
+\fB\-d, --ib-dev <dev>
+Use IB device <dev> (default first device found)
+.TP
+\fB\-i, --ib-port <port number>
+Use port <port number> of IB device (default 1)
+.TP
+\fB\-c, --connection <RC|UD>
+ Connection type RC or UD (default RC)
+.TP
+\fB\-m, --mtu <mtu>s
+Mtu size : 256 - 4096 (default port mtu)
+.TP
+\fB\-s, --size <size>
+Size of message to exchange (default 2)
+.TP
+\fB\-a, --all
+Run sizes from 2 till 2^23
+.TP
+\fB\-n, --iters <iterations>
+Number of exchanges (at least 5, default 1000)
+.TP
+\fB\-t, --tx-depth <depth>
+Size of tx queue (default 50)
+.TP
+\fB\-u, --qp-timeout <timeout>
+QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14
+.TP
+\fB\-S, --sl <sl>
+SL (default 0)
+.TP
+\fB\-x, --gid-index <index>
+Test uses GID with GID index (Default : IB - no gid)
+.TP
+\fB\-F, --CPU-freq
+Do not fail even if cpufreq_ondemand module is loaded
+.TP
+\fB\-V, --version
+Display version number
+.TP
+\fB\-b, --bidirectional
+Measure bidirectional bandwidth (default unidirectional)
+.TP
+\fB\-e, --events
+Sleep on CQ events (default poll)
+.TP
+\fB\-N, --no peak-bw
+Cancel peak-bw calculation (default with peak)
+.TP
+\fB\-o, --outs <num>
+Specify <num> as number of outstanding read/atom(default max of device)
+.TP
+\fB\-C, --report-cycles
+report times in cpu cycle units (default microseconds)
+.TP
+\fB\-H, --report-histogram
+Print out all results (default print summary only)
+.TP
+\fB\-U, --report-unsorted
+(implies -H) print out unsorted results (default sorted)
+.PP
+.SH SEE ALSO
+ib_write_bw(1), ib_send_bw(1), rdma_lat(1), ib_write_bw_postlist(1), rdma_bw(1), ib_write_lat(1), ib_send_lat(1), ib_clock_test(1)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/open-fabrics/perftest/manpages/ib_send_bw.1 Wed Feb 29 22:39:04 2012 +0000
@@ -0,0 +1,134 @@
+'\" t
+.\" Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+.\" Modified for Solaris to add the Solaris stability classification,
+.\" and to add a note about source availability.
+.\"
+.TH IB_SEND_BW/IB_SEND_LAT 1 " Oct 15, 2011"
+.SH "NAME"
+ib_send_bw, ib_send_lat - Bandwith and latency test using send transactions.
+.PP
+.SH SYNOPSIS
+.HP
+.nf
+ib_send_bw [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-c, --connection=<RC|UD>]
+ [-m, --mtu=<mtu>] [-s, --size=<size>] [-a, --all]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-u, --qp-timeout=<timeout] [-S, --sl=<sl>]
+ [-x, --gid-index=<index>] [-F, --CPU-freq] [-V, --version]
+ [-r, --rx-depth=<depth>] [-I, --inline_size=<size>]
+ [-b, --bidirectional] [-e, --events] [-N, --no peak-bw]
+ [-g, --mcg=<num>] [-M, --MGID=<multicast_gid>] [server]
+.fi
+.HP
+
+.nf
+ib_send_lat [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-c, --connection=<RC|UD>]
+ [-m, --mtu=<mtu>] [-s, --size=<size>] [-a, --all]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-u, --qp-timeout=<timeout] [-S, --sl=<sl>]
+ [-x, --gid-index=<index>] [-F, --CPU-freq] [-V, --version]
+ [--I, --inline_size=<size>] [-e, --events]
+ [-g, --mcg=<num>] [-M, --MGID=<multicast_gid>]
+ [-C, --report-cycles] [-H, --report-histogram]
+ [-U, --report-unsorted] [server]
+.fi
+
+
+.SH DESCRIPTION
+.PP
+.Nm
+The ib_send_bw and ib_send_lat tests are part of the "perftest" utilities that are used for InfiniBand related performance testing. The "perftest" utilities are micro bench mark utilities.
+
+.PP
+First a passive receiving instance is started.
+.RS 12
+
+ #ib_send_bw
+.RE
+.PP
+Then an active sending instance is started, giving it the address at which it will find a listening passive receiver.
+.PP
+.RS 12
+ #ib_send_bw server
+.RE
+.PP
+If options are used then the same options must be used for both client and server.
+.PP
+.SH OPTIONS
+.PP
+The following options are available for use on the command line:
+
+.TP 7
+\fB\-p, --port <port>
+Listen on/connect to port <port> (default 18515)
+.TP
+\fB\-d, --ib-dev <dev>
+Use IB device <dev> (default first device found)
+.TP
+\fB\-i, --ib-port <port number>
+Use port <port number> of IB device (default 1)
+.TP
+\fB\-c, --connection <RC|UD>
+ Connection type RC or UD (default RC)
+.TP
+\fB\-m, --mtu <mtu>s
+Mtu size : 256 - 4096 (default port mtu)
+.TP
+\fB\-s, --size <size>
+Size of message to exchange (default 2)
+.TP
+\fB\-a, --all
+Run sizes from 2 till 2^23
+.TP
+\fB\-n, --iters <iterations>
+Number of exchanges (at least 5, default 1000)
+.TP
+\fB\-t, --tx-depth <depth>
+Size of tx queue (default 50)
+.TP
+\fB\-u, --qp-timeout <timeout>
+QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14
+.TP
+\fB\-S, --sl <sl>
+SL (default 0)
+.TP
+\fB\-x, --gid-index <index>
+Test uses GID with GID index (Default : IB - no gid)
+.TP
+\fB\-F, --CPU-freq
+Do not fail even if cpufreq_ondemand module is loaded
+.TP
+\fB\-V, --version
+Display version number
+.TP
+\fB\-I, --inline_size <size>
+Max size of message to be sent in inline (default 0)
+.TP
+\fB\-e, --events
+Sleep on CQ events (default poll)
+.TP
+\fB\-g, --mcg=<num_of_qps>
+Send messages to multicast group with <num_of_qps> qps attached to it.
+.TP
+\fB\-M, --MGID=<multicast_gid>
+In multicast, uses <multicast_gid> as the group MGID
+.TP
+\fB\-b, --bidirectional
+Measure bidirectional bandwidth (default unidirectional)
+.TP
+\fB\-N, --no peak-bw
+Cancel peak-bw calculation (default with peak)
+.TP
+\fB\-C, --report-cycles
+report times in cpu cycle units (default microseconds)
+.TP
+\fB\-H, --report-histogram
+Print out all results (default print summary only)
+.TP
+\fB\-U, --report-unsorted
+(implies -H) print out unsorted results (default sorted)
+.PP
+.SH SEE ALSO
+ib_write_bw(1), ib_write_lat(1), rdma_lat(1), ib_read_bw(1), rdma_bw(1), ib_read_lat(1), ib_send_lat(1), ib_clock_test(1), ib_write_bw_post_list(1)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/open-fabrics/perftest/manpages/ib_write_bw.1 Wed Feb 29 22:39:04 2012 +0000
@@ -0,0 +1,137 @@
+'\" t
+.\" Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+.\" Modified for Solaris to add the Solaris stability classification,
+.\" and to add a note about source availability.
+.\"
+.TH IB_WRITE_BW/IB_WRITE_BW_POST_LIST/IB_WRITE_LAT 1 " Oct 15, 2011"
+.SH "NAME"
+ib_write_bw, ib_write_bw_poslist, ib_write_lat - Bandwith and latency test using RDMA write transactions.
+.PP
+.SH SYNOPSIS
+.HP
+.nf
+ib_write_bw [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-c, --connection=<RC|UD>]
+ [-m, --mtu=<mtu>] [-s, --size=<size>] [-a, --all]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-u, --qp-timeout=<timeout] [-S, --sl=<sl>]
+ [-x, --gid-index=<index>] [-F, --CPU-freq] [-V, --version]
+ [-I, --inline_size=<size>] [-b, --bidirectional]
+ [-N, --no peak-bw] [-q, --qp=<num>] [server]
+.fi
+.HP
+
+.nf
+ib_write_bw_postlist [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-c, --connection=<RC|UD>]
+ [-m, --mtu=<mtu>] [-s, --size=<size>] [-a, --all]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-u, --qp-timeout=<timeout] [-S, --sl=<sl>]
+ [-x, --gid-index=<index>] [-F, --CPU-freq] [-V, --version]
+ [-I, --inline_size=<size>] [-b, --bidirectional]
+ [-N, --no peak-bw] [-q, --qp=<num>] [server]
+.fi
+.HP
+
+.nf
+ib_write_lat [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-c, --connection=<RC|UD>]
+ [-m, --mtu=<mtu>] [-s, --size=<size>] [-a, --all]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-u, --qp-timeout=<timeout] [-S, --sl=<sl>]
+ [-x, --gid-index=<index>] [-F, --CPU-freq] [-V, --version]
+ [--I, --inline_size=<size>] [-C, --report-cycles]
+ [-H, --report-histogram] [-U, --report-unsorted] [server]
+.fi
+
+
+.SH DESCRIPTION
+.PP
+.Nm
+The ib_write_bw, ib_write_bw_postlist and ib_write_lat tests are part of the "perftest" utilities that are used for InfiniBand related performance testing. The "perftest" utilities are micro bench mark utilities.
+
+.PP
+First a passive receiving instance is started.
+.RS 12
+
+ #ib_write_bw
+.RE
+.PP
+Then an active sending instance is started, giving it the address at which it will find a listening passive receiver.
+.PP
+.RS 12
+ #ib_write_bw server
+.RE
+.PP
+If options are used then the same options must be used for both client and server.
+.PP
+.SH OPTIONS
+.PP
+The following options are available for use on the command line:
+
+.TP 7
+\fB\-p, --port <port>
+Listen on/connect to port <port> (default 18515)
+.TP
+\fB\-d, --ib-dev <dev>
+Use IB device <dev> (default first device found)
+.TP
+\fB\-i, --ib-port <port number>
+Use port <port number> of IB device (default 1)
+.TP
+\fB\-c, --connection <RC|UD>
+ Connection type RC or UD (default RC)
+.TP
+\fB\-m, --mtu <mtu>s
+Mtu size : 256 - 4096 (default port mtu)
+.TP
+\fB\-s, --size <size>
+Size of message to exchange (default 2)
+.TP
+\fB\-a, --all
+Run sizes from 2 till 2^23
+.TP
+\fB\-n, --iters <iterations>
+Number of exchanges (at least 5, default 1000)
+.TP
+\fB\-t, --tx-depth <depth>
+Size of tx queue (default 50)
+.TP
+\fB\-u, --qp-timeout <timeout>
+QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14
+.TP
+\fB\-S, --sl <sl>
+SL (default 0)
+.TP
+\fB\-x, --gid-index <index>
+Test uses GID with GID index (Default : IB - no gid)
+.TP
+\fB\-F, --CPU-freq
+Do not fail even if cpufreq_ondemand module is loaded
+.TP
+\fB\-V, --version
+Display version number
+.TP
+\fB\-I, --inline_size <size>
+Max size of message to be sent in inline (default 0)
+.TP
+\fB\-b, --bidirectional
+Measure bidirectional bandwidth (default unidirectional)
+.TP
+\fB\-N, --no peak-bw
+Cancel peak-bw calculation (default with peak)
+.TP
+\fB\-q, --qp <num>
+Specify <num> as number of QPs to use (default 1).
+.TP
+\fB\-C, --report-cycles
+report times in cpu cycle units (default microseconds)
+.TP
+\fB\-H, --report-histogram
+Print out all results (default print summary only)
+.TP
+\fB\-U, --report-unsorted
+(implies -H) print out unsorted results (default sorted)
+.PP
+.SH SEE ALSO
+ib_send_bw(1), ib_read_bw(1), rdma_lat(1), rdma_bw(1), ib_read_lat(1), ib_send_lat(1), ib_clock_test(1)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/open-fabrics/perftest/manpages/rdma_bw.1 Wed Feb 29 22:39:04 2012 +0000
@@ -0,0 +1,96 @@
+'\" t
+.\" Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+.\" Modified for Solaris to add the Solaris stability classification,
+.\" and to add a note about source availability.
+.\"
+.TH RDMA_BW/RDMA_LAT 1 " Oct 15, 2011"
+.SH "NAME"
+rdma_bw, rdma_lat - Bandwith and latency test using RDMA write transactions.
+.PP
+.SH SYNOPSIS
+.HP
+.nf
+rdma_bw [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-s, --size=<size>]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-S, --sl=<sl>] [-b, --bidirectional] [-c, --cma]
+ [server]
+.fi
+.HP
+
+.nf
+rdma_lat [-p, --port=<port>] [-d, --ib-dev=<dev>]
+ [-i, --ib-port=<port number.] [-s, --size=<size>]
+ [-n, --iters=<iterations>] [-t, --tx-depth=<depth>]
+ [-S, --sl=<sl>] [--I, --inline_size=<size>]
+ [-C, --report-cycles] [-H, --report-histogram]
+ [-U, --report-unsorted] [-c, --cma] [server]
+.fi
+
+
+.SH DESCRIPTION
+.PP
+.Nm
+The rdma_bw and rdma_lat tests are part of the "perftest" utilities that are used for InfiniBand related performance testing. The "perftest" utilities are micro bench mark utilities.
+
+.PP
+First a passive receiving instance is started.
+.RS 12
+
+ #rdma_bw
+.RE
+.PP
+Then an active sending instance is started, giving it the address at which it will find a listening passive receiver.
+.PP
+.RS 12
+ #rdma_bw server
+.RE
+.PP
+If options are used then the same options must be used for both client and server.
+.PP
+.SH OPTIONS
+.PP
+The following options are available for use on the command line:
+
+.TP 7
+\fB\-p, --port <port>
+Listen on/connect to port <port> (default 18515)
+.TP
+\fB\-d, --ib-dev <dev>
+Use IB device <dev> (default first device found)
+.TP
+\fB\-i, --ib-port <port number>
+Use port <port number> of IB device (default 1)
+.TP
+\fB\-s, --size <size>
+Size of message to exchange (default 2)
+.TP
+\fB\-n, --iters <iterations>
+Number of exchanges (at least 5, default 1000)
+.TP
+\fB\-t, --tx-depth <depth>
+Size of tx queue (default 50)
+.TP
+\fB\-S, --sl <sl>
+SL (default 0)
+.TP
+\fB\-I, --inline_size <size>
+Max size of message to be sent in inline (default 0)
+.TP
+\fB\-b, --bidirectional
+Measure bidirectional bandwidth (default unidirectional)
+.TP
+\fB\-c, --cma
+use RDMA CM
+.TP
+\fB\-C, --report-cycles
+report times in cpu cycle units (default microseconds)
+.TP
+\fB\-H, --report-histogram
+Print out all results (default print summary only)
+.TP
+\fB\-U, --report-unsorted
+(implies -H) print out unsorted results (default sorted)
+.PP
+.SH SEE ALSO
+ib_write_bw(1), ib_write_lat(1), ib_send_lat(1), ib_read_bw(1), ib_send_bw(1), ib_read_lat(1), ib_clock_test(1), ib_write_bw_post_list(1)
--- a/components/open-fabrics/perftest/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/perftest/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -1,7 +1,7 @@
diff -r -u /tmp/perftest-1.3.0/Makefile perftest-1.3.0/Makefile
--- /tmp/perftest-1.3.0/Makefile Thu Jan 20 01:37:35 2011
+++ perftest-1.3.0/Makefile Fri Feb 11 04:12:45 2011
-@@ -2,6 +2,7 @@
+@@ -2,10 +2,11 @@
MCAST_TESTS = send_bw send_lat
TESTS = write_bw_postlist write_lat write_bw read_lat read_bw
UTILS = clock_test
@@ -9,6 +9,11 @@
all: ${RDMACM_TESTS} ${MCAST_TESTS} ${TESTS} ${UTILS}
+-CFLAGS += -Wall -g -D_GNU_SOURCE -O2
++CFLAGS += -Wall -g -D_GNU_SOURCE -O3
+ BASIC_FILES = get_clock.c
+ EXTRA_FILES = perftest_resources.c
+ MCAST_FILES = multicast_resources.c
@@ -12,13 +13,14 @@
BASIC_HEADERS = get_clock.h
EXTRA_HEADERS = perftest_resources.h
@@ -38,47 +43,129 @@
clean:
$(foreach fname,${RDMACM_TESTS}, rm -f ${fname})
$(foreach fname,${MCAST_TESTS}, rm -f ib_${fname})
+diff -r -u /tmp/perftest-1.3.0/clock_test.c perftest-1.3.0/clock_test.c
+--- /tmp/perftest-1.3.0/clock_test.c Sun Nov 1 03:09:16 2009
++++ perftest-1.3.0/clock_test.c Fri Sep 30 08:08:29 2011
+@@ -20,6 +20,10 @@
+ c1 = get_cycles();
+ sleep(1);
+ c2 = get_cycles();
++#if defined(__SVR4) && defined(__sun)
++ printf("1 sec = %g usec\n", (double)((c2 - c1))/1000);
++#else
+ printf("1 sec = %g usec\n", (c2 - c1) / mhz);
++#endif
+ }
+ }
diff -r -u /tmp/perftest-1.3.0/get_clock.c perftest-1.3.0/get_clock.c
--- /tmp/perftest-1.3.0/get_clock.c Sun Dec 19 06:36:26 2010
+++ perftest-1.3.0/get_clock.c Fri Feb 11 04:12:46 2011
-@@ -45,6 +45,9 @@
+@@ -45,6 +45,10 @@
#include <unistd.h>
#include <stdio.h>
#include "get_clock.h"
+#if defined(__SVR4) && defined(__sun)
++#include <stdlib.h>
+#include <infiniband/ofa_solaris.h>
+#endif
#ifndef DEBUG
#define DEBUG 0
-@@ -137,7 +140,11 @@
+@@ -56,6 +60,8 @@
+ #define MEASUREMENTS 200
+ #define USECSTEP 10
+ #define USECSTART 100
++#define NSECSTART 100000
++#define NSECSTEP 10000
+
+ /*
+ Use linear regression to calculate cycles per microsecond.
+@@ -63,14 +69,14 @@
+ */
+ static double sample_get_cpu_mhz(void)
+ {
+- struct timeval tv1, tv2;
++ cycles_t tv1, tv2;
+ cycles_t start;
+ double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
+- double tx, ty;
++ cycles_t tx, ty;
+ int i;
+
+ /* Regression: y = a + b x */
+- long x[MEASUREMENTS];
++ cycles_t x[MEASUREMENTS];
+ cycles_t y[MEASUREMENTS];
+ double a; /* system call overhead in cycles */
+ double b; /* cycles per microsecond */
+@@ -78,25 +84,16 @@
+
+ for (i = 0; i < MEASUREMENTS; ++i) {
+ start = get_cycles();
++ tv1 = get_cycles();
+
+- if (gettimeofday(&tv1, NULL)) {
+- fprintf(stderr, "gettimeofday failed.\n");
+- return 0;
+- }
+-
+ do {
+- if (gettimeofday(&tv2, NULL)) {
+- fprintf(stderr, "gettimeofday failed.\n");
+- return 0;
+- }
+- } while ((tv2.tv_sec - tv1.tv_sec) * 1000000 +
+- (tv2.tv_usec - tv1.tv_usec) < USECSTART + i * USECSTEP);
++ tv2 = get_cycles();
++ } while ((tv2 - tv1) < NSECSTART + i * NSECSTEP);
+
+- x[i] = (tv2.tv_sec - tv1.tv_sec) * 1000000 +
+- tv2.tv_usec - tv1.tv_usec;
++ x[i] = (tv2 - tv1);
+ y[i] = get_cycles() - start;
+ if (DEBUG_DATA)
+- fprintf(stderr, "x=%ld y=%Ld\n", x[i], (long long)y[i]);
++ fprintf(stderr, "x=%lld y=%lld\n", x[i], y[i]);
+ }
+
+ for (i = 0; i < MEASUREMENTS; ++i) {
+@@ -134,10 +131,15 @@
+
+ static double proc_get_cpu_mhz(int no_cpu_freq_fail)
+ {
++#if !(defined(__SVR4) && defined(__sun))
FILE* f;
char buf[256];
++#else
++ sol_cpu_info_t *info;
++#endif
double mhz = 0.0;
-+#if defined(__SVR4) && defined(__sun)
-+ sol_cpu_info_t info;
-+#endif
+#if !(defined(__SVR4) && defined(__sun))
f = fopen("/proc/cpuinfo","r");
if (!f)
return 0.0;
-@@ -174,6 +181,12 @@
+@@ -174,6 +176,13 @@
}
}
fclose(f);
+#else
-+ if (!sol_get_cpu_info(&info))
-+ mhz = info.cpu_mhz;
++ if (sol_get_cpu_info(&info) > 0)
++ mhz = info[0].cpu_mhz;
+ else
+ return (0.0);
++ free(info);
+#endif
return mhz;
}
-@@ -184,9 +197,6 @@
+@@ -183,10 +192,10 @@
+ double sample, proc, delta;
sample = sample_get_cpu_mhz();
proc = proc_get_cpu_mhz(no_cpu_freq_fail);
++#if defined(__SVR4) && defined(__sun)
++ sample = sample * proc;
++#endif
- if (!proc || !sample)
- return 0;
@@ -89,30 +176,26 @@
diff -r -u /tmp/perftest-1.3.0/get_clock.h perftest-1.3.0/get_clock.h
--- /tmp/perftest-1.3.0/get_clock.h Sun Nov 1 03:09:16 2009
+++ perftest-1.3.0/get_clock.h Fri Feb 11 04:12:46 2011
-@@ -71,6 +71,23 @@
- return ret;
- }
+@@ -36,8 +36,18 @@
-+#elif defined(__sparc)
+ #ifndef GET_CLOCK_H
+ #define GET_CLOCK_H
++#if defined(__SVR4) && defined(__sun)
+#include <sys/times.h>
+#include <limits.h>
-+typedef long cycles_t;
-+
++#include <sys/time.h>
++typedef hrtime_t cycles_t;
+
+-#if defined (__x86_64__) || defined(__i386__)
+static inline cycles_t get_cycles()
+{
-+ struct timespec tp;
-+ long time;
-+
-+ if (clock_gettime(CLOCK_HIGHRES, &tp))
-+ return -1;
-+
-+ time = (tp.tv_sec * 1000000) + (tp.tv_nsec / 1000);
-+ return time;
++ return (gethrtime());
+}
+
- #else
- #warning get_cycles not implemented for this architecture: attempt asm/timex.h
- #include <asm/timex.h>
++#elif defined (__x86_64__) || defined(__i386__)
+ /* Note: only x86 CPUs which have rdtsc instruction are supported. */
+ typedef unsigned long long cycles_t;
+ static inline cycles_t get_cycles()
diff -r -u /tmp/perftest-1.3.0/rdma_bw.c perftest-1.3.0/rdma_bw.c
--- /tmp/perftest-1.3.0/rdma_bw.c Wed Apr 7 09:44:56 2010
+++ perftest-1.3.0/rdma_bw.c Fri Feb 11 04:12:46 2011
@@ -241,7 +324,7 @@
goto err;
}
-@@ -310,7 +313,7 @@
+@@ -310,13 +313,13 @@
if (!data->rem_dest)
goto err;
@@ -250,6 +333,13 @@
&data->rem_dest->qpn, &data->rem_dest->psn,
&data->rem_dest->rkey, &data->rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "%d:%s: Couldn't parse line <%.*s>\n",
+- pid, __func__, (int)sizeof msg, msg);
++ (int)pid, __func__, (int)sizeof msg, msg);
+ free(data->rem_dest);
+ goto err;
+ }
@@ -347,7 +350,7 @@
goto err5;
@@ -597,7 +687,16 @@
rdma_ack_cm_event(event);
rdma_destroy_id(data.cm_id);
rdma_destroy_event_channel(data.cm_channel);
-@@ -909,17 +912,17 @@
+@@ -904,22 +907,26 @@
+ }
+ }
+
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(0) * 1000000;
++#else
++ cycles_to_units = 1000000000;
++#endif
+
tsize = duplex ? 2 : 1;
tsize = tsize * size;
@@ -619,7 +718,7 @@
(unsigned long)(tcompleted[iters - 1] - tposted[0]) *
1024 / (tsize * iters));
}
-@@ -1046,7 +1049,7 @@
+@@ -1046,7 +1053,7 @@
pid = getpid();
printf("%d: | port=%d | ib_port=%d | size=%d | tx_depth=%d | sl=%d | iters=%d | duplex=%d | cma=%d |\n",
@@ -628,7 +727,7 @@
sl, iters, duplex, data.use_cma);
/* Done with parameter parsing. Perform setup. */
-@@ -1059,12 +1062,12 @@
+@@ -1059,12 +1066,12 @@
data.cm_channel = rdma_create_event_channel();
if (!data.cm_channel) {
fprintf(stderr, "%d:%s: rdma_create_event_channel failed\n",
@@ -643,7 +742,17 @@
return 1;
}
-@@ -1084,7 +1087,7 @@
+@@ -1079,12 +1086,17 @@
+ }
+ } else {
+ dev_list = ibv_get_device_list(NULL);
++ if (!dev_list) {
++ fprintf(stderr, "%d:%s: No IB devices found\n",
++ (int)pid, __func__);
++ return 1;
++ }
+
+ if (!ib_devname) {
data.ib_dev = dev_list[0];
if (!data.ib_dev) {
fprintf(stderr, "%d:%s: No IB devices found\n",
@@ -652,7 +761,7 @@
return 1;
}
} else {
-@@ -1093,7 +1096,7 @@
+@@ -1093,7 +1105,7 @@
break;
if (!data.ib_dev) {
fprintf(stderr, "%d:%s: IB device %s not found\n",
@@ -661,7 +770,7 @@
return 1;
}
}
-@@ -1109,7 +1112,7 @@
+@@ -1109,7 +1121,7 @@
data.my_dest.lid = pp_get_local_lid(ctx, data.ib_port);
if (!data.my_dest.lid) {
fprintf(stderr, "%d:%s: Local lid 0x0 detected. Is an SM running?\n",
@@ -670,7 +779,7 @@
return 1;
}
data.my_dest.qpn = ctx->qp->qp_num;
-@@ -1129,12 +1132,12 @@
+@@ -1129,12 +1141,12 @@
}
printf("%d: Local address: LID %#04x, QPN %#06x, PSN %#06x "
@@ -685,7 +794,7 @@
data.rem_dest->lid, data.rem_dest->qpn, data.rem_dest->psn,
data.rem_dest->rkey, data.rem_dest->vaddr);
-@@ -1219,7 +1222,7 @@
+@@ -1219,7 +1231,7 @@
if (ibv_post_send(qp, &ctx->wr, &bad_wr)) {
fprintf(stderr, "%d:%s: Couldn't post send: scnt=%d\n",
@@ -694,7 +803,7 @@
return 1;
}
++scnt;
-@@ -1235,17 +1238,17 @@
+@@ -1235,17 +1247,17 @@
tcompleted[ccnt] = get_cycles();
if (ne < 0) {
@@ -731,19 +840,19 @@
#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>
-@@ -59,7 +62,11 @@
- #include "get_clock.h"
+@@ -138,6 +141,11 @@
+ struct ibv_device *ib_dev = NULL;
- #define PINGPONG_RDMA_WRID 3
-+#if defined(__SVR4) && defined(__sun)
-+#define MAX_INLINE 372
-+#else
- #define MAX_INLINE 400
-+#endif
+ dev_list = ibv_get_device_list(NULL);
++ if (!dev_list) {
++ fprintf(stderr, "%d:%s: No IB devices found\n",
++ (int)pid, __func__);
++ return NULL;
++ }
- static int inline_size = MAX_INLINE;
- static int sl = 0;
-@@ -155,7 +162,8 @@
+ if (!ib_devname) {
+ ib_dev = dev_list[0];
+@@ -155,7 +163,8 @@
}
#define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000")
@@ -753,7 +862,7 @@
static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)
{
-@@ -185,7 +193,7 @@
+@@ -185,7 +194,7 @@
return -1;
}
@@ -762,7 +871,7 @@
&rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
if (parsed != 5) {
-@@ -220,7 +228,7 @@
+@@ -220,7 +229,7 @@
if (n < 0) {
fprintf(stderr, "%d:%s: %s for %s:%d\n",
@@ -771,7 +880,7 @@
data->servername, data->port);
goto err4;
}
-@@ -233,7 +241,7 @@
+@@ -233,7 +242,7 @@
if (rdma_resolve_addr(data->cm_id, NULL,
(struct sockaddr *)&sin, 2000)) {
fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n",
@@ -780,7 +889,7 @@
goto err2;
}
-@@ -248,7 +256,7 @@
+@@ -248,7 +257,7 @@
if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
fprintf(stderr, "%d:%s: unexpected CM event %d\n",
@@ -789,7 +898,7 @@
goto err1;
}
rdma_ack_cm_event(event);
-@@ -256,7 +264,7 @@
+@@ -256,7 +265,7 @@
retry_route:
if (rdma_resolve_route(data->cm_id, 2000)) {
fprintf(stderr, "%d:%s: rdma_resolve_route failed\n",
@@ -798,7 +907,7 @@
goto err2;
}
-@@ -271,7 +279,7 @@
+@@ -271,7 +280,7 @@
if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
fprintf(stderr, "%d:%s: unexpected CM event %d\n",
@@ -807,7 +916,7 @@
rdma_ack_cm_event(event);
goto err1;
}
-@@ -278,7 +286,7 @@
+@@ -278,7 +287,7 @@
rdma_ack_cm_event(event);
ctx = pp_init_ctx(data->cm_id, data);
if (!ctx) {
@@ -816,7 +925,7 @@
goto err2;
}
data->my_dest.psn = lrand48() & 0xffffff;
-@@ -294,7 +302,7 @@
+@@ -294,7 +303,7 @@
conn_param.private_data_len = sizeof(data->my_dest);
if (rdma_connect(data->cm_id, &conn_param)) {
@@ -825,7 +934,7 @@
goto err2;
}
-@@ -303,13 +311,13 @@
+@@ -303,13 +312,13 @@
if (event->event != RDMA_CM_EVENT_ESTABLISHED) {
fprintf(stderr, "%d:%s: unexpected CM event %d\n",
@@ -841,7 +950,7 @@
event->param.conn.private_data_len);
goto err1;
}
-@@ -332,7 +340,7 @@
+@@ -332,7 +341,7 @@
}
if (sockfd < 0) {
fprintf(stderr, "%d:%s: Couldn't connect to %s:%d\n",
@@ -850,7 +959,7 @@
goto err3;
}
ctx = pp_init_ctx(data->ib_dev, data);
-@@ -393,7 +401,7 @@
+@@ -393,7 +402,7 @@
goto err5;
if ( (n = getaddrinfo(NULL, service, &hints, &res)) < 0 ) {
@@ -859,7 +968,7 @@
gai_strerror(n), data->port);
goto err5;
}
-@@ -403,12 +411,12 @@
+@@ -403,12 +412,12 @@
sin.sin_family = AF_INET;
sin.sin_port = htons(data->port);
if (rdma_bind_addr(data->cm_id, (struct sockaddr *)&sin)) {
@@ -874,7 +983,7 @@
goto err3;
}
-@@ -417,13 +425,13 @@
+@@ -417,13 +426,13 @@
if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
fprintf(stderr, "%d:%s: bad event waiting for connect request %d\n",
@@ -890,7 +999,7 @@
__func__, event->param.conn.private_data_len);
goto err2;
}
-@@ -451,18 +459,18 @@
+@@ -451,18 +460,18 @@
conn_param.private_data = &data->my_dest;
conn_param.private_data_len = sizeof(data->my_dest);
if (rdma_accept(child_cm_id, &conn_param)) {
@@ -912,7 +1021,7 @@
goto err1;
}
rdma_ack_cm_event(event);
-@@ -482,7 +490,7 @@
+@@ -482,7 +491,7 @@
}
if (sockfd < 0) {
@@ -921,7 +1030,7 @@
__func__, data->port);
goto err4;
}
-@@ -491,7 +499,7 @@
+@@ -491,7 +500,7 @@
connfd = accept(sockfd, NULL, 0);
if (connfd < 0) {
perror("server accept");
@@ -930,7 +1039,7 @@
close(sockfd);
goto err4;
}
-@@ -551,7 +559,7 @@
+@@ -551,7 +560,7 @@
ctx->buf = memalign(page_size, ctx->size * 2);
if (!ctx->buf) {
fprintf(stderr, "%d:%s: Couldn't allocate work buf.\n",
@@ -939,7 +1048,7 @@
return NULL;
}
-@@ -565,7 +573,7 @@
+@@ -565,7 +574,7 @@
cm_id = (struct rdma_cm_id *)ptr;
ctx->context = cm_id->verbs;
if (!ctx->context) {
@@ -948,7 +1057,7 @@
__func__);
return NULL;
}
-@@ -575,7 +583,7 @@
+@@ -575,7 +584,7 @@
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
fprintf(stderr, "%d:%s: Couldn't get context for %s\n",
@@ -957,7 +1066,7 @@
return NULL;
}
}
-@@ -582,7 +590,7 @@
+@@ -582,7 +591,7 @@
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
@@ -966,7 +1075,7 @@
return NULL;
}
-@@ -592,13 +600,13 @@
+@@ -592,13 +601,13 @@
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, ctx->size * 2,
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
@@ -982,7 +1091,7 @@
__func__);
return NULL;
}
-@@ -605,7 +613,7 @@
+@@ -605,7 +614,7 @@
ctx->scq = ibv_create_cq(ctx->context, ctx->tx_depth, ctx, NULL, 0);
if (!ctx->scq) {
@@ -991,7 +1100,7 @@
__func__);
return NULL;
}
-@@ -628,7 +636,7 @@
+@@ -628,7 +637,7 @@
if (data->use_cma) {
if (rdma_create_qp(cm_id, ctx->pd, &attr)) {
@@ -1000,7 +1109,7 @@
return NULL;
}
ctx->qp = cm_id->qp;
-@@ -636,7 +644,7 @@
+@@ -636,7 +645,7 @@
} else {
ctx->qp = ibv_create_qp(ctx->pd, &attr);
if (!ctx->qp) {
@@ -1009,7 +1118,7 @@
return NULL;
}
{
-@@ -653,7 +661,7 @@
+@@ -653,7 +662,7 @@
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "%d:%s: Failed to modify QP to INIT\n",
@@ -1018,7 +1127,7 @@
return NULL;
}
}
-@@ -710,10 +718,10 @@
+@@ -710,10 +719,10 @@
return 0;
}
@@ -1031,7 +1140,7 @@
/* Create connection between client and server.
* We do it by exchanging data over a TCP socket connection. */
-@@ -727,7 +735,7 @@
+@@ -727,7 +736,7 @@
data->my_dest.rkey = ctx->mr->rkey;
data->my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
@@ -1040,7 +1149,7 @@
data->my_dest.rkey, data->my_dest.vaddr);
if (data->servername) {
-@@ -738,7 +746,7 @@
+@@ -738,7 +747,7 @@
return 1;
}
@@ -1049,7 +1158,7 @@
data->rem_dest->psn, data->rem_dest->rkey,
data->rem_dest->vaddr);
-@@ -783,7 +791,7 @@
+@@ -783,7 +792,7 @@
rc = ibv_post_recv(ctx->qp, &wr, &bad_wr);
if (rc) {
perror("ibv_post_recv");
@@ -1058,7 +1167,7 @@
__func__, rc);
}
}
-@@ -799,13 +807,13 @@
+@@ -799,13 +808,13 @@
} while (ne == 0);
if (wc.status)
@@ -1075,7 +1184,7 @@
(int)wc.wr_id);
}
-@@ -825,7 +833,7 @@
+@@ -825,7 +834,7 @@
ctx->wr.send_flags = IBV_SEND_SIGNALED;
ctx->wr.next = NULL;
if (ibv_post_send(ctx->qp, &ctx->wr, &bad_wr)) {
@@ -1084,7 +1193,7 @@
return;
}
do {
-@@ -834,13 +842,13 @@
+@@ -834,13 +843,13 @@
} while (ne == 0);
if (wc.status)
@@ -1101,7 +1210,7 @@
(int)wc.wr_id);
}
-@@ -855,13 +863,13 @@
+@@ -855,13 +864,13 @@
} while (ne == 0);
if (wc.status)
@@ -1118,7 +1227,7 @@
(int)wc.wr_id);
pp_post_recv(ctx);
}
-@@ -882,7 +890,7 @@
+@@ -882,7 +891,7 @@
ctx->wr.send_flags = IBV_SEND_SIGNALED;
ctx->wr.next = NULL;
if (ibv_post_send(ctx->qp, &ctx->wr, &bad_wr)) {
@@ -1127,7 +1236,7 @@
return;
}
do {
-@@ -891,13 +899,13 @@
+@@ -891,13 +900,13 @@
} while (ne == 0);
if (wc.status)
@@ -1144,7 +1253,7 @@
(int)wc.wr_id);
}
-@@ -910,7 +918,7 @@
+@@ -910,7 +919,7 @@
rc = rdma_disconnect(data.cm_id);
if (rc) {
perror("rdma_disconnect");
@@ -1153,7 +1262,7 @@
__func__);
return;
}
-@@ -919,7 +927,7 @@
+@@ -919,7 +928,7 @@
rdma_get_cm_event(data.cm_channel, &event);
if (event->event != RDMA_CM_EVENT_DISCONNECTED)
fprintf(stderr, "%d:%s: unexpected event during disconnect %d\n",
@@ -1162,7 +1271,26 @@
rdma_ack_cm_event(event);
rdma_destroy_id(data.cm_id);
rdma_destroy_event_channel(data.cm_channel);
-@@ -1164,12 +1172,12 @@
+@@ -989,10 +998,18 @@
+
+
+ if (options->cycles) {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = 1;
++#else
++ cycles_to_units = (1/get_cpu_mhz(0)) * 1000;
++#endif
+ units = "cycles";
+ } else {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(0);
++#else
++ cycles_to_units = 1000;
++#endif
+ units = "usec";
+ }
+
+@@ -1164,12 +1181,12 @@
data.cm_channel = rdma_create_event_channel();
if (!data.cm_channel) {
fprintf(stderr, "%d:%s: rdma_create_event_channel failed\n",
@@ -1177,7 +1305,7 @@
return 1;
}
-@@ -1184,12 +1192,12 @@
+@@ -1184,12 +1201,12 @@
}
printf("%d: Local address: LID %#04x, QPN %#06x, PSN %#06x "
@@ -1192,6 +1320,31 @@
data.rem_dest->lid, data.rem_dest->qpn, data.rem_dest->psn,
data.rem_dest->rkey, data.rem_dest->vaddr);
+diff -r -u /tmp/perftest-1.3.0/multicast_resources.h perftest-1.3.0/multicast_resources.h
+--- /tmp/perftest-1.3.0/multicast_resources.h Wed Mar 2 11:01:36 2011
++++ perftest-1.3.0/multicast_resources.h Fri Aug 26 05:14:56 2011
+@@ -68,7 +68,7 @@
+ #define DEF_PKEY_IDX 0
+ #define DEF_SLL 0
+ #define MAX_POLL_ITERATION_TIMEOUT 1000000
+-#define MCG_GID {255,1,0,0,0,2,201,133,0,0,0,0,0,0,0,0}
++#define MCG_GID {255,21,0,0,0,2,201,133,0,0,0,0,0,0,0,0}
+
+ // Definitions section for MADs
+ #define SUBN_ADM_ATTR_MC_MEMBER_RECORD 0x38
+@@ -80,10 +80,11 @@
+ #define DEF_TCLASS 0
+ #define DEF_FLOW_LABLE 0
+
++#if !(defined(__SVR4) && defined(__sun))
+ // Macro for 64 bit variables to switch to from net
+ #define ntohll(x) (((u_int64_t)(ntohl((int)((x << 32) >> 32))) << 32) | (unsigned int)ntohl(((int)(x >> 32))))
+ #define htonll(x) ntohll(x)
+-
++#endif
+ // generate a bit mask S bits width
+ #define MASK32(S) ( ((u_int32_t) ~0L) >> (32-(S)) )
+
diff -r -u /tmp/perftest-1.3.0/perftest_resources.c perftest-1.3.0/perftest_resources.c
--- /tmp/perftest-1.3.0/perftest_resources.c Tue Jan 25 23:31:57 2011
+++ perftest-1.3.0/perftest_resources.c Fri Feb 11 04:12:48 2011
@@ -1228,34 +1381,46 @@
} else {
switch (params->mtu) {
+@@ -869,7 +885,6 @@
+ close(sockfd);
+ return connfd;
+ }
+-
+ close(sockfd);
+ return connfd;
+ }
+@@ -882,6 +897,8 @@
+ struct pingpong_dest *my_dest,
+ struct pingpong_dest *rem_dest) {
+
++ int temp_reads = 0;
++
+ // Client.
+ if (params->machine == CLIENT) {
+ if (ctx_write_keys(my_dest,params)) {
+@@ -904,6 +921,18 @@
+ return -1;
+ }
+ }
++ // We could have tavor at one end and hermon at the other.
++ // To avoid a modify QP error set max_rd_atomic to lowest
++ // on either side of connection.
++ if (rem_dest->out_reads > my_dest->out_reads)
++ temp_reads = my_dest->out_reads;
++
++ if (my_dest->out_reads > rem_dest->out_reads)
++ my_dest->out_reads = rem_dest->out_reads;
++
++ if (temp_reads)
++ rem_dest->out_reads = temp_reads;
++
+ return 0;
+ }
+
diff -r -u /tmp/perftest-1.3.0/perftest_resources.h perftest-1.3.0/perftest_resources.h
--- /tmp/perftest-1.3.0/perftest_resources.h Tue Jan 25 23:31:57 2011
+++ perftest-1.3.0/perftest_resources.h Fri Feb 11 04:12:48 2011
-@@ -78,7 +78,11 @@
- #define DEF_GID_INDEX (-1)
- #define DEF_NUM_QPS (1)
- #define DEF_INLINE_BW (0)
-+#if defined(__SVR4) && defined(__sun)
-+#define DEF_INLINE_LT (372)
-+#else
- #define DEF_INLINE_LT (400)
-+#endif
- #define DEF_RX_RDMA (1)
- #define DEF_RX_SEND (600)
-
-@@ -96,7 +100,11 @@
- #define MIN_QP_NUM (1)
- #define MAX_QP_NUM (8)
- #define MIN_INLINE (0)
-+#if defined(__SVR4) && defined(__sun)
-+#define MAX_INLINE (372)
-+#else
- #define MAX_INLINE (400)
-+#endif
- #define MIN_QP_MCAST (1)
- #define MAX_QP_MCAST (56)
- #define MIN_RX (1)
-@@ -128,10 +136,10 @@
+@@ -128,10 +128,10 @@
#define KEY_MSG_SIZE_GID 98 // Message size with gid (MGID as well).
// The Format of the message we pass through sockets , without passing Gid.
@@ -1268,7 +1433,7 @@
// The Basic print format for all verbs.
#define BASIC_ADDR_FMT " %s address: LID %#04x QPN %#06x PSN %#06x"
-@@ -140,7 +148,7 @@
+@@ -140,7 +140,7 @@
#define READ_FMT " OUT %#04x"
// The print format of the pingpong_dest element for RDMA verbs.
@@ -1277,7 +1442,7 @@
// The print format of a global address or a multicast address.
#define GID_FMT " %s: %02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d\n"
-@@ -154,10 +162,10 @@
+@@ -154,10 +154,10 @@
#define RESULT_FMT_LAT " #bytes #iterations t_min[usec] t_max[usec] t_typical[usec]\n"
// Result print format
@@ -1301,6 +1466,67 @@
#include <time.h>
#include <infiniband/verbs.h>
+@@ -336,7 +337,11 @@
+ }
+ }
+
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f) * 1000000;
++#else
++ cycles_to_units = 1000000000;
++#endif
+ tsize = user_param->duplex ? 2 : 1;
+ tsize = tsize * user_param->size;
+
+@@ -443,6 +448,7 @@
+ ******************************************************************************/
+ int main(int argc, char *argv[]) {
+
++ int ret = 0;
+ int i = 0;
+ struct ibv_device *ib_dev = NULL;
+ struct pingpong_context *ctx;
+@@ -553,8 +559,10 @@
+
+ for (i = 1; i < 24 ; ++i) {
+ user_param.size = 1 << i;
+- if(run_iter(ctx,&user_param,&rem_dest))
+- return 17;
++ if(run_iter(ctx,&user_param,&rem_dest)) {
++ ret = 17;
++ goto exit;
++ }
+ print_report(&user_param);
+ }
+
+@@ -562,11 +570,13 @@
+
+ else {
+
+- if(run_iter(ctx,&user_param,&rem_dest))
+- return 17;
+-
++ if(run_iter(ctx,&user_param,&rem_dest)) {
++ ret = 17;
++ goto exit;
++ }
+ print_report(&user_param);
+ }
++exit:
+
+ if (ctx_close_connection(&user_param,&my_dest,&rem_dest)) {
+ fprintf(stderr,"Failed to close connection between server and client\n");
+@@ -575,6 +585,9 @@
+
+ printf(RESULT_LINE);
+
+- return destroy_ctx_resources(ctx);
++ if (destroy_ctx_resources(ctx))
++ return 1;
++ else
++ return ret;
+
+ }
diff -r -u /tmp/perftest-1.3.0/read_lat.c perftest-1.3.0/read_lat.c
--- /tmp/perftest-1.3.0/read_lat.c Tue Jan 25 23:31:57 2011
+++ perftest-1.3.0/read_lat.c Fri Feb 11 04:12:47 2011
@@ -1312,10 +1538,31 @@
#include <infiniband/verbs.h>
#include "get_clock.h"
+@@ -358,11 +359,20 @@
+
+
+ if (user_param->r_flag->cycles) {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = 1;
++#else
++ cycles_to_units =
++ (1/get_cpu_mhz(user_param->cpu_freq_f)) * 1000;
++#endif
+ units = "cycles";
+ } else {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f);
++#else
++ cycles_to_units = 1000;
+ units = "usec";
++#endif
+ }
+
+ if (user_param->r_flag->unsorted) {
diff -r -u /tmp/perftest-1.3.0/send_bw.c perftest-1.3.0/send_bw.c
--- /tmp/perftest-1.3.0/send_bw.c Thu Jan 20 07:37:18 2011
+++ perftest-1.3.0/send_bw.c Fri Feb 11 04:12:47 2011
-@@ -1,1162 +1,1162 @@
+@@ -1,1162 +1,1166 @@
-/*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
@@ -2800,7 +3047,7 @@
+ user_parm->size = MTU_SIZE(user_parm->curr_mtu);
+ }
+
-+ if (is_dev_hermon(ctx->context) != NOT_HERMON && user_parm->inline_size != 0)
++ if (is_dev_hermon(ctx->context) == NOT_HERMON && user_parm->inline_size != 0)
+ user_parm->inline_size = 0;
+
+ printf(" Inline data is used up to %d bytes message\n", user_parm->inline_size);
@@ -3129,7 +3376,11 @@
+ }
+ }
+
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f) * 1000000;
++#else
++ cycles_to_units = 1000000000;
++#endif
+
+ tsize = user_param->duplex ? 2 : 1;
+ tsize = tsize * user_param->size;
@@ -3640,6 +3891,123 @@
+ printf(RESULT_LINE);
+ return destroy_ctx_resources(ctx,&user_param,&my_dest,&rem_dest,&mcg_params);
+}
+diff -r -u /tmp/perftest-1.3.0/send_lat.c perftest-1.3.0/send_lat.c
+--- /tmp/perftest-1.3.0/send_lat.c Wed Mar 2 16:04:50 2011
++++ perftest-1.3.0/send_lat.c Fri Aug 26 05:29:53 2011
+@@ -61,7 +61,8 @@
+ struct ibv_sge *sge_list;
+ struct ibv_recv_wr *rwr;
+ struct ibv_context *context;
+- struct ibv_comp_channel *channel;
++ struct ibv_comp_channel *rx_channel;
++ struct ibv_comp_channel *tx_channel;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *rcq;
+@@ -259,13 +260,20 @@
+ test_result = 1;
+ }
+
+- if (ctx->channel) {
+- if (ibv_destroy_comp_channel(ctx->channel)) {
+- fprintf(stderr, "failed to destroy channel \n");
++ if (ctx->rx_channel) {
++ if (ibv_destroy_comp_channel(ctx->rx_channel)) {
++ fprintf(stderr, "failed to destroy rx_channel \n");
+ test_result = 1;
+ }
+ }
+
++ if (ctx->tx_channel) {
++ if (ibv_destroy_comp_channel(ctx->tx_channel)) {
++ fprintf(stderr, "failed to destroy tx_channel \n");
++ test_result = 1;
++ }
++ }
++
+ if (ibv_close_device(ctx->context)) {
+ fprintf(stderr, "failed to close device context\n");
+ test_result = 1;
+@@ -328,13 +336,20 @@
+ memset(ctx->buf, 0,buff_size);
+
+ if (user_parm->use_event) {
+- ctx->channel = ibv_create_comp_channel(ctx->context);
+- if (!ctx->channel) {
+- fprintf(stderr, "Couldn't create completion channel\n");
++ ctx->rx_channel = ibv_create_comp_channel(ctx->context);
++ if (!ctx->rx_channel) {
++ fprintf(stderr, "Couldn't create completion rx_channel\n");
+ return NULL;
+ }
+- } else
+- ctx->channel = NULL;
++ ctx->tx_channel = ibv_create_comp_channel(ctx->context);
++ if (!ctx->rx_channel) {
++ fprintf(stderr, "Couldn't create completion tx_channel\n");
++ return NULL;
++ }
++ } else {
++ ctx->rx_channel = NULL;
++ ctx->tx_channel = NULL;
++ }
+
+ ctx->pd = ibv_alloc_pd(ctx->context);
+ if (!ctx->pd) {
+@@ -348,13 +363,13 @@
+ return NULL;
+ }
+
+- ctx->scq = ibv_create_cq(ctx->context,user_parm->tx_depth,NULL,ctx->channel,0);
++ ctx->scq = ibv_create_cq(ctx->context,user_parm->tx_depth,NULL,ctx->tx_channel,0);
+ if (!ctx->scq) {
+ fprintf(stderr, "Couldn't create CQ\n");
+ return NULL;
+ }
+
+- ctx->rcq = ibv_create_cq(ctx->context,user_parm->rx_depth*user_parm->num_of_qps,NULL,ctx->channel,0);
++ ctx->rcq = ibv_create_cq(ctx->context,user_parm->rx_depth*user_parm->num_of_qps,NULL,ctx->rx_channel,0);
+ if (!ctx->rcq) {
+ fprintf(stderr, "Couldn't create CQ\n");
+ return NULL;
+@@ -583,10 +598,19 @@
+
+
+ if (user_param->r_flag->cycles) {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = 1;
++#else
++ cycles_to_units =
++ (1/get_cpu_mhz(user_param->cpu_freq_f)) * 1000;
++#endif
+ units = "cycles";
+ } else {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f);
++#else
++ cycles_to_units = 1000;
++#endif
+ units = "usec";
+ }
+
+@@ -649,7 +673,7 @@
+
+ // Server is polling on recieve first .
+ if (user_param->use_event) {
+- if (ctx_notify_events(ctx->rcq,ctx->channel)) {
++ if (ctx_notify_events(ctx->rcq, ctx->rx_channel)) {
+ fprintf(stderr , " Failed to notify events to CQ");
+ return 1;
+ }
+@@ -701,7 +725,7 @@
+ int s_ne;
+
+ if (user_param->use_event) {
+- if (ctx_notify_events(ctx->scq,ctx->channel)) {
++ if (ctx_notify_events(ctx->scq, ctx->tx_channel)) {
+ fprintf(stderr , " Failed to notify events to CQ");
+ return 1;
+ }
diff -r -u /tmp/perftest-1.3.0/multicast_resources.c perftest-1.3.0/multicast_resources.c
--- /tmp/perftest-1.3.0/multicast_resources.c Thu Dec 16 08:21:05 2010
+++ perftest-1.3.0/multicast_resources.c Fri Feb 11 04:12:48 2011
@@ -3664,6 +4032,27 @@
#include <time.h>
#include <infiniband/verbs.h>
+@@ -224,7 +225,7 @@
+ return NULL;
+ }
+
+- if (is_dev_hermon(ctx->context) != NOT_HERMON && user_parm->inline_size != 0)
++ if (is_dev_hermon(ctx->context) == NOT_HERMON && user_parm->inline_size != 0)
+ user_parm->inline_size = 0;
+
+ printf(" Inline data is used up to %d bytes message\n", user_parm->inline_size);
+@@ -384,7 +385,11 @@
+ }
+ }
+
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f) * 1000000;
++#else
++ cycles_to_units = 1000000000;
++#endif
+
+ tsize = user_param->duplex ? 2 : 1;
+ tsize = tsize * user_param->size;
diff -r -u /tmp/perftest-1.3.0/write_bw_postlist.c perftest-1.3.0/write_bw_postlist.c
--- /tmp/perftest-1.3.0/write_bw_postlist.c Thu Mar 3 17:03:54 2011
+++ perftest-1.3.0/write_bw_postlist.c Tue Mar 15 11:59:53 2011
@@ -3675,6 +4064,18 @@
#include <infiniband/verbs.h>
#include "get_clock.h"
+@@ -323,7 +324,11 @@
+ }
+ }
+
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f) * 1000000;
++#else
++ cycles_to_units = 1000000000;
++#endif
+
+ tsize = user_param->duplex ? 2 : 1;
+ tsize = tsize * user_param->size;
diff -r -u /tmp/perftest-1.3.0/write_lat.c perftest-1.3.0/write_lat.c
--- /tmp/perftest-1.3.0/write_lat.c Sat Feb 26 01:02:48 2011
+++ perftest-1.3.0/write_lat.c Tue Mar 15 12:01:35 2011
@@ -3686,3 +4087,23 @@
#include <infiniband/verbs.h>
#include "get_clock.h"
+@@ -330,10 +331,19 @@
+
+
+ if (user_param->r_flag->cycles) {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = 1;
++#else
++ cycles_to_units =
++ (1/get_cpu_mhz(user_param->cpu_freq_f)) * 1000;
++#endif
+ units = "cycles";
+ } else {
++#if !(defined(__SVR4) && defined(__sun))
+ cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f);
++#else
++ cycles_to_units = 1000;
++#endif
+ units = "usec";
+ }
+
--- a/components/open-fabrics/qperf/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/qperf/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -31,27 +31,25 @@
diff -r -u /tmp/730054/qperf-0.4.6/src/qperf.c qperf-0.4.6/src/qperf.c
--- /tmp/730054/qperf-0.4.6/src/qperf.c Mon Aug 31 00:00:40 2009
+++ qperf-0.4.6/src/qperf.c Mon Nov 8 11:10:17 2010
-@@ -53,7 +53,19 @@
+@@ -53,7 +53,17 @@
#include <sys/utsname.h>
#include "qperf.h"
+#if (defined(__SVR4) && defined(__sun))
+#include <sys/filio.h>
+#include <limits.h> /* times() */
-+#include <kstat.h> /* libkstat */
+#include <sys/types.h>
+#include <sys/processor.h>
+#include <sys/procset.h>
+#include <strings.h>
+#include <infiniband/ofa_solaris.h>
-+static kstat_ctl_t *kc = NULL; /* libkstat cookie */
+#endif
+
/*
* Configurable parameters. If your change makes this version of qperf
* incompatible with previous versions (usually a change to the Req structure),
-@@ -224,7 +236,9 @@
+@@ -224,7 +234,9 @@
static void sig_alrm(int signo, siginfo_t *siginfo, void *ucontext);
static void sig_quit(int signo, siginfo_t *siginfo, void *ucontext);
static void sig_urg(int signo, siginfo_t *siginfo, void *ucontext);
@@ -62,7 +60,7 @@
static void start_test_timer(int seconds);
static long str_size(char *arg, char *str);
static void strncopy(char *d, char *s, int n);
-@@ -257,7 +271,9 @@
+@@ -257,7 +269,9 @@
static STAT IStat;
static int ListenFD;
static LOOP *Loops;
@@ -72,27 +70,22 @@
static STAT RStat;
static int ShowIndex;
static SHOW ShowTable[256];
-@@ -601,6 +617,9 @@
+@@ -601,6 +615,9 @@
initialize();
set_signals();
do_args(&argv[1]);
+#if (defined(__SVR4) && defined(__sun))
-+ (void) kstat_close(kc);
++ solaris_fini();
+#endif
return 0;
}
-@@ -617,14 +636,24 @@
+@@ -617,14 +634,19 @@
for (i = 0; i < P_N; ++i)
if (ParInfo[i].index != i)
error(BUG, "initialize: ParInfo: out of order: %d", i);
+#if (defined(__SVR4) && defined(__sun))
-+ while ((kc = kstat_open()) == NULL) {
-+ if (errno == EAGAIN)
-+ (void) poll(NULL, 0, 200);
-+ else
-+ error(SYS, "cannot open /dev/kstat");
-+ }
++ solaris_init();
+#else
ProcStatFD = open("/proc/stat", 0);
if (ProcStatFD < 0)
@@ -107,7 +100,7 @@
/*
* Look for a colon and skip past it and any spaces.
*/
-@@ -643,6 +672,7 @@
+@@ -643,6 +665,7 @@
s++;
return s;
}
@@ -115,7 +108,7 @@
/*
-@@ -1667,13 +1697,18 @@
+@@ -1667,13 +1690,18 @@
{
char count[STRSIZE];
char speed[STRSIZE];
@@ -129,14 +122,14 @@
int mixed = 0;
FILE *fp = fopen("/proc/cpuinfo", "r");
+#else
-+ sol_cpu_info_t info;
++ sol_cpu_info_t *info;
+#endif
+#if !(defined(__SVR4) && defined(__sun))
if (!fp)
error(0, "cannot open /proc/cpuinfo");
cpu[0] = '\0';
-@@ -1732,6 +1767,7 @@
+@@ -1732,6 +1760,7 @@
/* CPU speed */
speed[0] = '\0';
@@ -144,7 +137,7 @@
if (!mixed) {
int n = strlen(cpu);
if (n < 3 || cpu[n-2] != 'H' || cpu[n-1] != 'z') {
-@@ -1745,7 +1781,23 @@
+@@ -1745,7 +1774,24 @@
}
}
}
@@ -152,14 +145,15 @@
+ cpu[0] = '\0';
+ speed[0] = '\0';
-+ if (!sol_get_cpu_info(&info)) {
-+ (void) strcpy(cpu, info.cpu_name);
-+ cpus = info.cpu_num;
++ if ((cpus = sol_get_cpu_info(&info)) > 0 ) {
++ (void) strcpy(cpu, info[0].cpu_name);
+
-+ if (info.cpu_mhz < 1000)
-+ snprintf(speed, sizeof(speed), " %dMHz", info.cpu_mhz);
++ if (info[0].cpu_mhz < 1000)
++ snprintf(speed, sizeof(speed), " %dMHz", info[0].cpu_mhz);
+ else
-+ snprintf(speed, sizeof(speed), " %.1fGHz", info.cpu_mhz/1000.0);
++ snprintf(speed, sizeof(speed), " %.1fGHz",
++ info[0].cpu_mhz/1000.0);
++ free(info);
+ } else {
+ return;
+ }
@@ -168,7 +162,7 @@
/* Number of CPUs */
if (cpus == 1)
count[0] = '\0';
-@@ -2615,14 +2667,22 @@
+@@ -2615,9 +2661,15 @@
static void
set_affinity(void)
{
@@ -177,21 +171,22 @@
+#endif
int a = Req.affinity;
++#if defined(__SVR4) && defined(__sun)
++ if (processor_bind(P_LWPID, P_MYID, a, NULL) != 0)
++ error(SYS, "cannot set processor affinity (cpu %d)", a);
++#else
if (!a)
return;
-+
-+#if defined(__SVR4) && defined(__sun)
-+ if (processor_bind(P_LWPID, P_MYID, a-1, NULL) != 0)
-+#else
-+
CPU_ZERO(&set);
+@@ -2624,6 +2676,7 @@
CPU_SET(a-1, &set);
if (sched_setaffinity(0, sizeof(set), &set) < 0)
+ error(SYS, "cannot set processor affinity (cpu %d)", a-1);
+#endif
- error(SYS, "cannot set processor affinity (cpu %d)", a-1);
}
-@@ -2771,9 +2831,74 @@
+
+@@ -2771,9 +2824,36 @@
/*
* Get various temporal parameters.
*/
@@ -203,57 +198,19 @@
get_times(CLOCK timex[T_N])
{
+ struct tms tms;
-+ size_t i, nr_cpus;
-+ kstat_t *ksp;
-+ kstat_named_t *knp;
-+ uint64_t t_user = 0, t_kernel = 0, t_idle = 0;
-+ uint64_t t_irq = 0, t_iowait = 0;
-+
++ sol_cpu_stats_t stats;
++
+ timex[T_REAL] = times(&tms);
+
-+ nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
-+
-+ /* Aggregate the value of all CPUs */
-+ for (i = 0; i < nr_cpus; i++) {
-+ ksp = kstat_lookup(kc, "cpu", i, "sys");
-+ if (ksp == NULL)
-+ return;
-+
-+ if (kstat_read(kc, ksp, NULL) == -1)
-+ return;
-+
-+ knp = (kstat_named_t *)kstat_data_lookup(ksp, "cpu_ticks_user");
-+ if (knp == NULL)
-+ return;
-+ t_user += knp->value.ui64;
-+
-+ knp = (kstat_named_t *)kstat_data_lookup(ksp,
-+ "cpu_ticks_kernel");
-+ if (knp == NULL)
-+ return;
-+ t_kernel += knp->value.ui64;
++ if ((sol_get_cpu_stats(&stats)) < 0)
++ return;
+
-+ knp = (kstat_named_t *)kstat_data_lookup(ksp, "cpu_ticks_idle");
-+ if (knp == NULL)
-+ return;
-+ t_idle += knp->value.ui64;
-+
-+ knp = (kstat_named_t *)kstat_data_lookup(ksp, "cpu_ticks_wait");
-+ if (knp == NULL)
-+ return;
-+ t_iowait += knp->value.ui64;
-+
-+ knp = (kstat_named_t *)kstat_data_lookup(ksp, "cpu_nsec_intr");
-+ if (knp == NULL)
-+ return;
-+ t_irq += knp->value.ui64; /* This is in NSEC */
-+ }
-+ timex[T_USER] = t_user;
++ timex[T_USER] = stats.t_user;
+ timex[T_NICE] = 0;
-+ timex[T_KERNEL] = t_kernel;
-+ timex[T_IDLE] = t_idle;
-+ timex[T_IOWAIT] = t_iowait;
-+ timex[T_IRQ] = NSEC_TO_TICK(t_irq); /* NSEC_TO_TICK */
++ timex[T_KERNEL] = stats.t_kernel;
++ timex[T_IDLE] = stats.t_idle;
++ timex[T_IOWAIT] = stats.t_iowait;
++ timex[T_IRQ] = NSEC_TO_TICK(stats.t_intr); /* NSEC_TO_TICK */
+ timex[T_SOFTIRQ] = 0;
+ timex[T_STEAL] = 0;
+}
@@ -266,7 +223,7 @@
int n;
char *p;
char buf[BUFSIZE];
-@@ -2802,6 +2927,7 @@
+@@ -2802,6 +2882,7 @@
while (n < T_N)
timex[n++] = 0;
}
@@ -289,7 +246,17 @@
#include <unistd.h>
#include <netinet/in.h>
#include <rdma/rdma_cma.h>
-@@ -566,11 +570,15 @@
+@@ -110,7 +114,8 @@
+ uint32_t srqn; /* Shared queue number */
+ uint32_t rkey; /* Remote key */
+ uint32_t alt_lid; /* Alternate Path Local LID */
+- uint32_t rd_atomic; /* Number of read/atomics supported */
++ uint32_t rd_atomic /* Number of read/atomics supported */
++ __attribute__ ((aligned (8))); /* align struct size to 64-bit binary */
+ } NODE;
+
+
+@@ -566,11 +571,15 @@
void
run_client_uc_bi_bw(void)
{
@@ -305,7 +272,7 @@
}
-@@ -580,7 +588,11 @@
+@@ -580,7 +589,11 @@
void
run_server_uc_bi_bw(void)
{
@@ -317,7 +284,7 @@
}
-@@ -590,6 +602,9 @@
+@@ -590,6 +603,9 @@
void
run_client_uc_bw(void)
{
@@ -327,7 +294,7 @@
par_use(L_ACCESS_RECV);
par_use(R_ACCESS_RECV);
par_use(L_NO_MSGS);
-@@ -597,6 +612,7 @@
+@@ -597,6 +613,7 @@
rd_params(IBV_QPT_UC, K64, 1, 0);
rd_client_bw(IBV_QPT_UC);
show_results(BANDWIDTH_SR);
@@ -335,7 +302,7 @@
}
-@@ -606,7 +622,11 @@
+@@ -606,7 +623,11 @@
void
run_server_uc_bw(void)
{
@@ -347,7 +314,7 @@
}
-@@ -616,8 +636,12 @@
+@@ -616,8 +637,12 @@
void
run_client_uc_lat(void)
{
@@ -360,7 +327,7 @@
}
-@@ -627,7 +651,11 @@
+@@ -627,7 +652,11 @@
void
run_server_uc_lat(void)
{
@@ -372,7 +339,7 @@
}
-@@ -637,9 +665,13 @@
+@@ -637,9 +666,13 @@
void
run_client_uc_rdma_write_bw(void)
{
@@ -386,7 +353,7 @@
}
-@@ -649,7 +681,11 @@
+@@ -649,7 +682,11 @@
void
run_server_uc_rdma_write_bw(void)
{
@@ -398,7 +365,7 @@
}
-@@ -659,8 +695,12 @@
+@@ -659,8 +696,12 @@
void
run_client_uc_rdma_write_lat(void)
{
@@ -411,7 +378,7 @@
}
-@@ -670,7 +710,11 @@
+@@ -670,7 +711,11 @@
void
run_server_uc_rdma_write_lat(void)
{
@@ -423,7 +390,7 @@
}
-@@ -680,9 +724,13 @@
+@@ -680,9 +725,13 @@
void
run_client_uc_rdma_write_poll_lat(void)
{
@@ -437,7 +404,7 @@
}
-@@ -692,7 +740,11 @@
+@@ -692,7 +741,11 @@
void
run_server_uc_rdma_write_poll_lat(void)
{
@@ -449,6 +416,42 @@
}
+@@ -1856,7 +1909,18 @@
+ struct sockaddr_in saddr ={
+ .sin_family = AF_INET,
+ .sin_addr.s_addr = htonl(INADDR_ANY),
++#if (defined(__SVR4) && defined(__sun))
++/*
++ * Use the default listen port for right now to
++ * workaround CR 7099964 sol_ofs needs to support allocation
++ * of dynamic ports on rdma_bind_addr()
++ * need to remove when this is fix
++ */
++#define DEF_RDMA_LISTEN_PORT 19785 /* Listen port */
++ .sin_port = htons(DEF_RDMA_LISTEN_PORT)
++#else
+ .sin_port = htons(0)
++#endif
+ };
+ CMINFO *cm = &dev->cm;
+
+@@ -1864,10 +1928,16 @@
+ error(0, "rdma_bind_addr failed");
+ port = ntohs(rdma_get_src_port(cm->id));
+ encode_uint32(&port, port);
++#if !(defined(__SVR4) && defined(__sun))
+ send_mesg(&port, sizeof(port), "RDMA CM TCP IPv4 server port");
++#endif
+
+ if (rdma_listen(cm->id, 0) != 0)
+ error(0, "rdma_listen failed");
++
++#if (defined(__SVR4) && defined(__sun))
++ send_mesg(&port, sizeof(port), "RDMA CM TCP IPv4 server port");
++#endif
+ cm_expect_event(dev, RDMA_CM_EVENT_CONNECT_REQUEST);
+ rd_create_qp(dev, cm->event->id->verbs, cm->event->id);
+
diff -r -u /tmp/730054/qperf-0.4.6/src/rds.c qperf-0.4.6/src/rds.c
--- /tmp/730054/qperf-0.4.6/src/rds.c Mon Aug 31 00:00:40 2009
+++ qperf-0.4.6/src/rds.c Mon Nov 8 11:10:19 2010
@@ -509,3 +512,14 @@
qgetnameinfo((SA *)&sa, salen, 0, 0, p, sizeof(p), NI_NUMERICSERV);
port = atoi(p);
if (!port)
+diff -r -u /tmp/730054/qperf-0.4.6/src/qperf.h qperf-0.4.6/src/qperf.h
+--- /tmp/730054/qperf-0.4.6/src/qperf.h Mon Aug 31 00:00:40 2009
++++ qperf-0.4.6/src/qperf.h Tue Oct 18 16:40:59 2011
+@@ -193,6 +193,7 @@
+ uint32_t no_cpus; /* Number of processors */
+ uint32_t no_ticks; /* Ticks per second */
+ uint32_t max_cqes; /* Maximum CQ entries */
++ __attribute__ ((aligned (8))) /* align with 32-bit & 64-bit binary */
+ CLOCK time_s[T_N]; /* Start times */
+ CLOCK time_e[T_N]; /* End times */
+ USTAT s; /* Send statistics */
--- a/components/open-fabrics/rds-tools/Makefile Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/rds-tools/Makefile Wed Feb 29 22:39:04 2012 +0000
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
#
include ../../../make-rules/shared-macros.mk
@@ -33,9 +33,13 @@
include $(WS_TOP)/make-rules/configure.mk
include ../ofed.mk
+CPPFLAGS += -I$(PROTO_DIR)/usr/include
CFLAGS += -O2 -Wall -D_XOPEN_SOURCE=500 -D__EXTENSIONS__ $(CPPFLAGS)
-LIBS += -lsocket -lnsl -llgrp -lkstat
+LIBS += -lsocket -lnsl -llgrp -lkstat -libverbs
+
+COMPONENT_PREP_ACTION = \
+ cp rds-vendor.c $(@D)/
CLEANUP_DIRS = $(PROTOUSRBINDIR)/net
--- a/components/open-fabrics/rds-tools/patches/base.patch Wed Feb 29 12:08:58 2012 -0800
+++ b/components/open-fabrics/rds-tools/patches/base.patch Wed Feb 29 22:39:04 2012 +0000
@@ -15,21 +15,36 @@
#include <sys/stat.h>
#include <sys/poll.h>
#include <ctype.h>
-@@ -22,8 +28,13 @@
+@@ -22,10 +28,16 @@
#include <fcntl.h>
#include <sched.h>
#include <getopt.h>
+#if !(defined(__SVR4) && defined(__sun))
#include <byteswap.h>
#include "rds.h"
+-
+#else
-+#include <infiniband/ofa_solaris.h>
+#include <sys/rds.h>
+#endif
+ #include "pfhack.h"
++#if defined(__SVR4) && defined(__sun)
++#include <infiniband/ofa_solaris.h>
++#endif
- #include "pfhack.h"
+ /*
+ *
+@@ -102,6 +114,10 @@
+
+ #define NR_STATS S__LAST
-@@ -110,6 +121,7 @@
++#if defined(__SVR4) && defined(__sun)
++int sol_ioctl(int, int, struct rds_info_arg *, socklen_t *, int *);
++#endif
++
+ /*
+ * Parents share a mapped array of these with their children. Each child
+ * gets one. It's used to communicate between the child and the parent
+@@ -110,6 +126,7 @@
struct child_control {
pid_t pid;
int ready;
@@ -37,7 +52,7 @@
struct timeval start;
struct counter cur[NR_STATS];
struct counter last[NR_STATS];
-@@ -254,7 +266,20 @@
+@@ -254,7 +271,20 @@
die("invalid host name or dotted quad '%s'\n", ptr);
}
@@ -58,7 +73,7 @@
static void usage(void)
{
fprintf(stderr, "rds-stress version %s\n", RDS_VERSION);
-@@ -281,6 +306,9 @@
+@@ -281,6 +311,9 @@
" -c measure cpu use with per-cpu soak processes\n"
" -V trace execution\n"
" -z print a summary at end of test only\n"
@@ -68,7 +83,7 @@
"\n"
"Example:\n"
" recv$ rds-stress\n"
-@@ -310,7 +338,7 @@
+@@ -310,7 +343,7 @@
static void check_parent(pid_t pid)
{
if (pid != getppid())
@@ -77,7 +92,7 @@
}
/*
-@@ -334,6 +362,7 @@
+@@ -334,6 +367,7 @@
msg_pattern[i] = k;
}
@@ -85,7 +100,7 @@
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define htonll(x) bswap_64(x)
#define ntohll(x) bswap_64(x)
-@@ -341,6 +370,7 @@
+@@ -341,6 +375,7 @@
#define htonll(x) (x)
#define ntohll(x) (x)
#endif
@@ -93,7 +108,7 @@
static void encode_hdr(struct header *dst, const struct header *hdr)
{
-@@ -584,7 +614,11 @@
+@@ -584,7 +619,11 @@
if (opts->receive_addr == 0)
return 1;
@@ -105,7 +120,7 @@
sin.sin_port = htons(opts->starting_port);
sin.sin_addr.s_addr = htonl(opts->receive_addr);
-@@ -677,7 +711,11 @@
+@@ -677,7 +716,11 @@
size = sizeof(struct rdma_key_o_meter)
+ 2 * nr_tasks * sizeof(*kt)
+ 2 * RDMA_MAX_TRACKED_KEYS * sizeof(*ks);
@@ -117,7 +132,7 @@
if (base == MAP_FAILED)
die_errno("alloc_rdma_buffers: mmap failed");
-@@ -828,7 +866,7 @@
+@@ -828,7 +871,7 @@
}
if (!failed)
@@ -126,7 +141,7 @@
(unsigned long long) pattern, addr);
}
-@@ -865,7 +903,11 @@
+@@ -865,7 +908,11 @@
/* We use mmap here rather than malloc, because it is always
* page aligned. */
len = 2 * opts->nr_tasks * opts->req_depth * (opts->rdma_vector * opts->rdma_size) + sys_page_size;
@@ -138,7 +153,7 @@
if (base == MAP_FAILED)
die_errno("alloc_rdma_buffers: mmap failed");
memset(base, 0x2f, len);
-@@ -915,17 +957,16 @@
+@@ -915,17 +962,16 @@
if (RDMA_OP_READ == hdr->rdma_op) {
if (opt.verify)
rds_fill_buffer(rdma_addr, rdma_size, hdr->rdma_pattern);
@@ -162,7 +177,7 @@
}
}
-@@ -947,7 +988,7 @@
+@@ -947,7 +993,7 @@
die("Unexpected RDMA op %u in request\n", in_hdr->rdma_op);
@@ -171,7 +186,7 @@
in_hdr->rdma_op == RDMA_OP_WRITE? "write to" : "read from",
rdma_size,
(unsigned long long) in_hdr->rdma_addr,
-@@ -1007,6 +1048,9 @@
+@@ -1007,6 +1053,9 @@
t->drain_rdmas = 0;
}
@@ -181,7 +196,7 @@
#define MSG_MAXIOVLEN 2
/*
-@@ -1560,7 +1604,12 @@
+@@ -1560,7 +1609,12 @@
struct timeval start;
int do_work = opts->simplex ? active : 1;
@@ -194,7 +209,7 @@
sin.sin_port = htons(opts->starting_port + 1 + id);
sin.sin_addr.s_addr = htonl(opts->receive_addr);
-@@ -1572,7 +1621,11 @@
+@@ -1572,7 +1626,11 @@
for (i = 0; i < opts->nr_tasks; i++) {
tasks[i].nr = i;
tasks[i].src_addr = sin;
@@ -206,7 +221,7 @@
tasks[i].dst_addr.sin_addr.s_addr = htonl(opts->send_addr);
tasks[i].dst_addr.sin_port = htons(opts->starting_port + 1 + i);
tasks[i].send_time = alloca(opts->req_depth * sizeof(struct timeval));
-@@ -1625,6 +1678,10 @@
+@@ -1625,6 +1683,10 @@
;
}
@@ -217,7 +232,7 @@
/* keep the pipeline full */
can_send = !!(pfd.revents & POLLOUT);
for (i = 0, t = tasks; i < opts->nr_tasks; i++, t++) {
-@@ -1665,8 +1722,12 @@
+@@ -1665,8 +1727,12 @@
uint32_t i;
len = opts->nr_tasks * sizeof(*ctl);
@@ -230,7 +245,7 @@
if (ctl == MAP_FAILED)
die("mmap of %u child control structs failed", opts->nr_tasks);
-@@ -1699,7 +1760,7 @@
+@@ -1699,7 +1765,7 @@
continue;
pid = waitpid(-1, NULL, WNOHANG);
if (pid)
@@ -239,7 +254,149 @@
sleep(1);
i--; /* try this child again */
}
-@@ -1967,7 +2028,7 @@
+@@ -1823,6 +1889,7 @@
+
+ if (disable)
+ return;
++#if !(defined(__SVR4) && defined(__sun))
+ if ((fp = fopen("/proc/stat", "r")) == NULL) {
+ fprintf(stderr, "Cannot open /proc/stat (%s) - "
+ "not printing cpu stats\n",
+@@ -1856,10 +1923,37 @@
+ }
+ }
+ fclose(fp);
++#else
++#define NSEC_TO_TICK(v) (v * sysconf(_SC_CLK_TCK)/1000000000)
++ sol_cpu_stats_t stats;
+
++ solaris_init();
++ if ((sol_get_cpu_stats(&stats)) < 0) {
++ disable = 1;
++ return;
++ }
++ solaris_fini();
++ current.times[0] = stats.t_user;
++ current.times[1] = 0;
++ current.times[2] = stats.t_kernel;
++ current.times[3] = stats.t_idle;
++ current.times[4] = stats.t_iowait;
++ current.times[5] = 0;
++ current.times[6] = 0;
++ current.times[7] = 0;
++ current.intr = NSEC_TO_TICK(stats.t_intr); /* NSEC_TO_TICK */
++
++#endif
++
+ if (initialize) {
++#if !(defined(__SVR4) && defined(__sun))
+ printf(",user:percent,system:percent,idle:percent"
+ ",irq:percent,intr:count");
++#else
++ /* Solaris kstat doesn't provide irq/softirq info. */
++ printf(",user:percent,system:percent,idle:percent"
++ ",intr:count");
++#endif
+ } else {
+ struct sys_stats sys;
+ unsigned long sum = 0;
+@@ -1884,6 +1978,7 @@
+ * 5 irq
+ * 6 softirq
+ */
++#if !(defined(__SVR4) && defined(__sun))
+ printf(",%f,%f,%f,%f,%Lu",
+ (sys.times[0] + sys.times[1]) * scale,
+ sys.times[2] * scale,
+@@ -1890,6 +1985,14 @@
+ (sys.times[3] + sys.times[4]) * scale,
+ (sys.times[5] + sys.times[6]) * scale,
+ sys.intr);
++#else
++ /* Solaris kstat doesn't provide irq/softirq info. */
++ printf(",%f,%f,%f,%Lu",
++ (sys.times[0] + sys.times[1]) * scale,
++ sys.times[2] * scale,
++ (sys.times[3] + sys.times[4]) * scale,
++ sys.intr);
++#endif
+ }
+ prev = current;
+ }
+@@ -1903,6 +2006,10 @@
+ static socklen_t buflen = 0;
+ static int sock_fd = -1;
+ int i, count, item_size;
++#if defined(__SVR4) && defined(__sun)
++ socklen_t len;
++ struct rds_info_arg arg;
++#endif
+
+ if (sock_fd < 0) {
+ sock_fd = socket(pf, SOCK_SEQPACKET, 0);
+@@ -1912,6 +2019,7 @@
+
+ /* We should only loop once on the first call; after that the
+ * buffer requirements for RDS counters should not change. */
++#if !(defined(__SVR4) && defined(__sun))
+ while ((item_size = getsockopt(sock_fd, sol, RDS_INFO_COUNTERS, curr, &buflen)) < 0) {
+ if (errno != ENOSPC)
+ die_errno("getsockopt(RDS_INFO_COUNTERS) failed");
+@@ -1919,7 +2027,29 @@
+ if (!curr)
+ die_errno("Cannot allocate buffer for stats counters");
+ }
++#else
++ int retcode;
+
++ retcode = sol_ioctl(
++ sock_fd, RDS_INFO_COUNTERS, &arg, &buflen, &item_size);
++ if (retcode != 0) {
++ if (retcode == 1) {
++ die_errno("ioctl(RDS_INFO_COUNTERS) failed");
++ } else if (retcode == 2) {
++ fprintf(stderr, "%s: Unable to allocate memory "
++ "for %u bytes of info: %s\n",
++ "rds-stress", buflen, strerror(errno));
++ return;
++ } else {
++ fprintf(stderr, "%s: Unable to access "
++ "RDS_INFO_COUNTERS statistics: %s\n",
++ "rds-stress", strerror(errno));
++ return;
++ }
++ }
++
++#endif
++
+ if (item_size > sizeof(*ctr))
+ die("Bad counter item size in RDS_INFO_COUNTERS (got %d, max %zd)\n",
+ item_size, sizeof(*ctr));
+@@ -1932,8 +2062,11 @@
+ }
+
+ for (i = 0; i < count; ++i)
++#if !(defined(__SVR4) && defined(__sun))
+ memcpy(ctr + i, curr + i * item_size, item_size);
+-
++#else
++ memcpy(ctr + i, ((void *)(uintptr_t)arg.datap) + i * item_size, item_size);
++#endif
+ gettimeofday(&now, NULL);
+
+ if (initialize) {
+@@ -1957,6 +2090,10 @@
+ memcpy(prev, ctr, count * sizeof(*ctr));
+ last_ts = now;
+
++#if defined(__SVR4) && defined(__sun)
++ free((void *)(uintptr_t)arg.datap);
++#endif
++
+ get_stats(initialize);
+ }
+
+@@ -1967,7 +2104,7 @@
pid = waitpid(-1, &status, wflags);
if (pid < 0)
@@ -248,7 +405,7 @@
if (pid == 0)
return 0;
-@@ -1975,15 +2036,15 @@
+@@ -1975,15 +2112,15 @@
if (WEXITSTATUS(status) == 0)
return 1;
die("child pid %u exited with status %d\n",
@@ -267,7 +424,7 @@
}
static void release_children_and_wait(struct options *opts,
-@@ -2139,7 +2200,12 @@
+@@ -2139,7 +2276,12 @@
control_fd = -1;
if (nr_running) {
@@ -280,7 +437,7 @@
kill(ctl[i].pid, SIGTERM);
stop_soakers(soak_arr);
}
-@@ -2517,7 +2583,11 @@
+@@ -2517,7 +2659,11 @@
/* an extra terminating entry which will be all 0s */
len = (nr_soak + 1) * sizeof(struct soak_control);
soak_arr = mmap(NULL, len, PROT_READ|PROT_WRITE,
@@ -292,7 +449,7 @@
if (soak_arr == MAP_FAILED)
die("mmap of %ld soak control structs failed", nr_soak);
-@@ -2589,6 +2659,7 @@
+@@ -2589,6 +2735,7 @@
{ "rtprio", no_argument, NULL, 'R' },
{ "verify", no_argument, NULL, 'v' },
{ "trace", no_argument, NULL, 'V' },
@@ -300,7 +457,7 @@
{ "rdma-use-once", required_argument, NULL, OPT_RDMA_USE_ONCE },
{ "rdma-use-get-mr", required_argument, NULL, OPT_RDMA_USE_GET_MR },
-@@ -2652,7 +2723,7 @@
+@@ -2652,7 +2799,7 @@
while(1) {
int c, index;
@@ -309,7 +466,7 @@
long_options, &index);
if (c == -1)
break;
-@@ -2711,6 +2782,10 @@
+@@ -2711,6 +2858,10 @@
case 'V':
opts.tracing = 1;
break;
@@ -320,7 +477,7 @@
case OPT_USE_CONG_MONITOR:
opts.use_cong_monitor = parse_ull(optarg, 1);
break;
-@@ -2786,6 +2861,7 @@
+@@ -2786,6 +2937,7 @@
if (opts.rdma_size && 0)
opts.rdma_size = (opts.rdma_size + 4095) & ~4095;
@@ -346,7 +503,7 @@
diff -r -u /tmp/rds-tools-2.0.4/rds-info.c rds-tools-2.0.7/rds-info.c
--- /tmp/rds-tools-2.0.4/rds-info.c Wed Aug 4 15:25:10 2010
+++ rds-tools-2.0.7/rds-info.c Thu Feb 24 13:27:51 2011
-@@ -42,16 +42,27 @@
+@@ -42,16 +42,28 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <errno.h>
@@ -361,6 +518,7 @@
+#if defined(__SVR4) && defined(__sun)
+#include <sys/rds.h>
++#include <infiniband/ofa_solaris.h>
+#else
#include "rds.h"
+#endif
@@ -375,7 +533,18 @@
#define min(a, b) (a < b ? a : b)
#define array_size(foo) (sizeof(foo) / sizeof(foo[0]))
-@@ -234,8 +245,10 @@
+@@ -76,6 +88,10 @@
+
+ char *progname = "rds-info";
+
++#if defined(__SVR4) && defined(__sun)
++int sol_ioctl(int, int, struct rds_info_arg *, socklen_t *, int *);
++#endif
++
+ /* Like inet_ntoa, but can be re-entered several times without clobbering
+ * the previously returned string. */
+ static const char *paddr(int af, const void *addrp)
+@@ -234,8 +250,10 @@
print_msgs, "Send", 0 },
['t'] = { RDS_INFO_RETRANS_MESSAGES, "retransmit queue messages",
print_msgs, "Retransmit", 0 },
@@ -386,18 +555,17 @@
['I'] = { RDS_INFO_IB_CONNECTIONS, "IB transport connections",
print_ib_conns, NULL, 0 },
};
-@@ -266,6 +279,10 @@
+@@ -266,6 +284,9 @@
char optstring[258] = "v+";
int given_options = 0;
socklen_t len = 0;
+#if defined(__SVR4) && defined(__sun)
-+ socklen_t ulen;
+ struct rds_info_arg arg;
+#endif
void *data = NULL;
int fd;
int each;
-@@ -322,6 +339,7 @@
+@@ -322,6 +343,7 @@
(given_options && !infos[i].option_given))
continue;
@@ -405,25 +573,33 @@
/* read in the info until we get a full snapshot */
while ((each = getsockopt(fd, sol, infos[i].opt_val, data,
&len)) < 0) {
-@@ -345,15 +363,60 @@
+@@ -345,15 +367,47 @@
return 1;
}
}
+#else
-+ /* 1st call gets the length of the data available */
-+ ulen = 0;
-+ bzero(&arg, sizeof (struct rds_info_arg));
-+ arg.lenp = (uint64_t)(uintptr_t)&ulen;
-+ arg.datap = NULL;
-+ each = ioctl(fd, infos[i].opt_val, &arg);
-+ if ((each < 0) && (errno != ENOSPC)) {
-+ verbosef(0, stderr, "%s: Unable get statistics: %s\n",
-+ progname, strerror(errno));
-+ return 1;
++ int retcode;
+
++ retcode = sol_ioctl(fd, infos[i].opt_val, &arg, &len, &each);
++ if (retcode != 0) {
++ if (retcode == 1) {
++ fprintf(stderr, "%s: Unable get statistics: "
++ "%s\n", progname, strerror(errno));
++ return 1;
++ } else if (retcode == 2) {
++ fprintf(stderr, "%s: Unable to allocate memory"
++ " for %u bytes of info: %s\n",
++ progname, len, strerror(errno));
++ return 1;
++ } else {
++ fprintf(stderr, "%s: Unable to get statistics:"
++ " %s\n", progname, strerror(errno));
++ return 1;
++ }
+ }
-
++
+ /* No data at the driver */
-+ if (ulen == 0)
++ if (len == 0)
+ invalid_opt = 1;;
+#endif
+
@@ -432,29 +608,9 @@
+#if !(defined(__SVR4) && defined(__sun))
infos[i].print(data, each, len, infos[i].extra);
+-
+#else
-+ do {
-+ arg.datap = (uint64_t)(uintptr_t)realloc(
-+ (char *)(uintptr_t)arg.datap, ulen);
-+ if (arg.datap == NULL) {
-+ verbosef(0, stderr, "%s: Unable to allocate memory "
-+ "for %u bytes of info: %s\n",
-+ progname, ulen, strerror(errno));
-+ return 1;
-+ }
-
-+ /* 2nd call gets the data */
-+ len = ulen;
-+ each = ioctl(fd, infos[i].opt_val, &arg);
-+ if ((each < 0) && (errno != ENOSPC)) {
-+ verbosef(0, stderr,
-+ "%s: Unable get statistics: %s\n",
-+ progname, strerror(errno));
-+ return 1;
-+ }
-+ } while (ulen > len);
-+
-+ infos[i].print((void *)(uintptr_t)arg.datap, each, ulen,
++ infos[i].print((void *)(uintptr_t)arg.datap, each, len,
+ infos[i].extra);
+#endif
if (given_options && --given_options == 0)
@@ -985,7 +1141,7 @@
diff -r -u /tmp/rds-tools-2.0.4/Makefile.in rds-tools-2.0.7/Makefile.in
--- /tmp/rds-tools-2.0.4/Makefile.in Wed Aug 4 15:25:11 2010
+++ rds-tools-2.0.7/Makefile.in Thu Feb 24 13:27:51 2011
-@@ -4,10 +4,14 @@
+@@ -4,18 +4,22 @@
mandir = $(DESTDIR)@mandir@
incdir = $(DESTDIR)@includedir@
@@ -998,11 +1154,12 @@
+CFLAGS += -O2 -Wall -Iinclude
+CPPFLAGS += -D_XOPEN_SOURCE=500 -D__EXTENSIONS__ \
+ -DDEBUG_EXE -DRDS_VERSION=\"@VERSION@\" -MD -MP -MF $(@D)/.$(basename $(@F)).d
-+LDFLAGS += -lsocket -lnsl -llgrp
++LDFLAGS += -libverbs -lsocket -lnsl -llgrp
HEADERS = kernel-list.h pfhack.h include/rds.h
- COMMON_SOURCES = pfhack.c
-@@ -15,7 +19,7 @@
+-COMMON_SOURCES = pfhack.c
++COMMON_SOURCES = pfhack.c rds-vendor.c
+ SOURCES = $(addsuffix .c,$(PROGRAMS)) $(COMMON_SOURCES)
CLEAN_OBJECTS = $(addsuffix .o,$(PROGRAMS)) $(subst .c,.o,$(COMMON_SOURCES))
# This is the default
@@ -1133,7 +1290,7 @@
diff -r -u /tmp/rds-tools-2.0.4/rds-stress.1 rds-tools-2.0.7/rds-stress.1
--- /tmp/rds-tools-2.0.4/rds-stress.1 Wed Aug 4 15:25:11 2010
+++ rds-tools-2.0.7/rds-stress.1 Thu Feb 24 13:27:52 2011
-@@ -1,99 +1,103 @@
+@@ -1,99 +1,102 @@
-.Dd May 15, 2007
-.Dt RDS-STRESS 1
-.Os
@@ -1165,8 +1322,7 @@
+.nf
+rds-stress [-p port_number] -r [receive_address] [-s send_address]
+ [-a ack_bytes] [-q request_bytes] [-D rdma_bytes]
-+ [-d queue_depth] [-t Ar nr_tasks] [-c] [-R] [-V] [-v] [-o]
-+ [-I iovecs] -M [nr] [-z] [-g lgrpid]
++ [-d queue_depth] [-t nr_tasks] [-c] [-R] [-V] [-v]
+.fi
-.Sh DESCRIPTION
@@ -1283,7 +1439,7 @@
RDSv3 is capable of transmitting part of a message via RDMA directly from
application buffer to application buffer. This option enables RDMA support
in rds-stress: request packets include parameters for an RDMA READ or WRITE
-@@ -100,20 +104,25 @@
+@@ -100,20 +103,25 @@
operation, which the receiving process executes at the time the ACK packet
is sent.
See section "Message Sizes" below.
@@ -1314,7 +1470,7 @@
This causes rds-stress to create child tasks which just consume CPU cycles.
One task is created for each CPU in the system. First each child observes the
maximum rate at which it can consume cycles. This means that this option
-@@ -121,50 +130,78 @@
+@@ -121,50 +129,67 @@
use of the system by observing the lesser rate at which the children consume
cycles. This option is *not* shared between the active and passive instances.
It must be specified on each rds-stress command line.
@@ -1334,17 +1490,6 @@
-.El
-.Pp
+.TP
-+\fB\-o
-+Datagrams sent one way only (default is both)
-+.TP
-+\fB\-I iovecs
-+RDMA: number of user buffers to target (default is 1, max is 512)
-+.TP
-+\fB\-M nr
-+RDMA: mode (0=readwrite,1=readonly,2=writeonly)
-+.TP
-+\fB\-g lgrpid
-+bind the process to the specified lgrp
+.PP
-.Ss Message Sizes
@@ -1412,7 +1557,7 @@
This is the percentage of available CPU resources on this machine that are being
consumed since rds-stress started running. It will show -1.00 if -c is not
given. It is calculated based on the amount of CPU resources that CPU soaking
-@@ -171,4 +208,3 @@
+@@ -171,4 +196,3 @@
tasks are able to consume. This lets it measure CPU use by the system, say in
interrupt handlers, that task-based CPU accounting does not include.
For this to work rds-stress must be started with -c on an idle system.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/open-fabrics/rds-tools/rds-vendor.c Wed Feb 29 22:39:04 2012 +0000
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+/*
+ * OFED Solaris ioctl wrapper
+ */
+#if defined(__SVR4) && defined(__sun)
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <strings.h>
+#include <unistd.h>
+#include <stropts.h>
+#include <sys/rds.h>
+
+int
+sol_ioctl(int fd, int opt_val, struct rds_info_arg *argp, socklen_t *ulen,
+ int *item_size)
+{
+ int len;
+ struct rds_info_arg arg = *argp;
+
+ /* 1st call gets the length of the data available */
+ *ulen = 0;
+ bzero(&arg, sizeof (struct rds_info_arg));
+ arg.lenp = (uint64_t)(uintptr_t)ulen;
+ arg.datap = NULL;
+ *item_size = ioctl(fd, opt_val, &arg);
+ argp->lenp = arg.lenp;
+ argp->datap = arg.datap;
+ if ((*item_size < 0) && (errno != ENOSPC))
+ return (1);
+
+ do {
+ if (*ulen == 0)
+ return (0);
+ if (arg.datap)
+ arg.datap = (uint64_t)(uintptr_t)realloc(
+ (char *)(uintptr_t)arg.datap, *ulen);
+ else
+ arg.datap = (uint64_t)(uintptr_t)malloc(*ulen);
+
+ if (arg.datap == NULL)
+ return (2);
+
+ /* 2nd call gets the data */
+ len = *ulen;
+ *item_size = ioctl(fd, opt_val, &arg);
+ argp->lenp = arg.lenp;
+ argp->datap = arg.datap;
+ if ((*item_size < 0) && (errno != ENOSPC)) {
+ return (3);
+ }
+ } while (*ulen > len);
+ return (0);
+}
+#endif