18117487 saquery -m -t 50000 failed with multiple VFs created
authorBoris Chiu <boris.chiu@oracle.com>
Thu, 27 Mar 2014 10:19:43 -0700
changeset 1793 b253729db1c0
parent 1792 5cea652172c6
child 1794 942d8ee5d880
18117487 saquery -m -t 50000 failed with multiple VFs created 18368329 Should use snprintf to convert numeric link_layer to string for ibstat to work
components/open-fabrics/ibutils/patches/base.patch
components/open-fabrics/infiniband-diags/patches/base.patch
components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c
components/open-fabrics/libibumad/patches/base.patch
components/open-fabrics/libibverbs/solaris_compatibility.c
components/open-fabrics/opensm/patches/base.patch
--- a/components/open-fabrics/ibutils/patches/base.patch	Wed Mar 26 23:18:55 2014 -0700
+++ b/components/open-fabrics/ibutils/patches/base.patch	Thu Mar 27 10:19:43 2014 -0700
@@ -947,6 +947,18 @@
  
  #define CR_CLASS        0x09
  #define CR_ATTR_50      0x50
+diff -r -u /tmp/ibutils-1.5.7/ibis/src/ibis_base.h ibutils-1.5.7/ibis/src/ibis_base.h
+--- /tmp/ibutils-1.5.7/ibis/src/ibis_base.h	Wed Feb 24 02:09:25 2010
++++ ibutils-1.5.7/ibis/src/ibis_base.h	Mon Mar 24 15:09:53 2014
[email protected]@ -64,7 +64,7 @@
+ #define IBIS_FILE_PATH_MAX	PATH_MAX
+ #endif
+ 
+-#define MAX_LOCAL_IBPORTS 32
++#define MAX_LOCAL_IBPORTS MAX_PORTS
+ #define DISP_HANDLE_MAX 32
+ 
+ END_C_DECLS
 diff -r -u /tmp/ibutils-1.5.7/ibis/src/ibis_wrap.c ibutils-1.5.7/ibis/src/ibis_wrap.c
 --- /tmp/ibutils-1.5.7/ibis/src/ibis_wrap.c	Mon Nov  1 16:03:44 2010
 +++ ibutils-1.5.7/ibis/src/ibis_wrap.c	Mon Jun  6 15:35:52 2011
--- a/components/open-fabrics/infiniband-diags/patches/base.patch	Wed Mar 26 23:18:55 2014 -0700
+++ b/components/open-fabrics/infiniband-diags/patches/base.patch	Thu Mar 27 10:19:43 2014 -0700
@@ -1803,16 +1803,17 @@
 diff -r -u /tmp/infiniband-diags-1.5.8/src/saquery.c infiniband-diags-1.5.8/src/saquery.c
 --- /tmp/infiniband-diags-1.5.8/src/saquery.c	Wed Feb 16 02:13:21 2011
 +++ infiniband-diags-1.5.8/src/saquery.c	Thu Jun 23 10:36:40 2011
[email protected]@ -105,7 +105,7 @@
[email protected]@ -104,8 +104,7 @@
+ /**
   * Declare some globals because I don't want this to be too complex.
   */
- #define MAX_PORTS (8)
+-#define MAX_PORTS (8)
 -#define DEFAULT_SA_TIMEOUT_MS (1000)
 +#define DEFAULT_SA_TIMEOUT_MS (10000)
  static struct query_res result;
  
  enum {
[email protected]@ -1757,7 +1757,7 @@
[email protected]@ -1757,7 +1756,7 @@
  			requested_lid = (uint16_t) strtoul(argv[0], NULL, 0);
  			requested_lid_flag++;
  		} else if (node_print_desc == NAME_OF_GUID) {
@@ -1885,27 +1886,7 @@
  	"Down",
  	"Initializing",
  	"Armed",
[email protected]@ -96,6 +97,19 @@
- 	"PhyTest"
- };
- 
-+static const char *port_link_layer_str(uint8_t link_layer)
-+{
-+	switch (link_layer) {
-+		case IBV_LINK_LAYER_UNSPECIFIED:
-+		case IBV_LINK_LAYER_INFINIBAND:
-+			return "IB";
-+		case IBV_LINK_LAYER_ETHERNET:
-+			return "Ethernet";
-+		default:
-+			return "Unknown";
-+        }
-+};
-+
- static int port_dump(umad_port_t * port, int alone)
- {
- 	char *pre = "";
[email protected]@ -112,10 +126,10 @@
[email protected]@ -112,10 +113,10 @@
  	printf("%sPort %d:\n", hdrpre, port->portnum);
  	printf("%sState: %s\n", pre,
  	       (unsigned)port->state <=
@@ -1918,16 +1899,24 @@
  	printf("%sRate: %d\n", pre, port->rate);
  	printf("%sBase lid: %d\n", pre, port->base_lid);
  	printf("%sLMC: %d\n", pre, port->lmc);
[email protected]@ -122,7 +136,7 @@
- 	printf("%sSM lid: %d\n", pre, port->sm_lid);
- 	printf("%sCapability mask: 0x%08x\n", pre, ntohl(port->capmask));
- 	printf("%sPort GUID: 0x%016" PRIx64 "\n", pre, ntohll(port->port_guid));
--	printf("%sLink layer: %s\n", pre, port->link_layer);
-+	printf("%sLink layer: %s\n", pre, port_link_layer_str(atoi(port->link_layer)));
- 	return 0;
- }
[email protected]@ -167,13 +168,13 @@
+ 
+ static int ports_list(char names[][UMAD_CA_NAME_LEN], int n)
+ {
+-	uint64_t guids[64];
++	uint64_t guids[MAX_PORTS];
+ 	int found, ports, i;
  
[email protected]@ -245,7 +259,7 @@
+-	for (i = 0, found = 0; i < n && found < 64; i++) {
++	for (i = 0, found = 0; i < n && found < MAX_PORTS; i++) {
+ 		if ((ports =
+ 		     umad_get_ca_portguids(names[i], guids + found,
+-					   64 - found)) < 0)
++					   MAX_PORTS - found)) < 0)
+ 			return -1;
+ 		found += ports;
+ 	}
[email protected]@ -245,7 +246,7 @@
  		if (i >= n)
  			IBPANIC("'%s' IB device can't be found", argv[0]);
  
--- a/components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c	Wed Mar 26 23:18:55 2014 -0700
+++ b/components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c	Thu Mar 27 10:19:43 2014 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -54,6 +54,7 @@
 
 #include <infiniband/verbs.h>
 #include <infiniband/arch.h>
+#include <infiniband/umad.h>
 
 #include <sys/ib/adapters/hermon/hermon_ioctl.h>
 
@@ -87,7 +88,6 @@
 static di_node_t	di_rootnode;
 char *argv0 = "solaris_set_nodedesc";
 
-#define	MAX_HCAS	32
 static struct nodedesc_read_info_s {
 	boolean_t	info_valid;
 	uint64_t	guid;
--- a/components/open-fabrics/libibumad/patches/base.patch	Wed Mar 26 23:18:55 2014 -0700
+++ b/components/open-fabrics/libibumad/patches/base.patch	Thu Mar 27 10:19:43 2014 -0700
@@ -15,9 +15,29 @@
 diff -r -u /tmp/libibumad-1.3.7/include/infiniband/umad.h libibumad-1.3.7/include/infiniband/umad.h
 --- /tmp/libibumad-1.3.7/include/infiniband/umad.h	Mon Feb 14 01:08:29 2011
 +++ libibumad-1.3.7/include/infiniband/umad.h	Fri May 13 04:28:12 2011
[email protected]@ -96,7 +96,11 @@
[email protected]@ -45,7 +45,15 @@
+ #endif				/* __cplusplus */
  
- #define UMAD_MAX_PORTS		64
+ BEGIN_C_DECLS
+-#define UMAD_MAX_DEVICES 32
++/* To accomodate max. of 16 PCIe slots and 63 VFs per HCA, */
++/* we define	MAX_HCAS = (63VFs + 1PF) * 16HCAs */
++/* 		MAX_PORTS = MAX_HCAS * 2ports per HCA */
++#define MAX_HCAS	(64*16)
++#define MAX_PORTS	(MAX_HCAS*2)
++#define UMAD_MAX_DEVICES MAX_HCAS
++#define	UMAD_MAX_HCAS	MAX_HCAS
++#define	UMAD_MAX_PORTS	MAX_PORTS
++
+ #define UMAD_ANY_PORT	0
+ typedef struct ib_mad_addr {
+ 	uint32_t qpn;
[email protected]@ -94,9 +102,13 @@
+ #define SYS_IB_MAD_PORT		"port"
+ #define SYS_IB_MAD_DEV		"ibdev"
+ 
+-#define UMAD_MAX_PORTS		64
++#define UMAD_MAX_PORTS		MAX_PORTS
  
 +#if defined(__SVR4) && defined(__sun)
 +#define	UMAD_DEV_DIR		"/dev/infiniband/ofs"
--- a/components/open-fabrics/libibverbs/solaris_compatibility.c	Wed Mar 26 23:18:55 2014 -0700
+++ b/components/open-fabrics/libibverbs/solaris_compatibility.c	Thu Mar 27 10:19:43 2014 -0700
@@ -99,8 +99,6 @@
 #define	RDMA_USER_CM_MAX_ABI_VERSION	4 /* rdma_cma_abi.h */
 
 #define	MLX4	0
-#define	MAX_HCAS				260
-#define	MAX_HCA_PORTS				16
 #define	HW_DRIVER_MAX_NAME_LEN			20
 #define	UVERBS_KERNEL_SYSFS_NAME_BASE		"uverbs"
 #define	UMAD_KERNEL_SYSFS_NAME_BASE		"umad"
@@ -117,6 +115,9 @@
 #define	INFINIHOST_DEVICE_ID_2			0x5a45
 #define	INFINIHOST_DEVICE_ID_4			0x6279
 
+#define	MAX_HCAS				(64*16)
+#define	MAX_PORTS				(MAX_HCAS*2)
+
 /*
  * sol_uverbs_drv_status is the status of what libibverbs knows
  * about the status of sol_uverbs driver.
@@ -162,7 +163,7 @@
 	char		ibd_node_guid_external_str[20];
 	char		ibd_sys_image_guid[20];
 	char		ibd_fw_ver[16];
-	char		ibd_name[8];
+	char		ibd_name[16];
 	int		ibd_boardid_index;
 	uint_t		ibd_device_id;
 } ibdev_cache_info_t;
@@ -176,7 +177,7 @@
 	uint_t		uvc_vendor_id;
 	uint_t		uvc_device_id;
 	int		uvc_hca_instance;
-	char		uvc_ibdev_name[8];
+	char		uvc_ibdev_name[16];
 	char		uvc_ibdev_hca_path[MAXPATHLEN];
 } uverbs_cache_info_t;
 static uverbs_cache_info_t	uverbs_dev_cache[MAX_HCAS];
@@ -187,7 +188,7 @@
 	int		umc_port;
 	char		umc_ib_dev[16];
 } umad_cache_info_t;
-static umad_cache_info_t	umad_dev_cache[MAX_HCAS * MAX_HCA_PORTS];
+static umad_cache_info_t	umad_dev_cache[MAX_PORTS];
 static int			umad_abi_version = -1;
 
 pthread_once_t		oneTimeInit = PTHREAD_ONCE_INIT;
@@ -228,8 +229,7 @@
 static int
 umad_cache_add(uint_t dev_num, int port, char *ibdev)
 {
-	if ((dev_num >= (MAX_HCAS  * MAX_HCA_PORTS)) ||
-	    (umad_cache_cnt >= (MAX_HCAS  * MAX_HCA_PORTS))) {
+	if ((dev_num >= MAX_PORTS) || (umad_cache_cnt >= MAX_PORTS)) {
 		fprintf(stderr, "dev %d: exceeds umad cache size\n", dev_num);
 		return (1);
 	}
@@ -485,7 +485,7 @@
 	sol_umad_ioctl_info_t		*umad_infop;
 	sol_umad_ioctl_port_info_t	*port_infop;
 
-	for (minor = 0; minor < MAX_HCAS * MAX_HCA_PORTS; minor++) {
+	for (minor = 0; minor < MAX_PORTS; minor++) {
 		snprintf(umad_devpath, MAXPATHLEN, "%s/%s%d",
 		    IB_OFS_DEVPATH_PREFIX, UMAD_KERNEL_SYSFS_NAME_BASE,
 		    minor);
@@ -497,7 +497,7 @@
 			save_errno = errno;
 	}
 
-	if ((minor == MAX_HCAS * MAX_HCA_PORTS) && (fd < 0)) {
+	if ((minor == MAX_PORTS) && (fd < 0)) {
 		if (! save_errno)
 			save_errno = errno;
 		fprintf(stderr, "failed to open sol_umad: %s\n",
@@ -506,12 +506,12 @@
 	}
 
 	bufsize = sizeof (sol_umad_ioctl_info_t) +
-	    (sizeof (sol_umad_ioctl_port_info_t) * MAX_HCAS * MAX_HCA_PORTS);
+	    (sizeof (sol_umad_ioctl_port_info_t) * MAX_PORTS);
 
 	buf = malloc(bufsize);
 	memset(buf, 0, bufsize);
 	umad_infop = (sol_umad_ioctl_info_t *)buf;
-	umad_infop->umad_port_cnt = MAX_HCAS * MAX_HCA_PORTS;
+	umad_infop->umad_port_cnt = MAX_PORTS;
 
 	if (ioctl(fd, IB_USER_MAD_GET_PORT_INFO, umad_infop) != 0) {
 		fprintf(stderr, "sol_umad ioctl failed: %s\n",
@@ -594,7 +594,7 @@
 	memset(&uverbs_dev_cache, 0, (sizeof (uverbs_cache_info_t) * MAX_HCAS));
 	memset(&ibdev_cache, 0, (sizeof (ibdev_cache_info_t) * MAX_HCAS * 2));
 	memset(&umad_dev_cache, 0,
-	    (sizeof (umad_cache_info_t) * MAX_HCAS * MAX_HCA_PORTS));
+	    (sizeof (umad_cache_info_t) * MAX_PORTS));
 
 	initialized = B_TRUE;
 }
@@ -1144,7 +1144,18 @@
 			len = 1 + sprintf(buf, "0x%08x",
 			    port_attr.port_cap_flags);
 		} else if (strcmp(path, "link_layer") == 0) {
-			len = 1 + sprintf(buf, "%d", port_attr.link_layer);
+			switch (port_attr.link_layer) {
+				case IBV_LINK_LAYER_UNSPECIFIED:
+				case IBV_LINK_LAYER_INFINIBAND:
+					len = 1 + sprintf(buf, "%s", "IB");
+					break;
+				case IBV_LINK_LAYER_ETHERNET:
+					len =
+					    1 + sprintf(buf, "%s", "Ethernet");
+					break;
+				default:
+					len = 1 + sprintf(buf, "%s", "Unknown");
+			}
 		}
 	}
 exit:
@@ -1395,7 +1406,7 @@
 	(void) pthread_mutex_unlock(&umad_cache_mutex);
 
 	if (check_path(path, CP_UMAD, &dev_num)) {
-		if (dev_num >= MAX_HCAS * MAX_HCA_PORTS) {
+		if (dev_num >= MAX_PORTS) {
 			fprintf(stderr, "Invalid Path: %s\n", path);
 			goto exit;
 		}
--- a/components/open-fabrics/opensm/patches/base.patch	Wed Mar 26 23:18:55 2014 -0700
+++ b/components/open-fabrics/opensm/patches/base.patch	Thu Mar 27 10:19:43 2014 -0700
@@ -665,6 +665,18 @@
  }
  
  /*
+diff -r -u /tmp/opensm-3.3.9/include/vendor/osm_vendor_ibumad.h opensm-3.3.9/include/vendor/osm_vendor_ibumad.h
+--- /tmp/opensm-3.3.9/include/vendor/osm_vendor_ibumad.h	Sun Mar  6 23:51:12 2011
++++ opensm-3.3.9/include/vendor/osm_vendor_ibumad.h	Mon Mar 24 15:10:20 2014
[email protected]@ -64,7 +64,7 @@
+ *
+ *********/
+ #define OSM_DEFAULT_RETRY_COUNT 3
+-#define OSM_UMAD_MAX_CAS	32
++#define OSM_UMAD_MAX_CAS	UMAD_MAX_HCAS
+ #define OSM_UMAD_MAX_PORTS_PER_CA	2
+ #define OSM_UMAD_MAX_AGENTS	32
+ 
 diff -r -u /tmp/opensm-3.3.9/configure opensm-3.3.9/configure
 --- /tmp/opensm-3.3.9/configure	Mon Mar  7 00:01:54 2011
 +++ opensm-3.3.9/configure	Mon Mar 14 03:13:28 2011