# HG changeset patch # User boris.chiu@oracle.com # Date 1462848089 25200 # Node ID 9032c8d91972b19b9fef547f69206233253ac7e4 # Parent b08608def60729b268720c58267f0c9d087b7cfb PSARC/2016/182 IB HCA driver interface and other updates 22567714 solaris_set_nodedesc interfaces need to be HCA agnostic(Userland) 23102715 infiniband-diags fails to build on s12-97 (nightly) diff -r b08608def607 -r 9032c8d91972 components/open-fabrics/infiniband-diags/manpages/set_nodedesc.sh.1m --- a/components/open-fabrics/infiniband-diags/manpages/set_nodedesc.sh.1m Thu May 05 02:19:04 2016 -0700 +++ b/components/open-fabrics/infiniband-diags/manpages/set_nodedesc.sh.1m Mon May 09 19:41:29 2016 -0700 @@ -1,9 +1,9 @@ '\" t -.\" Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. +.\" Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. .\" Modified for Solaris to add the Solaris stability classification, .\" and to add a note about source availability. .\" -.TH SET_NODEDESC.SH 1M "4 April 2011" "USER COMMANDS" +.TH SET_NODEDESC.SH 1M "21 Oct 2015" "USER COMMANDS" .SH "NAME" set_nodedesc.sh \- Set/View Node Description String for Infiniband Host Controller Adapters (HCA)s @@ -37,8 +37,8 @@ will be used if common node descriptor is not set. .sp \fBset_nodedesc.sh\fR called without arguments, sets the common node descriptor -string to the \fBhostname\fR(1M) of the system. The common Node descriptor string -can be set to a string other than the \fBhostname\fR(1M) using the -N option. +string to the \fBhostname\fR(1) of the system. The common Node descriptor string +can be set to a string other than the \fBhostname\fR(1) using the -N option. .sp The HCA specific Node Description String can be set using the -H and -G option. The Global Unique IDentifier (GUID) specified with -G identifies @@ -47,11 +47,12 @@ description String. .sp The -v option can be used to view the Node descriptors for all HCAs on the -system. +system. It will fail if no node description has been written on the system +previously. .sp .SH "EXAMPLES" .TP -\fB1. Set Node descriptor string to hostname(1M) for all HCAs:\fR +\fB1. Set Node descriptor string to hostname(1) for all HCAs:\fR .PP set_nodedesc.sh .PP diff -r b08608def607 -r 9032c8d91972 components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c --- a/components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c Thu May 05 02:19:04 2016 -0700 +++ b/components/open-fabrics/infiniband-diags/solaris_set_nodedesc.c Mon May 09 19:41:29 2016 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -35,57 +35,58 @@ */ #if defined(__SVR4) && defined(__sun) -#pragma ident "@(#)solaris_set_nodedesc.c 1.2 11/01/25 SMI" - #include #include +#include #include #include #include -#include -#include #include #include #include #include #include -#include #include #include #include #include - -#include +#include "ibdiag_common.h" +#include /* - * Local defines for HCA driver IOCTLs, used while + * Local defines for sol_uverbs IOCTLs, used while * building on build system without the change in * header files. */ -#ifdef HERMON_NODEDESC_UPDATE_STR -#define HERMON_NODEDESC_UPDATE_STRING 0x00000001 -#endif -#ifndef HERMON_NODEDESC_UPDATE_HCA_STRING -#define HERMON_NODEDESC_UPDATE_HCA_STRING 0x00000002 -#undef HERMON_NODEDESC_UPDATE_HCA_MAP -#endif -#ifndef HERMON_IOCTL_GET_NODEDESC -#define HERMON_IOCTL_GET_NODEDESC (('t' << 8) | 0x31) +#ifndef UVERBS_NODEDESC_UPDATE_STRING +#define UVERBS_IOCTL_GET_NODEDESC ('v' << 8) | 0x04 +#define UVERBS_IOCTL_SET_NODEDESC ('v' << 8) | 0x05 +#define UVERBS_NODEDESC_UPDATE_STRING 0x00000001 +#define UVERBS_NODEDESC_UPDATE_HCA_STRING 0x00000002 + +typedef struct sol_uverbs_nodedesc_s { + int32_t uverbs_solaris_abi_version; + char node_desc_str[64]; + uint32_t node_desc_update_flag; +} sol_uverbs_nodedesc_t; #endif -#define NODEDESC_UPDATE_STRING 0x00000001 -#define NODEDESC_UPDATE_HCA_STRING 0x00000002 +/* + * Override verbs abi version. + * If the build system doesn't have the intended + * header file then override with the intended abi version. + * These changes can be deleted once the build system has + * the correct header file. + */ +#if (IB_USER_VERBS_SOLARIS_ABI_VERSION == 2) +#undef IB_USER_VERBS_SOLARIS_ABI_VERSION +#define IB_USER_VERBS_SOLARIS_ABI_VERSION 3 +#endif + #define NODEDESC_READ 0x80000000 -#include "ibdiag_common.h" -static char *devpath_prefix = "/devices"; -static char *devpath_suffix = ":devctl"; -static char *ib_hca_driver_list[] = { - "hermon", NULL -}; -static di_node_t di_rootnode; char *argv0 = "solaris_set_nodedesc"; static struct nodedesc_read_info_s { @@ -95,7 +96,14 @@ boolean_t ofuv_name_valid; char ofuv_name[64]; } nd_read_info_arr[MAX_HCAS]; -int nd_read_info_cnt; +int nd_read_info_cnt = 0; + +static int +read_nodedesc_ioctl(struct ibv_context *context, + sol_uverbs_nodedesc_t *nodedesc); +static int +write_nodedesc_ioctl(struct ibv_context *context, + sol_uverbs_nodedesc_t *nodedesc); static void print_read_info() @@ -113,24 +121,18 @@ } static void -update_read_info_hwnames() +update_read_info_hwnames(struct ibv_device **dev_list, int num_devices) { - struct ibv_device **dev_list; - int num_devices, i; + int i; uint64_t dev_guid; char *dev_name; size_t dev_name_len; - dev_list = ibv_get_device_list(&num_devices); - if (!dev_list) { - fprintf(stderr, "No IB devices found\n"); - return; - } - - for (i = 0; i < num_devices; ++i) { + for (i = 0; dev_list[i] != 0 && i < num_devices; ++i) { int j; - dev_guid = (uint64_t)ntohll(ibv_get_device_guid(dev_list[i])); + dev_guid = (uint64_t)ntohll( + ibv_get_device_guid(dev_list[i])); dev_name = (char *)ibv_get_device_name(dev_list[i]); dev_name_len = strlen(dev_name) + 1; for (j = 0; j < nd_read_info_cnt; j++) { @@ -138,13 +140,12 @@ nd_read_info_arr[j].guid == dev_guid) { memcpy(nd_read_info_arr[j].ofuv_name, dev_name, dev_name_len); - nd_read_info_arr[j].ofuv_name_valid = B_TRUE; + nd_read_info_arr[j].ofuv_name_valid = + B_TRUE; break; } } } - - ibv_free_device_list(dev_list); } static void @@ -155,195 +156,233 @@ nd_len = strlen(nd_str) + 1; nd_read_info_arr[nd_read_info_cnt].info_valid = B_TRUE; nd_read_info_arr[nd_read_info_cnt].guid = guid; - memcpy(nd_read_info_arr[nd_read_info_cnt].nd_string, nd_str, nd_len); + memcpy(nd_read_info_arr[nd_read_info_cnt].nd_string, + nd_str, nd_len); nd_read_info_cnt++; } static void -do_driver_read_ioctl(char *drivername) +do_driver_read_ioctl(struct ibv_device *device) { - di_node_t hcanode, childnode; - char *devpath; - char *access_devname; - int devlength, devfd, rc = -1; - uint64_t *hca_guid; + int rc; + uint64_t hca_guid; + struct ibv_context *context; + sol_uverbs_nodedesc_t *nodedescp; + + /* Get the context for the device */ + context = ibv_open_device(device); + if (!context) { + IBERROR("Unable to open the device.\n"); + /* NOTREACHED */ + } - if ((hcanode = di_drv_first_node(drivername, di_rootnode)) - == DI_NODE_NIL) { - return; + if (context->device != device) { + IBERROR("Device not set.\n"); + /* NOTREACHED */ + } + + /* Allocate the memory for node descriptor */ + nodedescp = (sol_uverbs_nodedesc_t *)malloc( + sizeof (sol_uverbs_nodedesc_t)); + if (nodedescp == NULL) { + IBERROR("Memory allocation failed.\n"); + /* NOTREACHED */ } - while (hcanode != DI_NODE_NIL) { - childnode = di_child_node(hcanode); - while (childnode != DI_NODE_NIL) { - if (di_prop_lookup_int64(DDI_DEV_T_ANY, - childnode, "hca-guid", - (int64_t **)&hca_guid) != 1) { - childnode = di_sibling_node(childnode); - continue; - } else { - break; - } - } - if (childnode == DI_NODE_NIL) { - hcanode = di_drv_next_node(hcanode); - continue; - } + nodedescp->uverbs_solaris_abi_version = + IB_USER_VERBS_SOLARIS_ABI_VERSION; - devpath = di_devfs_path(hcanode); - devlength = strlen(devpath_prefix) + strlen(devpath) + - strlen(devpath_suffix) + 2; - access_devname = malloc(devlength); - (void) snprintf(access_devname, devlength, "%s%s%s", - devpath_prefix, devpath, devpath_suffix); - if ((devfd = open(access_devname, O_RDONLY)) < 0) { - IBERROR("open device file %s failed", access_devname); - free(access_devname); - hcanode = di_drv_next_node(hcanode); - continue; - } - if (strcmp(drivername, "hermon") == 0) { - hermon_nodedesc_ioctl_t nodedesc_ioctl; - - if ((rc = ioctl(devfd, HERMON_IOCTL_GET_NODEDESC, - (void *)&nodedesc_ioctl)) != 0) { - IBERROR("hermon ioctl failure"); - free(access_devname); - close(devfd); - hcanode = di_drv_next_node(hcanode); - continue; - } - add_read_info_arr((char *)nodedesc_ioctl.node_desc_str, - *hca_guid); - } else { - IBERROR("drivername != hermon: %s", drivername); - } - - free(access_devname); - close(devfd); - hcanode = di_drv_next_node(hcanode); + /* Get the guid for the device */ + hca_guid = (uint64_t)ntohll(ibv_get_device_guid(device)); + if (!hca_guid) { + IBERROR("ibv_get_device_guid failed.\n"); + /* NOTREACHED */ } + /* Read node descriptor */ + rc = read_nodedesc_ioctl(context, nodedescp); + if (rc != 0) { + IBERROR("Failed to read node descriptor.\n"); + /* NOTREACHED */ + } + + add_read_info_arr((char *)nodedescp->node_desc_str, + hca_guid); + +read_nodedesc_exit_1: + /* release the allocated memory */ + free(nodedescp); +read_nodedesc_exit_2: + /* Close the device */ + ibv_close_device(context); } static int -do_driver_update_ioctl(char *drivername, char *node_desc, char *hca_desc, - uint64_t inp_hca_guid, uint32_t update_flag) +do_driver_update_ioctl(struct ibv_device *device, char *node_desc, + char *hca_desc, uint32_t update_flag) { - di_node_t hcanode, childnode; - char *devpath; - char *access_devname; - int devlength, devfd, rc = -1; - uint64_t *hca_guid; - char *desc_str = (node_desc ? node_desc : hca_desc); + int rc; + struct ibv_context *context; + sol_uverbs_nodedesc_t *nodedescp; + char *desc_str; - if ((hcanode = di_drv_first_node(drivername, di_rootnode)) - == DI_NODE_NIL) { - return (-1); + desc_str = (node_desc ? node_desc : hca_desc); + + /* Get context for the device */ + context = ibv_open_device(device); + if (!context) { + IBERROR("Unable to open the device.\n"); + /* NOTREACHED */ } - while (hca_desc && hcanode != DI_NODE_NIL) { - childnode = di_child_node(hcanode); - while (childnode != DI_NODE_NIL) { - if (di_prop_lookup_int64(DDI_DEV_T_ANY, - childnode, "hca-guid", - (int64_t **)&hca_guid) != 1) { - childnode = di_sibling_node(childnode); - continue; - } else { - break; - } - } - if (*hca_guid == inp_hca_guid) - break; - hcanode = di_drv_next_node(hcanode); + if (context->device != device) { + IBERROR("Device not set.\n"); + /* NOTREACHED */ } - if ((hca_desc && childnode == DI_NODE_NIL) || - hcanode == DI_NODE_NIL) { - IBERROR("matching GUID not found"); - return (-1); + /* Allocate the memory for node descriptor */ + nodedescp = (sol_uverbs_nodedesc_t *)malloc( + sizeof (sol_uverbs_nodedesc_t)); + if (nodedescp == NULL) { + IBERROR("Memory allocation failed.\n"); + /* NOTREACHED */ } - devpath = di_devfs_path(hcanode); - devlength = strlen(devpath_prefix) + strlen(devpath) + - strlen(devpath_suffix) + 2; - access_devname = malloc(devlength); - (void) snprintf(access_devname, devlength, "%s%s%s", - devpath_prefix, devpath, devpath_suffix); - if ((devfd = open(access_devname, O_RDONLY)) < 0) { - IBERROR("open device file %s failed", access_devname); - free(access_devname); - return (rc); - } - if (strcmp(drivername, "hermon") == 0) { - hermon_nodedesc_ioctl_t nodedesc_ioctl; + strncpy(nodedescp->node_desc_str, desc_str, 64); + nodedescp->node_desc_update_flag = update_flag; + nodedescp->uverbs_solaris_abi_version = + IB_USER_VERBS_SOLARIS_ABI_VERSION; - strncpy(nodedesc_ioctl.node_desc_str, desc_str, 64); - if (update_flag & NODEDESC_UPDATE_STRING) - nodedesc_ioctl.node_desc_update_flag = - HERMON_NODEDESC_UPDATE_STRING; - else if (update_flag & NODEDESC_UPDATE_HCA_STRING) - nodedesc_ioctl.node_desc_update_flag = - HERMON_NODEDESC_UPDATE_HCA_STRING; - else { - IBERROR("Invalid option"); - exit(-1); - } - if ((rc = ioctl(devfd, HERMON_IOCTL_SET_NODEDESC, - (void *)&nodedesc_ioctl)) != 0) { - IBERROR("hermon ioctl failure"); - } - } else { - IBERROR("drivername != hermon: %s", drivername); - } + rc = write_nodedesc_ioctl(context, nodedescp); + if (rc != 0) + IBERROR("Failed to set node descriptor.\n"); - free(access_devname); - close(devfd); + free(nodedescp); + +write_nodedesc_exit: + ibv_close_device(context); return (rc); } static void -read_nodedesc() +read_nodedesc(struct ibv_device **device_list, int num_devices) { - int i; + int i; - if ((di_rootnode = di_init("/", DINFOCPYALL | DINFOFORCE)) - == DI_NODE_NIL) { - IBERROR("read_nodedesc di_init failure"); - return; - } - for (i = 0; ib_hca_driver_list[i]; i++) - do_driver_read_ioctl(ib_hca_driver_list[i]); - di_fini(di_rootnode); + for (i = 0; device_list[i] != 0 && i < num_devices; i++) + do_driver_read_ioctl(device_list[i]); } static int -update_nodedesc(char *cmn_nodedesc, char *hca_nodedesc, uint64_t guid, +update_nodedesc(struct ibv_device **device_list, int num_devices, + char *cmn_nodedesc, char *hca_nodedesc, uint64_t guid, uint32_t update_flag) { - int i, rc = 0; + int i, rc = -1; + uint64_t dev_guid; + boolean_t matched = B_FALSE; + + if (cmn_nodedesc && hca_nodedesc == NULL) { + for (i = 0; i < num_devices; i++) { + rc = do_driver_update_ioctl(device_list[i], + cmn_nodedesc, hca_nodedesc, update_flag); + if (rc != 0) + continue; + else + break; + } + + if (rc != 0 && i == num_devices) + IBERROR("Failed to set the node descriptor.\n"); + + return (rc); + } - if ((di_rootnode = di_init("/", DINFOCPYALL | DINFOFORCE)) - == DI_NODE_NIL) { - IBERROR("di_init failure"); + if (hca_nodedesc && guid) { + for (i = 0; i < num_devices; i++) { + dev_guid = (uint64_t)ntohll(ibv_get_device_guid( + device_list[i])); + if (!dev_guid) { + continue; + } + if (dev_guid == guid) { + matched = B_TRUE; + rc = do_driver_update_ioctl(device_list[i], + cmn_nodedesc, hca_nodedesc, update_flag); + break; + } else { + continue; + } + } + + if (matched == B_FALSE) { + IBERROR("No guid matched.\n"); + /* NOTREACHED */ + } + + if (rc != 0) { + IBERROR("Failed to set the node descriptor.\n"); + /* NOTREACHED */ + } + } + + return (rc); +} + +static int +read_nodedesc_ioctl(struct ibv_context *context, + sol_uverbs_nodedesc_t *nodedesc) +{ + + int ret; + + /* + * Use ioctl call to sol_uverbs module. + */ + if (!context || !nodedesc) { return (-1); } - for (i = 0; ib_hca_driver_list[i]; i++) { - rc = do_driver_update_ioctl(ib_hca_driver_list[i], - cmn_nodedesc, hca_nodedesc, guid, - update_flag); - if (!rc) - break; + + ret = ioctl(context->cmd_fd, UVERBS_IOCTL_GET_NODEDESC, nodedesc); + if (ret != 0) { + if (ret == EINVAL) + IBERROR("ABI version check failed.\n"); + else + IBERROR("UVERBS_IOCTL_GET_NODEDESC ioctl failed.\n"); + + /* NOTREACHED */ } - if (rc) - IBERROR("Updated failed for all HCA drivers"); + + return (0); +} + +static int +write_nodedesc_ioctl(struct ibv_context *context, + sol_uverbs_nodedesc_t *nodedesc) +{ + int ret; - di_fini(di_rootnode); - return (rc); + /* + * Use ioctl call to sol_uverbs module. + */ + if (!context || !nodedesc) + return (-1); + + ret = ioctl(context->cmd_fd, UVERBS_IOCTL_SET_NODEDESC, nodedesc); + if (ret != 0) { + if (ret == EINVAL) + IBERROR("ABI version check failed.\n"); + else + IBERROR("UVERBS_IOCTL_SET_NODEDESC ioctl failed.\n"); + + /* NOTREACHED */ + } + + return (0); } + static void usage(void) { @@ -429,14 +468,17 @@ int main(int argc, char **argv) { - int rc; - char *nodedesc = NULL, *hcadesc = NULL; - uint32_t update_flag = 0; - struct utsname uts_name; - uint64_t hca_guid; - boolean_t guid_inited = B_FALSE; - extern int ibdebug; - char nodename[64]; + int rc; + char *nodedesc = NULL; + char *hcadesc = NULL; + uint32_t update_flag = 0; + struct utsname uts_name; + uint64_t hca_guid; + boolean_t guid_inited = B_FALSE; + extern int ibdebug; + char nodename[64]; + struct ibv_device **device_list = NULL; + int num_devices = 0; static char const str_opts[] = "N:H:G:vd"; static const struct option long_opts[] = { @@ -460,18 +502,18 @@ if (!nodedesc) { usage(); rc = -1; - goto free_and_ret; + goto free_and_ret_2; } - update_flag |= NODEDESC_UPDATE_STRING; + update_flag |= UVERBS_NODEDESC_UPDATE_STRING; break; case 'H': hcadesc = nodedesc_substr_cat(argv, argc, B_FALSE); if (!hcadesc) { usage(); rc = -1; - goto free_and_ret; + goto free_and_ret_2; } - update_flag |= NODEDESC_UPDATE_HCA_STRING; + update_flag |= UVERBS_NODEDESC_UPDATE_HCA_STRING; break; case 'G': guid_inited = B_TRUE; @@ -486,7 +528,7 @@ default: usage(); rc = -1; - goto free_and_ret; + goto free_and_ret_2; } } @@ -494,42 +536,53 @@ if (nodedesc || hcadesc || guid_inited == B_TRUE) { usage(); rc = -1; - goto free_and_ret; + goto free_and_ret_2; } - read_nodedesc(); - update_read_info_hwnames(); + device_list = ibv_get_device_list(&num_devices); + if (!device_list) { + IBERROR("ibv_get_device_list failed.\n"); + /* NOTREACHED */ + } + + read_nodedesc(device_list, num_devices); + update_read_info_hwnames(device_list, num_devices); print_read_info(); - return (0); + rc = 0; + goto free_and_ret_1; } if (hcadesc && guid_inited == B_FALSE) { IBERROR("No GUID specified for HCA Node descriptor"); - usage(); - rc = -1; - goto free_and_ret; + /* NOTREACHED */ + } + + device_list = ibv_get_device_list(&num_devices); + if (!device_list) { + IBERROR("ibv_get_device_list failed.\n"); + /* NOTREACHED */ } if (nodedesc) { - rc = update_nodedesc(nodedesc, NULL, 0, - NODEDESC_UPDATE_STRING); + rc = update_nodedesc(device_list, num_devices, + nodedesc, NULL, 0, UVERBS_NODEDESC_UPDATE_STRING); if (rc) { IBERROR("write common node descriptor " "failed"); - rc = -1; - goto free_and_ret; + /* NOTREACHED */ } } if (hcadesc) { - rc = update_nodedesc(NULL, hcadesc, hca_guid, - NODEDESC_UPDATE_HCA_STRING); + rc = update_nodedesc(device_list, num_devices, + NULL, hcadesc, hca_guid, + UVERBS_NODEDESC_UPDATE_HCA_STRING); if (rc) { IBERROR("update_hca_noddesc failed"); - rc = -1; - goto free_and_ret; + /* NOTREACHED */ } - return (0); + rc = 0; + goto free_and_ret_1; } @@ -537,8 +590,7 @@ if (uname(&uts_name) < 0) { IBERROR("Node descriptor unspecified" "& uts_name failed"); - rc = -1; - goto free_and_ret; + /* NOTREACHED */ } /* @@ -550,15 +602,19 @@ if (nodename[strlen(nodename)] != ' ') (void) strncat(nodename, " ", 1); - rc = update_nodedesc(nodename, NULL, 0, - NODEDESC_UPDATE_STRING); + rc = update_nodedesc(device_list, num_devices, + nodename, NULL, 0, + UVERBS_NODEDESC_UPDATE_STRING); if (rc) { IBERROR("write common node descriptor failed"); - rc = -1; + /* NOTREACHED */ } } -free_and_ret: +free_and_ret_1: + ibv_free_device_list(device_list); + +free_and_ret_2: if (nodedesc) free(nodedesc); if (hcadesc)