1 /* |
1 /* |
2 * Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved. |
2 * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. |
3 * |
3 * |
4 * This software is available to you under a choice of one of two |
4 * This software is available to you under a choice of one of two |
5 * licenses. You may choose to be licensed under the terms of the GNU |
5 * licenses. You may choose to be licensed under the terms of the GNU |
6 * General Public License (GPL) Version 2, available from the file |
6 * General Public License (GPL) Version 2, available from the file |
7 * COPYING in the main directory of this source tree, or the |
7 * COPYING in the main directory of this source tree, or the |
99 #define MLX4_UVERBS_MAX_ABI_VERSION 3 /* mlx4-abi.h */ |
99 #define MLX4_UVERBS_MAX_ABI_VERSION 3 /* mlx4-abi.h */ |
100 #define RDMA_USER_CM_MIN_ABI_VERSION 3 /* rdma_cma_abi.h */ |
100 #define RDMA_USER_CM_MIN_ABI_VERSION 3 /* rdma_cma_abi.h */ |
101 #define RDMA_USER_CM_MAX_ABI_VERSION 4 /* rdma_cma_abi.h */ |
101 #define RDMA_USER_CM_MAX_ABI_VERSION 4 /* rdma_cma_abi.h */ |
102 |
102 |
103 #define MLX4 0 |
103 #define MLX4 0 |
104 #define MAX_HCAS 260 |
|
105 #define MAX_HCA_PORTS 16 |
|
106 #define HW_DRIVER_MAX_NAME_LEN 20 |
104 #define HW_DRIVER_MAX_NAME_LEN 20 |
107 #define UVERBS_KERNEL_SYSFS_NAME_BASE "uverbs" |
105 #define UVERBS_KERNEL_SYSFS_NAME_BASE "uverbs" |
108 #define UMAD_KERNEL_SYSFS_NAME_BASE "umad" |
106 #define UMAD_KERNEL_SYSFS_NAME_BASE "umad" |
109 #define IB_HCA_DEVPATH_PREFIX "/dev/infiniband/hca" |
107 #define IB_HCA_DEVPATH_PREFIX "/dev/infiniband/hca" |
110 #define IB_OFS_DEVPATH_PREFIX "/dev/infiniband/ofs" |
108 #define IB_OFS_DEVPATH_PREFIX "/dev/infiniband/ofs" |
116 #define PCI_DEVICE_ID_MELLANOX_HERMON_QDR 0x6354 |
114 #define PCI_DEVICE_ID_MELLANOX_HERMON_QDR 0x6354 |
117 #define PCI_DEVICE_ID_MELLANOX_HERMON_DDR_PCIE2 0x6732 |
115 #define PCI_DEVICE_ID_MELLANOX_HERMON_DDR_PCIE2 0x6732 |
118 #define PCI_DEVICE_ID_MELLANOX_HERMON_QDR_PCIE2 0x673c |
116 #define PCI_DEVICE_ID_MELLANOX_HERMON_QDR_PCIE2 0x673c |
119 #define INFINIHOST_DEVICE_ID_2 0x5a45 |
117 #define INFINIHOST_DEVICE_ID_2 0x5a45 |
120 #define INFINIHOST_DEVICE_ID_4 0x6279 |
118 #define INFINIHOST_DEVICE_ID_4 0x6279 |
|
119 |
|
120 #define MAX_HCAS (64*16) |
|
121 #define MAX_PORTS (MAX_HCAS*2) |
121 |
122 |
122 /* |
123 /* |
123 * sol_uverbs_drv_status is the status of what libibverbs knows |
124 * sol_uverbs_drv_status is the status of what libibverbs knows |
124 * about the status of sol_uverbs driver. |
125 * about the status of sol_uverbs driver. |
125 */ |
126 */ |
162 uint_t ibd_hw_rev; |
163 uint_t ibd_hw_rev; |
163 char ibd_node_guid_str[20]; |
164 char ibd_node_guid_str[20]; |
164 char ibd_node_guid_external_str[20]; |
165 char ibd_node_guid_external_str[20]; |
165 char ibd_sys_image_guid[20]; |
166 char ibd_sys_image_guid[20]; |
166 char ibd_fw_ver[16]; |
167 char ibd_fw_ver[16]; |
167 char ibd_name[8]; |
168 char ibd_name[16]; |
168 int ibd_boardid_index; |
169 int ibd_boardid_index; |
169 uint_t ibd_device_id; |
170 uint_t ibd_device_id; |
170 } ibdev_cache_info_t; |
171 } ibdev_cache_info_t; |
171 |
172 |
172 /* hermon - hence 2 */ |
173 /* hermon - hence 2 */ |
176 uint_t uvc_valid; |
177 uint_t uvc_valid; |
177 uint_t uvc_ibdev_abi_version; |
178 uint_t uvc_ibdev_abi_version; |
178 uint_t uvc_vendor_id; |
179 uint_t uvc_vendor_id; |
179 uint_t uvc_device_id; |
180 uint_t uvc_device_id; |
180 int uvc_hca_instance; |
181 int uvc_hca_instance; |
181 char uvc_ibdev_name[8]; |
182 char uvc_ibdev_name[16]; |
182 char uvc_ibdev_hca_path[MAXPATHLEN]; |
183 char uvc_ibdev_hca_path[MAXPATHLEN]; |
183 } uverbs_cache_info_t; |
184 } uverbs_cache_info_t; |
184 static uverbs_cache_info_t uverbs_dev_cache[MAX_HCAS]; |
185 static uverbs_cache_info_t uverbs_dev_cache[MAX_HCAS]; |
185 static int uverbs_abi_version = -1; |
186 static int uverbs_abi_version = -1; |
186 |
187 |
187 typedef struct umad_cache_info_s { |
188 typedef struct umad_cache_info_s { |
188 uint_t umc_valid; |
189 uint_t umc_valid; |
189 int umc_port; |
190 int umc_port; |
190 char umc_ib_dev[16]; |
191 char umc_ib_dev[16]; |
191 } umad_cache_info_t; |
192 } umad_cache_info_t; |
192 static umad_cache_info_t umad_dev_cache[MAX_HCAS * MAX_HCA_PORTS]; |
193 static umad_cache_info_t umad_dev_cache[MAX_PORTS]; |
193 static int umad_abi_version = -1; |
194 static int umad_abi_version = -1; |
194 |
195 |
195 pthread_once_t oneTimeInit = PTHREAD_ONCE_INIT; |
196 pthread_once_t oneTimeInit = PTHREAD_ONCE_INIT; |
196 static int umad_cache_cnt = 0; |
197 static int umad_cache_cnt = 0; |
197 static int ibdev_cache_cnt = 0; |
198 static int ibdev_cache_cnt = 0; |
234 } |
235 } |
235 |
236 |
236 static int |
237 static int |
237 umad_cache_add(uint_t dev_num, int port, char *ibdev) |
238 umad_cache_add(uint_t dev_num, int port, char *ibdev) |
238 { |
239 { |
239 if ((dev_num >= (MAX_HCAS * MAX_HCA_PORTS)) || |
240 if ((dev_num >= MAX_PORTS) || (umad_cache_cnt >= MAX_PORTS)) { |
240 (umad_cache_cnt >= (MAX_HCAS * MAX_HCA_PORTS))) { |
|
241 fprintf(stderr, "dev %d: exceeds umad cache size\n", dev_num); |
241 fprintf(stderr, "dev %d: exceeds umad cache size\n", dev_num); |
242 return (1); |
242 return (1); |
243 } |
243 } |
244 |
244 |
245 umad_dev_cache[dev_num].umc_port = port; |
245 umad_dev_cache[dev_num].umc_port = port; |
369 uverbs_cache_init() |
369 uverbs_cache_init() |
370 { |
370 { |
371 uverbs_cache_info_t info; |
371 uverbs_cache_info_t info; |
372 int dev_num, fd, i, bufsize, hca_cnt; |
372 int dev_num, fd, i, bufsize, hca_cnt; |
373 char uverbs_devpath[MAXPATHLEN]; |
373 char uverbs_devpath[MAXPATHLEN]; |
374 #ifndef IB_USER_VERBS_V2_IN_V1 |
|
375 sol_uverbs_info_t *uverbs_infop; |
374 sol_uverbs_info_t *uverbs_infop; |
376 sol_uverbs_hca_info_t *hca_infop; |
375 sol_uverbs_hca_info_t *hca_infop; |
377 #else |
|
378 sol_uverbs_info_v2_t *uverbs_infop; |
|
379 sol_uverbs_hca_info_v2_t *hca_infop; |
|
380 #endif |
|
381 char *buf; |
376 char *buf; |
382 |
377 |
383 snprintf(uverbs_devpath, MAXPATHLEN, "%s/%s%d", |
378 snprintf(uverbs_devpath, MAXPATHLEN, "%s/%s%d", |
384 IB_OFS_DEVPATH_PREFIX, UVERBS_KERNEL_SYSFS_NAME_BASE, |
379 IB_OFS_DEVPATH_PREFIX, UVERBS_KERNEL_SYSFS_NAME_BASE, |
385 sol_uverbs_minor_dev); |
380 sol_uverbs_minor_dev); |
392 fprintf(stderr, "sol_uverbs failed to open: %s\n", |
387 fprintf(stderr, "sol_uverbs failed to open: %s\n", |
393 strerror(errno)); |
388 strerror(errno)); |
394 goto error_exit1; |
389 goto error_exit1; |
395 } |
390 } |
396 |
391 |
397 #ifndef IB_USER_VERBS_V2_IN_V1 |
|
398 bufsize = sizeof (sol_uverbs_info_t) + sizeof (sol_uverbs_hca_info_t) * |
392 bufsize = sizeof (sol_uverbs_info_t) + sizeof (sol_uverbs_hca_info_t) * |
399 MAX_HCAS; |
393 MAX_HCAS; |
400 #else |
|
401 bufsize = sizeof (sol_uverbs_info_v2_t) + |
|
402 sizeof (sol_uverbs_hca_info_v2_t) * MAX_HCAS; |
|
403 #endif |
|
404 buf = malloc(bufsize); |
394 buf = malloc(bufsize); |
405 memset(buf, 0, bufsize); |
395 memset(buf, 0, bufsize); |
406 #ifndef IB_USER_VERBS_V2_IN_V1 |
|
407 uverbs_infop = (sol_uverbs_info_t *)buf; |
396 uverbs_infop = (sol_uverbs_info_t *)buf; |
408 #else |
|
409 uverbs_infop = (sol_uverbs_info_v2_t *)buf; |
|
410 #endif |
|
411 uverbs_infop->uverbs_hca_cnt = MAX_HCAS; |
397 uverbs_infop->uverbs_hca_cnt = MAX_HCAS; |
412 |
398 |
413 if (ioctl(fd, UVERBS_IOCTL_GET_HCA_INFO, uverbs_infop) != 0) { |
399 if (ioctl(fd, UVERBS_IOCTL_GET_HCA_INFO, uverbs_infop) != 0) { |
414 fprintf(stderr, "sol_uverbs ioctl failed: %s\n", |
400 fprintf(stderr, "sol_uverbs ioctl failed: %s\n", |
415 strerror(errno)); |
401 strerror(errno)); |
482 int port_cnt, bufsize; |
468 int port_cnt, bufsize; |
483 char umad_devpath[MAXPATHLEN], *buf; |
469 char umad_devpath[MAXPATHLEN], *buf; |
484 sol_umad_ioctl_info_t *umad_infop; |
470 sol_umad_ioctl_info_t *umad_infop; |
485 sol_umad_ioctl_port_info_t *port_infop; |
471 sol_umad_ioctl_port_info_t *port_infop; |
486 |
472 |
487 for (minor = 0; minor < MAX_HCAS * MAX_HCA_PORTS; minor++) { |
473 for (minor = 0; minor < MAX_PORTS; minor++) { |
488 snprintf(umad_devpath, MAXPATHLEN, "%s/%s%d", |
474 snprintf(umad_devpath, MAXPATHLEN, "%s/%s%d", |
489 IB_OFS_DEVPATH_PREFIX, UMAD_KERNEL_SYSFS_NAME_BASE, |
475 IB_OFS_DEVPATH_PREFIX, UMAD_KERNEL_SYSFS_NAME_BASE, |
490 minor); |
476 minor); |
491 |
477 |
492 if ((fd = open(umad_devpath, O_RDWR)) > 0) |
478 if ((fd = open(umad_devpath, O_RDWR)) > 0) |
494 |
480 |
495 if ((! save_errno) && (errno != ENOENT)) |
481 if ((! save_errno) && (errno != ENOENT)) |
496 save_errno = errno; |
482 save_errno = errno; |
497 } |
483 } |
498 |
484 |
499 if ((minor == MAX_HCAS * MAX_HCA_PORTS) && (fd < 0)) { |
485 if ((minor == MAX_PORTS) && (fd < 0)) { |
500 if (! save_errno) |
486 if (! save_errno) |
501 save_errno = errno; |
487 save_errno = errno; |
502 fprintf(stderr, "failed to open sol_umad: %s\n", |
488 fprintf(stderr, "failed to open sol_umad: %s\n", |
503 strerror(save_errno)); |
489 strerror(save_errno)); |
504 return (0); |
490 return (0); |
505 } |
491 } |
506 |
492 |
507 bufsize = sizeof (sol_umad_ioctl_info_t) + |
493 bufsize = sizeof (sol_umad_ioctl_info_t) + |
508 (sizeof (sol_umad_ioctl_port_info_t) * MAX_HCAS * MAX_HCA_PORTS); |
494 (sizeof (sol_umad_ioctl_port_info_t) * MAX_PORTS); |
509 |
495 |
510 buf = malloc(bufsize); |
496 buf = malloc(bufsize); |
511 memset(buf, 0, bufsize); |
497 memset(buf, 0, bufsize); |
512 umad_infop = (sol_umad_ioctl_info_t *)buf; |
498 umad_infop = (sol_umad_ioctl_info_t *)buf; |
513 umad_infop->umad_port_cnt = MAX_HCAS * MAX_HCA_PORTS; |
499 umad_infop->umad_port_cnt = MAX_PORTS; |
514 |
500 |
515 if (ioctl(fd, IB_USER_MAD_GET_PORT_INFO, umad_infop) != 0) { |
501 if (ioctl(fd, IB_USER_MAD_GET_PORT_INFO, umad_infop) != 0) { |
516 fprintf(stderr, "sol_umad ioctl failed: %s\n", |
502 fprintf(stderr, "sol_umad ioctl failed: %s\n", |
517 strerror(errno)); |
503 strerror(errno)); |
518 |
504 |
591 } |
577 } |
592 |
578 |
593 memset(&uverbs_dev_cache, 0, (sizeof (uverbs_cache_info_t) * MAX_HCAS)); |
579 memset(&uverbs_dev_cache, 0, (sizeof (uverbs_cache_info_t) * MAX_HCAS)); |
594 memset(&ibdev_cache, 0, (sizeof (ibdev_cache_info_t) * MAX_HCAS * 2)); |
580 memset(&ibdev_cache, 0, (sizeof (ibdev_cache_info_t) * MAX_HCAS * 2)); |
595 memset(&umad_dev_cache, 0, |
581 memset(&umad_dev_cache, 0, |
596 (sizeof (umad_cache_info_t) * MAX_HCAS * MAX_HCA_PORTS)); |
582 (sizeof (umad_cache_info_t) * MAX_PORTS)); |
597 |
583 |
598 initialized = B_TRUE; |
584 initialized = B_TRUE; |
599 } |
585 } |
600 |
586 |
601 /* |
587 /* |
1144 } |
1130 } |
1145 len = 1 + sprintf(buf, "%f", rate); |
1131 len = 1 + sprintf(buf, "%f", rate); |
1146 } else if (strcmp(path, "cap_mask") == 0) { |
1132 } else if (strcmp(path, "cap_mask") == 0) { |
1147 len = 1 + sprintf(buf, "0x%08x", |
1133 len = 1 + sprintf(buf, "0x%08x", |
1148 port_attr.port_cap_flags); |
1134 port_attr.port_cap_flags); |
|
1135 } else if (strcmp(path, "link_layer") == 0) { |
|
1136 switch (port_attr.link_layer) { |
|
1137 case IBV_LINK_LAYER_UNSPECIFIED: |
|
1138 case IBV_LINK_LAYER_INFINIBAND: |
|
1139 len = 1 + sprintf(buf, "%s", "IB"); |
|
1140 break; |
|
1141 case IBV_LINK_LAYER_ETHERNET: |
|
1142 len = |
|
1143 1 + sprintf(buf, "%s", "Ethernet"); |
|
1144 break; |
|
1145 default: |
|
1146 len = 1 + sprintf(buf, "%s", "Unknown"); |
|
1147 } |
1149 } |
1148 } |
1150 } |
1149 } |
1151 exit: |
1150 exit: |
1152 return (len); |
1151 return (len); |
1153 } |
1152 } |
1394 } |
1393 } |
1395 } |
1394 } |
1396 (void) pthread_mutex_unlock(&umad_cache_mutex); |
1395 (void) pthread_mutex_unlock(&umad_cache_mutex); |
1397 |
1396 |
1398 if (check_path(path, CP_UMAD, &dev_num)) { |
1397 if (check_path(path, CP_UMAD, &dev_num)) { |
1399 if (dev_num >= MAX_HCAS * MAX_HCA_PORTS) { |
1398 if (dev_num >= MAX_PORTS) { |
1400 fprintf(stderr, "Invalid Path: %s\n", path); |
1399 fprintf(stderr, "Invalid Path: %s\n", path); |
1401 goto exit; |
1400 goto exit; |
1402 } |
1401 } |
1403 if (!umad_dev_cache[dev_num].umc_valid) { |
1402 if (!umad_dev_cache[dev_num].umc_valid) { |
1404 goto exit; |
1403 goto exit; |