# This patch does the src changes specific to solaris.
# It is developed By solaris PSIF team. We plan to have a common upstream repo
# and submit these changes to it, but do not yet have a target date of doing it.
diff -r d391efa34f11 src/cq.c
--- a/src/cq.c Thu Aug 25 02:14:25 2016 -0700
+++ b/src/cq.c Thu Sep 29 05:56:09 2016 -0700
@@ -43,7 +43,9 @@
#include <strings.h>
#include <pthread.h>
#include <errno.h>
+#if !(defined(__SVR4) && defined(__sun))
#include <endian.h>
+#endif
#include <string.h>
#include "sif.h"
@@ -153,7 +155,7 @@
qp->sq.sw->head_seq = sq_seq_num;
sif_log(uc, SIF_CQ,
- "wr_id 0x%lx on qp/sq %d sq_seq_num %d",
+ "wr_id 0x%"PRIx64" on qp/sq %d sq_seq_num %d",
wc->wr_id, cqe->qp, sq_seq_num);
if (uc->debug_mask & SIF_SQSOVRN)
@@ -225,7 +227,7 @@
}
#endif
/* TBD: Handle sg list case */
- sif_log(uc, SIF_CQ,"wr_id 0x%lx queue len %d", wc->wr_id, rq_len);
+ sif_log(uc, SIF_CQ,"wr_id 0x%"PRIx64" queue len %d", wc->wr_id, rq_len);
return 0;
}
@@ -269,7 +271,7 @@
*/
sif_log(uc, log_level,
"error compl on cq %d, seq_num %d qpn 0x%x, "
- "wc_id %Ld status %s(%d), vendor_err 0x%x",
+ "wc_id %"SIFPRIx64" status %s(%d), vendor_err 0x%x",
cq->index, lcqe.seq_num, qpn, lcqe.wc_id.rq_id,
string_enum_psif_wc_status(lcqe.status),
lcqe.status, lcqe.vendor_err);
diff -r d391efa34f11 src/encoding.h
--- a/src/encoding.h Thu Aug 25 02:14:25 2016 -0700
+++ b/src/encoding.h Thu Sep 29 05:56:09 2016 -0700
@@ -37,19 +37,30 @@
#define _SIF_ENCODING_H
#include <sys/types.h>
+#if !(defined(__SVR4) && defined(__sun))
#include "kernel_types.h"
+#endif
#include "psif_hw_data.h"
#define true 1
#define false 0
#define bool int
+#define _BIT(nr) (1UL << (nr))
+#if defined(__SVR4) && defined(__sun)
+#include <sys/atomic.h>
+typedef volatile uint32_t atomic_t;
+#define test_bit(ptr, flags) (*ptr & _BIT(flags))
+#define set_bit(ptr, flags) atomic_or_64(ptr, _BIT(flags))
+#define atomic_inc_return(ptr) atomic_inc_32_nv(ptr)
+#define atomic_dec_return(ptr) atomic_dec_32_nv(ptr)
+#else
typedef volatile __u32 atomic_t;
-#define _BIT(nr) (1UL << (nr))
#define test_bit(ptr, flags) __sync_and_and_fetch(ptr, _BIT(flags))
#define set_bit(ptr, flags) __sync_or_and_fetch(ptr, _BIT(flags))
#define atomic_inc_return(ptr) __sync_add_and_fetch(ptr, 1)
#define atomic_dec_return(ptr) __sync_sub_and_fetch(ptr, 1)
+#endif
#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
#define PSIF_WC_STATUS_DUPL_COMPL_ERR (PSIF_WC_STATUS_FIELD_MAX-1)
@@ -92,11 +103,18 @@
#define min(x,y) ((x) > (y) ? (y) : (x))
#if (EPSC_VERSION < EPSC_API_VERSION(1,5))
+#if defined(__SVR4) && defined(__sun)
+#define cpu_to_be32 htonl
+#define cpu_to_be64 htonll
+#define be32_to_cpu ntohl
+#define be64_to_cpu ntohll
+#else
#define cpu_to_be32 htobe32
#define cpu_to_be64 htobe64
#define be32_to_cpu be32toh
#define be64_to_cpu be64toh
#endif
+#endif
#ifdef __x86_64
#undef wmb
diff -r d391efa34f11 src/hwprint.c
--- a/src/hwprint.c Thu Aug 25 02:14:25 2016 -0700
+++ b/src/hwprint.c Thu Sep 29 05:56:09 2016 -0700
@@ -34,6 +34,8 @@
* hwprint.c: User level wrapper for hardware (generated) struct print support
*/
#include <sys/types.h>
+#if !(defined(__SVR4) && defined(__sun))
#define OS_PRIx64 "lx"
+#endif
#include "sif.h"
#include "psif_hw_print.c"
diff -r d391efa34f11 src/mr.c
--- a/src/mr.c Thu Aug 25 02:14:25 2016 -0700
+++ b/src/mr.c Thu Sep 29 05:56:09 2016 -0700
@@ -45,6 +45,7 @@
#include "sif_abi.h"
#include "mr.h"
#include <assert.h>
+#include <errno.h>
struct ibv_mr *sif_reg_mr(struct ibv_pd *ibpd, void *addr,
size_t length, int access)
@@ -103,7 +104,7 @@
sctx->bypass_mr_ref = malloc(ref_sz);
if (!sctx->bypass_mr_ref) {
sif_log(sctx, SIF_INFO,
- "Failed to allocate 0x%lx bytes of storage for bypass MR references\n",
+ "Failed to allocate 0x%zx bytes of storage for bypass MR references\n",
ref_sz);
ret = ENOMEM;
goto bypass_ref_alloc_failed;
@@ -169,6 +170,37 @@
return 0;
}
+#if defined(__SVR4) && defined(__sun)
+/*
+ * Libibverbs will not validate the function ptr for non-NULL value for fmr
+ * callbacks.If undefined, application will core dump if it uses any of
+ * these calls.
+ */
+struct ibv_mr *sif_reg_mr_relaxed(struct ibv_pd *ibpd, void *addr, size_t length, int access)
+{
+ struct sif_context *sctx = to_sctx(ibpd->context);
+
+ sif_log(sctx, SIF_INFO, "sif_reg_relaxed_mr Not Implemented");
+ errno = ENOTSUP;
+
+ return NULL;
+}
+
+int sif_dereg_mr_relaxed(struct ibv_mr *ibmr)
+{
+ struct sif_context *sctx = to_sctx(ibmr->context);
+ sif_log(sctx, SIF_INFO, "sif_dereg_relaxed_mr Not Implemented");
+ return ENOTSUP;
+}
+
+int sif_flush_relaxed_mr(struct ibv_pd *ibpd)
+{
+ struct sif_context *sctx = to_sctx(ibpd->context);
+ sif_log(sctx, SIF_INFO, "sif_flush_relaxed_mr Not Implemented");
+ return ENOTSUP;
+}
+#endif
+
struct ibv_mw *sif_alloc_mw(struct ibv_pd *ibpd, enum ibv_mw_type type)
{
fprintf(stderr, PFX "%s Not implemented\n", __func__);
diff -r d391efa34f11 src/sif.c
--- a/src/sif.c Thu Aug 25 02:14:25 2016 -0700
+++ b/src/sif.c Thu Sep 29 05:56:09 2016 -0700
@@ -44,6 +44,7 @@
#include <sys/mman.h>
#include <pthread.h>
#include <string.h>
+#include <infiniband/verbs.h>
#ifndef HAVE_IBV_READ_SYSFS_FILE
#include <sys/types.h>
@@ -76,6 +77,8 @@
HCA(SUN, PSIF_VF),
HCA(SUN, SIBS_PF),
HCA(SUN, SIBS_VF),
+ HCA(SUN, SIBS2_PF),
+ HCA(SUN, SIBS2_VF),
};
static struct ibv_context_ops sif_ctx_ops = {
@@ -94,6 +97,12 @@
.rereg_mr = sif_rereg_mr,
.dereg_mr = sif_dereg_mr,
+#if defined(__SVR4) && defined(__sun)
+ .reg_mr_relaxed= sif_reg_mr_relaxed,
+ .dereg_mr_relaxed = sif_dereg_mr_relaxed,
+ .flush_relaxed_mr = sif_flush_relaxed_mr,
+#endif
+
.alloc_mw = sif_alloc_mw,
.bind_mw = sif_bind_mw,
.dealloc_mw = sif_dealloc_mw,
@@ -163,17 +172,32 @@
struct sif_device *sdev = to_sdev(ibdev);
char value[20];
char param_path[IBV_SYSFS_PATH_MAX];
+#if defined(__SVR4) && defined(__sun)
+/*
+ * See CR 22214822,22245798:collect buffer mode disabled for solaris
+ * SQ gives better performance on SPARC mode for
+ * smaller message sizes.
+ */
+ const char *cb_mode;
+ int mode;
+ const char *user_cb_disable;
+#endif
const char* dbm;
const char *cfum;
const char *odm;
const char *fast_copy_mnem;
const int feature_mnemonics_size = 500;
char feature_mnemonics[feature_mnemonics_size];
+#if defined(__SSE__) || defined(__AVX__)
int disa_x_wc;
+#endif
int ret = 0;
#ifdef HAVE_VERBS_REGISTER_DRIVER
context = to_sctx(ibv_ctx);
+#if defined(__SVR4) && defined(__sun)
+ ibv_ctx->device = &v_device->device;
+#endif
#else
context = malloc(sizeof(*context));
if (!context)
@@ -181,6 +205,9 @@
/* The sif specific part needs to be zeroed out */
memset(context, 0, sizeof(*context));
ibv_ctx = &context->ibv_ctx;
+#if defined(__SVR4) && defined(__sun)
+ context->ibv_ctx.device = ibdev;
+#endif
#endif
/* Satisfy valgrind */
@@ -211,8 +238,13 @@
(SIF_UVERBS_ABI_MINOR_VERSION & 0xff);
ibv_ctx->cmd_fd = cmd_fd;
+#if defined(__SVR4) && defined(__sun)
+ ret = ibv_cmd_get_context(ibv_ctx, &cmd.ibv_cmd,
+ sizeof cmd.ibv_cmd, &resp.ibv_resp, sizeof resp);
+#else
ret = ibv_cmd_get_context(ibv_ctx, &cmd.ibv_cmd,
sizeof cmd, &resp.ibv_resp, sizeof resp);
+#endif
if (ret)
goto err_free;
@@ -232,8 +264,22 @@
odm = getenv("SIF_OPT_DISABLE_MASK");
context->opt_disable_mask = odm ? strtoll(odm, NULL, 0) : 0;
+#if defined(__SSE__) || defined(__AVX__)
disa_x_wc = context->opt_disable_mask & SOD_EXPLICIT_WC;
+#endif
+#if defined(__SVR4) && defined(__sun)
+ user_cb_disable = getenv("SIF_DISABLE_USER_CB");
+ context->user_cb_disable = user_cb_disable ? strtoll(user_cb_disable, NULL, 0) : 0;
+ /*
+ * SQ mode is the default mode on solaris
+ * set env variable SIF_CB_MODE = 1 to change to cb_mode
+ */
+ cb_mode = getenv("SIF_CB_MODE");
+ mode = cb_mode ? strtol(cb_mode, NULL, 0) : 0;
+ context->default_flags |= mode? 0: SQ_mode;
+ sif_log(context, SIF_INFO, "Mode: %s", mode? "CB/SQ": "SQ");
+#endif
probe_cpu_features();
sif_log(context, SIF_VERBS, "CPU features detected: %s",
cpu_features_to_str(feature_mnemonics_size, feature_mnemonics));
@@ -391,6 +437,12 @@
dev->revision = revision;
dev->min_resp_ms = 0; /* Will be set by the first allocated context */
+#if defined(__SVR4) && defined(__sun)
+ /*
+ * SIFMT_BYPASS mode is not supported for userspace on Solaris, mr_size is
+ * not required */
+ dev->mr_size = 4096;
+#else
if (ibv_read_sysfs_file("/sys/module/sif", "parameters/mr_size",
value, sizeof value) < 0) {
fprintf(stderr, "Failed to read mr_size\n");
@@ -398,6 +450,7 @@
return NULL;
}
sscanf(value, "%i", &dev->mr_size);
+#endif
#ifndef HAVE_VERBS_REGISTER_DRIVER
dev->ibv_dev.ops = sif_dev_ops;
diff -r d391efa34f11 src/sif.h.in
--- a/src/sif.h.in Thu Aug 25 02:14:25 2016 -0700
+++ b/src/sif.h.in Thu Sep 29 05:56:09 2016 -0700
@@ -34,6 +34,9 @@
* sif.h: main entry points and initialization for userspace library
*/
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif /* HAVE_CONFIG_H */
#ifndef _SIF_H
#define _SIF_H
#if defined(__cplusplus)
@@ -54,6 +57,8 @@
#define PCI_DEVICE_ID_PSIF_VF 0x2089
#define PCI_DEVICE_ID_SIBS_PF 0x2188
#define PCI_DEVICE_ID_SIBS_VF 0x2189
+#define PCI_DEVICE_ID_SIBS2_PF 0x2198
+#define PCI_DEVICE_ID_SIBS2_VF 0x2199
#endif
#define PSIF_REVISION(sdevice) ((sdevice)->revision)
@@ -65,6 +70,15 @@
/* SIF_OPT_DISABLE_FLAGS: */
#define SOD_EXPLICIT_WC 0x1
+#if defined(__SVR4) && defined(__sun)
+#define SIF_PAGE_SHIFT 12
+#define SIF_PAGE_SIZE (1 << SIF_PAGE_SHIFT)
+
+#define SIFPRIx64 PRIx64
+#include "sif_user.h"
+#else
+#define SIFPRIx64 "llx"
+
enum sif_vendor_flags
{
MMU_special = 0x1, /* Use special mmu setup in associated mappings
@@ -103,6 +117,7 @@
FLUSH_SQ,
FLUSH_RQ
};
+#endif
struct sif_device {
struct verbs_device verbs_dev;
@@ -203,6 +218,9 @@
struct sif_mr **bypass_mr_ref;
int bypass_cnt; /* Number of bypass MRs initiated - to avoid slow lookup unless bypass */
uint64_t debug_mask;
+#if defined(__SVR4) && defined(__sun)
+ uint64_t user_cb_disable;
+#endif
uint64_t opt_disable_mask;
void (* fast_copy)(void *dst, const void *src, int len);
};
@@ -262,7 +280,11 @@
struct sif_mr {
struct ibv_mr ibv_mr;
+#if defined(__SVR4) && defined(__sun)
+ off64_t uv2dma;
+#else
off_t uv2dma;
+#endif
enum sif_mem_type mem_type;
};
@@ -373,6 +395,10 @@
struct ibv_pd *pd, void *addr, size_t length,
int access);
int sif_dereg_mr(struct ibv_mr *mr);
+struct ibv_mr *sif_reg_mr_relaxed(struct ibv_pd *pd, void *addr,
+ size_t length, int access);
+int sif_dereg_mr_relaxed(struct ibv_mr *mr);
+int sif_flush_relaxed_mr(struct ibv_pd *pd);
struct ibv_mw *sif_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type);
int sif_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, struct ibv_mw_bind *mw_bind);
int sif_dealloc_mw(struct ibv_mw *mw);
@@ -389,6 +415,8 @@
struct ibv_qp *sif_create_qp_ex(struct ibv_context *context,
struct ibv_qp_init_attr_ex *qp_init_attr_ex);
+struct ibv_srq *sif_create_srq_ex(struct ibv_context *context,
+ struct ibv_srq_init_attr_ex *srq_init_attr_ex);
int sif_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
int attr_mask, struct ibv_qp_init_attr *init_attr);
int sif_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
@@ -402,8 +430,6 @@
struct ibv_srq *sif_create_srq(struct ibv_pd *pd,
struct ibv_srq_init_attr *attr);
-struct ibv_srq *sif_create_srq_ex(struct ibv_context *context,
- struct ibv_srq_init_attr_ex *srq_init_attr_ex);
int sif_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr,
int attr_mask);
@@ -425,6 +451,11 @@
void sif_set_legacy_xrc(struct ibv_srq *srq, void *legacy_xrc_srq);
#endif
+#if HAVE_DECL_IBV_QPT_XRC
+void *sif_get_legacy_xrc(struct ibv_srq *srq);
+void sif_set_legacy_xrc(struct ibv_srq *srq, void *legacy_xrc_srq);
+#endif
+
#if defined(__cplusplus)
}
#endif
diff -r d391efa34f11 src/sndrcv.c
--- a/src/sndrcv.c Thu Aug 25 02:14:25 2016 -0700
+++ b/src/sndrcv.c Thu Sep 29 05:56:09 2016 -0700
@@ -38,19 +38,22 @@
# include <config.h>
#endif /* HAVE_CONFIG_H */
-#include <endian.h>
#include <stdlib.h>
#include <stdio.h>
-#include <strings.h>
#include <pthread.h>
#include <errno.h>
#include <netinet/in.h>
#include <sys/mman.h>
-#include <endian.h>
-#include <string.h>
#include <assert.h>
-#if defined(__x86_64)
-# include <immintrin.h>
+#if (defined(__SVR4) && defined(__sun))
+# include <unistd.h>
+# include <stropts.h>
+#else
+# include <endian.h>
+# include <string.h>
+# if defined(__x86_64)
+# include <immintrin.h>
+# endif
#endif
#include "sif.h"
#include "encoding.h"
@@ -87,8 +90,8 @@
* WA for "SW might overtake SQS", bug #3595, comment 53, solution #2:
* Check SQS in SQ-mode before posting WQE
* if SQ-mode, ring SQ-doorbell
- else post WQE to collect buffer
- */
+ * else post WQE to collect buffer
+ */
while (wr) {
const int nmbr_wrs_to_bulk_process = 32;
@@ -195,6 +198,7 @@
sif_log(uc, SIF_INFO, "queue full - rq %d entries %d len %d",
rq->index, rq->entries, rq_len);
atomic_dec_return(&rq->sw->length);
+ errno = ENOMEM;
ret = -ENOMEM;
goto err_post_recv;
}
@@ -202,6 +206,7 @@
sif_log(uc, SIF_INFO, "too many sges - rq %d sges configured %d, sges in wr %d",
rq->index, rq->sge_entries, wr->num_sge);
atomic_dec_return(&rq->sw->length);
+ errno = EINVAL;
ret = -EINVAL;
goto err_post_recv;
}
@@ -218,7 +223,7 @@
sge[i].length = wr->sg_list[i].length;
sge[i].lkey = lkey;
sif_log(uc, SIF_RCV,
- "sg_adr 0x%lx sg_len %d lkey %d",
+ "sg_adr 0x%"PRIx64" sg_len %d lkey %d",
wr->sg_list[i].addr, wr->sg_list[i].length, lkey);
}
@@ -241,7 +246,8 @@
wmb();
sif_log(uc, SIF_RCV, "Exit: success");
err_post_recv:
- *bad_wr = wr;
+ if (bad_wr)
+ *bad_wr = wr;
pthread_spin_unlock(&rq->lock);
#if SIF_UVERBS_ABI_VERSION > SIF_UVERBS_VERSION(3,3)
@@ -368,6 +374,7 @@
if (wr->num_sge > sq->sge_entries) {
sif_log(uc, SIF_SND, "attempt to post wr with %d/%d sg entries",
wr->num_sge, sq->sge_entries);
+ errno = EINVAL;
return -EINVAL;
}
@@ -379,6 +386,7 @@
sif_log(uc, SIF_INFO,
"queue full - head %d tail %d entries %d",
head, sq_seq, sq->entries);
+ errno = ENOMEM;
ret = -ENOMEM;
goto fail;
}
@@ -386,7 +394,7 @@
sqe = get_sq_entry(sq, sq_seq);
/* Efficienty set all the fields in the second u64 word in the WR to zero */
- ((u64 *)&wqe)[1] = 0;
+ memset(&wqe, 0, sizeof(wqe));
wqe.wr.sq_seq = sq_seq;
wqe.wr.collect_length = 0;
@@ -396,13 +404,13 @@
/* wqe.wr.port ignored from userspace, not QP0/1 */
wqe.wr.tsu_sl = qp->tsl;
wqe.wr.eps_tag = qp->eps_tag;
- wqe.wr.fence = (wr->send_flags & IBV_SEND_FENCE); /* RC only */
+ wqe.wr.fence = !!(wr->send_flags & IBV_SEND_FENCE); /* RC only */
wqe.wr.completion = sq->complete_all || wr->send_flags & IBV_SEND_SIGNALED;
wqe.wr.local_qp = qp->qp_idx;
wqe.wr.imm = wr->imm_data;
/* TBD: only set if wr opcode allows it */
- wqe.wr.se = (wr->send_flags & IBV_SEND_SOLICITED);
+ wqe.wr.se = !!(wr->send_flags & IBV_SEND_SOLICITED);
ret = prep_remote_addr(qp, wr, &wqe);
if (ret)
@@ -461,7 +469,7 @@
wh->sq_seq = sq_seq;
wh->used = true;
- sif_log(uc, SIF_SND, "wr_id %lx at tail 0x%x sq_seq_num %d%s",
+ sif_log(uc, SIF_SND, "wr_id %"PRIx64" at tail 0x%x sq_seq_num %d%s",
wr->wr_id, sq_seq & sq->mask, wqe.wr.sq_seq, (wqe.wr.completion ? " [req.compl]" : ""));
/* We can safely checksum any "hole" due to end misalignment + byte swap
@@ -693,7 +701,7 @@
la->addr = wr->sg_list[0].addr + mr_uv2dma(uc,lkey);
la->lkey = lkey;
la->length = wr->sg_list[0].length;
- sif_log(uc, SIF_SND, "single sge dma addr 0x%Lx, message len %d, key %d, uv2dma %Lx",
+ sif_log(uc, SIF_SND, "single sge dma addr 0x%"SIFPRIx64", message len %d, key %d, uv2dma %"SIFPRIx64"",
la->addr, la->length, lkey, mr_uv2dma(uc,lkey));
} else if (wr->num_sge > 1) {
struct sif_sq *sq = &qp->sq;
@@ -712,11 +720,11 @@
sge[i].length = wr->sg_list[i].length;
la->length += sge[i].length;
sif_log(uc, SIF_SND,
- "sg_list[%d]: sge entry: dma addr 0x%Lx, len = %d, lkey %d",
+ "sg_list[%d]: sge entry: dma addr 0x%"SIFPRIx64", len = %d, lkey %d",
i, sge[i].base_addr, sge[i].length, sge[i].lkey);
}
sif_log(uc, SIF_SND,
- "ready with sgl_start %p, sg list addr 0x%Lx, "
+ "ready with sgl_start %p, sg list addr 0x%"SIFPRIx64", "
"message len %d, lkey %d, sge %p",
sgl_start, la->addr, la->length, la->lkey, sge);
@@ -760,12 +768,14 @@
case IBV_QPT_UC:
case IBV_QPT_RC:
break;
+#if !(defined(__SVR4) && defined(__sun))
case IBV_QPT_XRC_SEND:
#if HAVE_DECL_IBV_QPT_XRC
case IBV_QPT_XRC:
#endif
wqe->wr.xrc_hdr.xrqd_id = wr->qp_type.xrc.remote_srqn;
break;
+#endif
default:
sif_log(uc, SIF_INFO,
"unhandled transport type %d", qp->verbs_qp.qp.qp_type);
@@ -885,12 +895,13 @@
sif_log(uc, SIF_INFO,
"Inline space exhausted: available %d, copied %d, len %d",
space, copy, len);
+ errno = ENOMEM;
return -ENOMEM;
}
- sif_log(uc, SIF_SND, "Copy sg len %d from user addr 0x%Lx to %p",
+ sif_log(uc, SIF_SND, "Copy sg len %d from user addr 0x%"SIFPRIx64" to %p",
len, addr, &dbuf[copy]);
- memcpy(&dbuf[copy], (void *)addr, len);
+ memcpy(&dbuf[copy], (void *)(intptr_t)addr, len);
copy += len;
}
diff -r d391efa34f11 src/sndrcv.h
--- a/src/sndrcv.h Thu Aug 25 02:14:25 2016 -0700
+++ b/src/sndrcv.h Thu Sep 29 05:56:09 2016 -0700
@@ -36,7 +36,9 @@
#ifndef _SIF_SNDRCV_H
#define _SIF_SNDRCV_H
+#if !(defined(__SVR4) && defined(__sun))
#include <kernel_types.h>
+#endif
/* ib verbs callbacks */
int sif_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr);
diff -r d391efa34f11 src/uverbs.c
--- a/src/uverbs.c Thu Aug 25 02:14:25 2016 -0700
+++ b/src/uverbs.c Thu Sep 29 05:56:09 2016 -0700
@@ -50,8 +50,11 @@
#include "encoding.h"
#include "sif_user.h"
#include "sif_abi.h"
+#if !(defined(__SVR4) && defined(__sun))
#include "kernel_types.h"
+#endif
#include "psif_hw_setget.h"
+#include <unistd.h>
#include "uverbs.h"
unsigned roundup_pwr2(unsigned x)
@@ -75,6 +78,14 @@
}
}
+#if defined(__SVR4) && defined(__sun)
+uint64_t page_align(uint64_t addr)
+{
+ int mask = getpagesize() - 1;
+
+ return (addr + mask) & ~mask;
+}
+#endif
void* find_map_desc(struct sif_context *sctx, enum desc_type type, __u32 index,
struct sif_qp *qp)
@@ -82,9 +93,13 @@
int i;
void *base = NULL;
#ifdef HAVE_VERBS_REGISTER_DRIVER
+#if defined(__SVR4) && defined(__sun)
+ int fd = sctx->verbs_ctx.context.mmap_fd;
+#else
int fd = sctx->verbs_ctx.context.cmd_fd;
+#endif
#else
- int fd = sctx->ibv_ctx.cmd_fd;
+ int fd = sctx->verbs_ctx.context.cmd_fd;
#endif
struct sif_desc_cache* dc = &sctx->dc[type];
uint32_t blk_idx = index / sctx->dc[type].ba.entry_per_block;
@@ -94,14 +109,22 @@
if (dc->d[i].cmd == SIF_MAP_NONE) {
enum sif_mmap_cmd cmd = dt_to_mmap(type);
/* The requested block is not mapped, try to map it: */
+#if defined(__SVR4) && defined(__sun)
+ off64_t offset = mmap_set_cmd(cmd, blk_idx);
+#else
off_t offset = mmap_set_cmd(cmd, blk_idx);
+#endif
int prot = PROT_READ;
- if (i != SIFD_SQ_HW)
+ if (type != SIFD_SQ_HW)
prot |= PROT_WRITE;
+#if defined(__SVR4) && defined(__sun)
+ base = mmap64(NULL, dc->ba.block_sz, prot, MAP_SHARED, fd, offset);
+#else
base = mmap(NULL, dc->ba.block_sz, prot, MAP_SHARED, fd, offset);
+#endif
if (base == MAP_FAILED) {
perror("find_map_desc");
- fprintf(stderr, "type %d block sz %d offset 0x%lx\n", type, dc->ba.block_sz, offset);
+ fprintf(stderr, "type %d block sz %d offset 0x%"PRIx64"\n", type, dc->ba.block_sz, offset);
base = NULL;
break;
}
@@ -213,10 +236,15 @@
int map_rq(struct sif_rq *rq, struct sif_context *sctx)
{
+
#ifdef HAVE_VERBS_REGISTER_DRIVER
+#if defined(__SVR4) && defined(__sun)
+ int fd = sctx->verbs_ctx.context.mmap_fd;
+#else
int fd = sctx->verbs_ctx.context.cmd_fd;
+#endif
#else
- int fd = sctx->ibv_ctx.cmd_fd;
+ int fd = sctx->verbs_ctx.context.cmd_fd;
#endif
rq->m.sz = rq->extent * rq->entries;
@@ -234,8 +262,14 @@
if (rq->m.sz) {
/* Map the receive queue for this QP */
+#if defined(__SVR4) && defined(__sun)
+ rq->m.sz = page_align(rq->m.sz);
+ off64_t offset = mmap_set_cmd(SIF_MAP_RQ, rq->index);
+ rq->m.base = mmap64(NULL, rq->m.sz, PROT_WRITE|PROT_READ, MAP_SHARED, fd, offset);
+#else
off_t offset = mmap_set_cmd(SIF_MAP_RQ, rq->index);
rq->m.base = mmap(NULL, rq->m.sz, PROT_WRITE|PROT_READ, MAP_SHARED, fd, offset);
+#endif
if (rq->m.base == MAP_FAILED)
return errno;
} else {
@@ -252,7 +286,11 @@
struct ibv_context *ctx = qp->verbs_qp.qp.context;
struct sif_device *dev = to_sdev(ctx->device);
struct sif_context *sctx = to_sctx(ctx);
+#if defined(__SVR4) && defined(__sun)
+ int fd = ctx->mmap_fd;
+#else
int fd = ctx->cmd_fd;
+#endif
int ret;
qp->sq.m.sz = qp->sq.extent * qp->sq.entries;
@@ -279,8 +317,14 @@
goto sq_done;
if (qp->sq.m.sz) {
/* Map the send queue for this QP */
+#if defined(__SVR4) && defined(__sun)
+ off64_t offset = mmap_set_cmd(SIF_MAP_SQ, qp->qp_idx);
+ qp->sq.m.sz = page_align(qp->sq.m.sz);
+ qp->sq.m.base = mmap64(NULL, qp->sq.m.sz, PROT_WRITE|PROT_READ, MAP_SHARED, fd, offset);
+#else
off_t offset = mmap_set_cmd(SIF_MAP_SQ, qp->qp_idx);
qp->sq.m.base = mmap(NULL, qp->sq.m.sz, PROT_WRITE|PROT_READ, MAP_SHARED, fd, offset);
+#endif
if (qp->sq.m.base == MAP_FAILED)
return errno;
} else
@@ -324,14 +368,17 @@
struct ibv_context *ctx = cq->ibv_cq.context;
struct sif_device *dev = to_sdev(ctx->device);
struct sif_context *sctx = to_sctx(ctx);
+#if defined(__SVR4) && defined(__sun)
+ int fd = ctx->mmap_fd;
+ int gross_cqe = cq->entries;
+#else
int fd = ctx->cmd_fd;
int gross_cqe = roundup_pwr2(cq->ibv_cq.cqe);
/* Adjust available cqes on rev 1-3 - Bug #3657 */
if (PSIF_REVISION(dev) <= 3)
gross_cqe = roundup_pwr2(cq->ibv_cq.cqe + 1);
-
- cq->entries = cq->ibv_cq.cqe;
+#endif
cq->mask = gross_cqe - 1;
cq->extent = sizeof(struct psif_cq_entry);
cq->m.sz = gross_cqe * cq->extent;
@@ -347,8 +394,14 @@
return errno;
if (cq->m.sz) {
+#if defined(__SVR4) && defined(__sun)
+ cq->m.sz = page_align(cq->m.sz);
+ off64_t offset = mmap_set_cmd(SIF_MAP_CQ, cq->index);
+ cq->m.base = mmap64(NULL, cq->m.sz, PROT_READ, MAP_SHARED, fd, offset);
+#else
off_t offset = mmap_set_cmd(SIF_MAP_CQ, cq->index);
cq->m.base = mmap(NULL, cq->m.sz, PROT_READ, MAP_SHARED, fd, offset);
+#endif
if (cq->m.base == MAP_FAILED)
return errno;
} else
@@ -385,10 +438,15 @@
{
struct ibv_context *ctx = pd->ibv_pd.context;
struct sif_device *dev = to_sdev(ctx->device);
+#if defined(__SVR4) && defined(__sun)
+ int fd = ctx->mmap_fd;
+ off64_t offset;
+#else
int fd = ctx->cmd_fd;
off_t offset;
+#endif
struct sif_map *cb = &pd->cb;
- size_t sz = 0x1000;
+ size_t sz = getpagesize();
if (dev->abi_version < abi_ver(3,0)) {
fprintf(stderr, "driver v.%d.%d does not support direct user level access\n",
@@ -397,7 +455,11 @@
}
offset = mmap_set_cmd(SIF_MAP_CB, cb->index);
+#if defined(__SVR4) && defined(__sun)
+ cb->base = mmap64(NULL, sz, PROT_WRITE, MAP_SHARED, fd, offset);
+#else
cb->base = mmap(NULL, sz, PROT_WRITE, MAP_SHARED, fd, offset);
+#endif
if (cb->base == MAP_FAILED)
return errno;
diff -r d391efa34f11 src/verbs.c
--- a/src/verbs.c Thu Aug 25 02:14:25 2016 -0700
+++ b/src/verbs.c Thu Sep 29 05:56:09 2016 -0700
@@ -91,7 +91,7 @@
struct sif_pd *pd;
struct ibv_alloc_pd cmd;
struct sif_alloc_pd_resp resp;
- int ret;
+ int ret = 0;
pd = malloc(sizeof *pd);
if (!pd)
@@ -103,6 +103,11 @@
goto create_failed;
pd->cb.index = resp.e.cb_idx;
+
+#if defined(__SVR4) && defined(__sun)
+ if (resp.e.reserved == 1)
+ uc->default_flags |= SQ_mode;
+#endif
pthread_spin_init(&pd->lock, PTHREAD_PROCESS_PRIVATE);
if (user_mode_verbs(context)) {
@@ -180,7 +185,7 @@
if (user_mode_verbs(ibpd->context)) {
ret = sif_unmap_cb(pd);
if (ret) {
- sif_log(uc, SIF_INFO, "failed to unmap cb %d (base %p sz %ld - errno %d",
+ sif_log(uc, SIF_INFO, "failed to unmap cb %d (base %p sz %"PRId64" - errno %d",
pd->cb.index, pd->cb.base, pd->cb.sz, errno);
return ret;
}
@@ -217,6 +222,7 @@
return NULL;
memset(cq, 0, sizeof(*cq));
+ memset(&cmd, 0, sizeof (cmd));
cmd.e.flags = to_sctx(context)->default_flags;
/* support for proxy cqs */
@@ -232,6 +238,9 @@
goto create_failed;
cq->index = resp.e.cq_idx;
+#if defined(__SVR4) && defined(__sun)
+ cq->entries = resp.e.cq_entries;
+#endif
pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE);
if (user_mode_verbs(context)) {
@@ -293,6 +302,11 @@
if (ret)
return ret;
}
+ if (user_mode_verbs(ibcq->context)) {
+ ret = sif_unmap_cq(cq);
+ if (ret)
+ return ret;
+ }
free(cq);
return 0;
@@ -527,9 +541,14 @@
enum sif_flush_type qtype)
{
struct sif_modify_qp_cmd mcmd;
+ int mask = 0;
memset(&mcmd, 0, sizeof(mcmd));
mcmd.e.flush = qtype;
- return ibv_cmd_modify_qp(ibqp, attr, 0, &mcmd.ibv_cmd, sizeof(mcmd));
+#if defined(__SVR4) && defined(__sun)
+ mask = IBV_QP_STATE;
+ attr->qp_state = IBV_QPS_ERR;
+#endif
+ return ibv_cmd_modify_qp(ibqp, attr, mask, &mcmd.ibv_cmd, sizeof(mcmd));
}
#endif
@@ -573,6 +592,31 @@
qp->remote_lid = attr->ah_attr.dlid;
if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_RTR) && qp->sq.sw)
qp->tsl = qp->sq.sw->tsl;
+ if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_RESET) &&
+ user_mode_verbs(ibqp->context)) {
+ struct sif_context *uc = to_sctx(ibqp->context);
+ int nfixup;
+
+ if (has_sq(qp) && qp->sq.cq) {
+ nfixup = sif_fixup_cqes(qp->sq.cq, &qp->sq, qp);
+ if (nfixup < 0) {
+ sif_log(uc, SIF_INFO, "fixup cqes on sq %d failed with errno %d",
+ qp->qp_idx, errno);
+ return ret;
+ }
+ sif_log(uc, SIF_QP, "fixup cqes fixed %d CQEs in sq.cq", nfixup);
+
+ if (qp->rq && qp->rq->cq && (qp->rq->cq != qp->sq.cq)) {
+ nfixup = sif_fixup_cqes(qp->rq->cq, &qp->sq, qp);
+ if (nfixup < 0) {
+ sif_log(uc, SIF_INFO, "fixup cqes on sq %d failed with errno %d",
+ qp->qp_idx, errno);
+ return ret;
+ }
+ sif_log(uc, SIF_QP, "fixup cqes fixed %d CQEs in rq.cq", nfixup);
+ }
+ }
+ }
return 0;
}
@@ -650,8 +694,12 @@
struct ibv_srq_init_attr_ex *attr)
{
struct sif_rq *rq;
+#if HAVE_IBV_XRC_OPS
+ struct sif_create_xsrq_cmd cmd;
+#else
+ struct sif_create_srq_cmd cmd;
+#endif
struct sif_cq *cq = to_scq(attr->cq);
- struct sif_create_xsrq_cmd cmd;
struct sif_create_srq_resp resp;
struct sif_context *sctx = to_sctx(context);
int ret;
@@ -668,13 +716,24 @@
if (!rq)
return NULL;
memset(rq, 0, sizeof(*rq));
+ memset(&cmd, 0, sizeof(cmd));
+ memset(&resp, 0, sizeof(resp));
+
cmd.e.flags = sif_get_default_flags(context);
cmd.e.res1 = 0;
+#if HAVE_IBV_XRC_OPS
ret = ibv_cmd_create_srq_ex(context, &rq->verbs_srq,
sizeof(rq->verbs_srq),
attr, &cmd.ibv_cmd, sizeof(cmd),
&resp.ibv_resp, sizeof(resp));
+#else
+ ret = ibv_cmd_create_srq(attr->pd, &rq->verbs_srq.srq,
+ (struct ibv_srq_init_attr *)attr,
+ &cmd.ibv_cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp));
+#endif
+
if (ret) {
sif_log(sctx, SIF_INFO, "%s: failed to create srq, ret %d\n", __func__, ret);
goto create_failed;
@@ -715,7 +774,7 @@
struct ibv_srq *sif_create_srq(struct ibv_pd *ibpd, struct ibv_srq_init_attr *attr)
{
- struct ibv_srq_init_attr_ex attr_ex;
+ struct ibv_srq_init_attr_ex attr_ex = {0};
struct ibv_srq *srq;
attr_ex.srq_context = attr->srq_context;
attr_ex.attr = attr->attr;
@@ -789,6 +848,11 @@
cq->xsrq = NULL;
}
+ if (rq->m.sz) {
+ ret = munmap(rq->m.base, rq->m.sz);
+ rq->m.sz = 0;
+ }
+
free(rq);
return 0;
}
diff -r d391efa34f11 src/xrc.h
--- a/src/xrc.h Thu Aug 25 02:14:25 2016 -0700
+++ b/src/xrc.h Thu Sep 29 05:56:09 2016 -0700
@@ -34,6 +34,9 @@
* xrc.h: XRC support device specific calls.
*/
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif /* HAVE_CONFIG_H */
#ifndef _SIF_XRC_H
#define _SIF_XRC_H
#include <infiniband/driver.h>