23188922 libsif to support TSL policy
23743649 Userland changes for sqs-flush errors
--- a/components/open-fabrics/libsif/include/sif/sif_int_user.h Mon Jul 25 14:27:35 2016 -0700
+++ b/components/open-fabrics/libsif/include/sif/sif_int_user.h Tue Jul 26 11:56:10 2016 -0700
@@ -55,6 +55,12 @@
* We use the extension here to communicate with the driver
* (for correct debugfs reporting)
*/
+
+enum sq_sw_state {
+ FLUSH_SQ_IN_PROGRESS = 0,
+ FLUSH_SQ_IN_FLIGHT = 1,
+};
+
typedef struct sif_sq_sw {
struct psif_sq_sw d; /* Hardware visible descriptor */
/* separate the cache lines */
@@ -62,8 +68,18 @@
__u16 last_seq; /* Last used sq seq.num (req. sq->lock) */
/* Last sq seq.number seen in a compl (req. cq->lock) */
volatile __u16 head_seq;
+ __u16 trusted_seq;
+ __u8 tsl;
+ volatile __u64 flags;
}sif_sq_sw_t;
+enum rq_sw_state {
+ FLUSH_RQ_IN_PROGRESS = 0,
+ FLUSH_RQ_IN_FLIGHT = 1,
+ FLUSH_RQ_FIRST_TIME = 2,
+ RQ_IS_INVALIDATED = 3,
+};
+
typedef struct sif_rq_sw {
struct psif_rq_sw d; /* Hardware visible descriptor */
/* separate the cache lines */
@@ -71,8 +87,15 @@
/* current length of queue as #posted - #completed */
volatile uint32_t length;
__u32 next_seq; /* First unused sequence number */
+ volatile __u64 flags;
}sif_rq_sw_t;
+enum cq_sw_state {
+ CQ_POLLING_NOT_ALLOWED = 0,
+ CQ_POLLING_IGNORED_SEQ = 1,
+ FLUSH_SQ_FIRST_TIME = 2,
+};
+
typedef struct sif_cq_sw {
struct psif_cq_sw d; /* Hardware visible descriptor */
/* separate the cache lines */
@@ -88,6 +111,7 @@
__u32 miss_cnt;
/* Number of times 1 or more in-flight completions was seen */
__u32 miss_occ;
+ volatile __u64 flags;
}sif_cq_sw_t;
#ifdef __cplusplus--- a/components/open-fabrics/libsif/include/sif/sif_user.h Mon Jul 25 14:27:35 2016 -0700
+++ b/components/open-fabrics/libsif/include/sif/sif_user.h Tue Jul 26 11:56:10 2016 -0700
@@ -53,8 +53,8 @@
* features are available.
*
*/
-#define SIF_UVERBS_ABI_MAJOR_VERSION 3
-#define SIF_UVERBS_ABI_MINOR_VERSION 4
+#define SIF_UVERBS_ABI_MAJOR_VERSION 4
+#define SIF_UVERBS_ABI_MINOR_VERSION 0
/* Bw comp */
#define SIF_UVERBS_VERSION(x, y) ((x) << 8 | (y))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/open-fabrics/libsif/patches/003-Bug23188922.patch Tue Jul 26 11:56:10 2016 -0700
@@ -0,0 +1,15 @@
+# This patch is fix for the Bug-23188922
+# It is developed By solaris PSIF team. We plan to have a common upstream repo
+# and submit these changes to it, but do not yet have a target date of doing it
+diff -r 6299a4e4e707 src/verbs.c
+--- a/src/verbs.c Wed Jul 20 03:05:26 2016 -0700
++++ b/src/verbs.c Wed Jul 20 04:48:17 2016 -0700
[email protected]@ -665,6 +665,8 @@
+ qp->remote_qp = attr->dest_qp_num;
+ if (attr_mask & IBV_QP_AV)
+ qp->remote_lid = attr->ah_attr.dlid;
++ if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_RTR) && qp->sq.sw)
++ qp->tsl = qp->sq.sw->tsl;
+ return 0;
+ }
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/open-fabrics/libsif/patches/004-Bug23743649.patch Tue Jul 26 11:56:10 2016 -0700
@@ -0,0 +1,100 @@
+# This patch is fix for the Bug-22995977
+# It is developed By solaris PSIF team. We plan to have a common upstream repo
+# and submit these changes to it, but do not yet have a target date of doing it
+diff -r 7ecef74fde26 src/sndrcv.c
+--- a/src/sndrcv.c Wed Jul 20 03:09:52 2016 -0700
++++ b/src/sndrcv.c Wed Jul 20 04:48:09 2016 -0700
[email protected]@ -68,6 +68,9 @@
+ #define CB_KICK_MASK (CB_KICK_ALIGN - 1)
+
+ #define SQS_ACTIVE (get_psif_sq_hw__sq_next(qp->sq.hw) != 0xFFFFFFFF)
++
++bool reliable_qp(enum ibv_qp_type type);
++
+ #ifdef _ILP32
+ int sif_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr)
+ {
[email protected]@ -96,6 +99,13 @@
+ struct sif_qp *qp = to_sqp(ibqp);
+ struct sif_sq *sq = &qp->sq;
+
++#ifdef HAVE_VERBS_REGISTER_DRIVER
++ struct ibv_qp *ibv_qp = &qp->verbs_qp.qp;
++#else
++ struct ibv_qp *ibv_qp = &qp->ibv_qp;
++#endif
++
++
+ if (!qp->sq.m.sz) /* no user space mappings, go via kernel */
+ return ibv_cmd_post_send(ibqp, wr, bad_wr);
+
[email protected]@ -128,16 +138,43 @@
+ }
+ pthread_mutex_unlock(&sq->lock);
+ }
++ if (reliable_qp(ibv_qp->qp_type)
++ && ibv_qp->state == IBV_QPS_ERR) {
++ ret = 0;
++ goto flush_sq_wa4074;
++ }
+
+ return 0;
+
+ err_post_send:
+ pthread_mutex_unlock(&sq->lock);
+ *bad_wr = wr;
++
++flush_sq_wa4074:
++ /* WA #4074, issue SQ flush if QP in ERROR. */
++#if SIF_UVERBS_ABI_VERSION > SIF_UVERBS_VERSION(3,3)
++ if (reliable_qp(ibv_qp->qp_type)
++ && ibv_qp->state == IBV_QPS_ERR) {
++
++ struct sif_context *uc = to_sctx(ibqp->context);
++ struct ibv_qp_attr attr;
++ memset(&attr, 0, sizeof(attr));
++
++ if (flush_qp(ibv_qp, &attr, FLUSH_SQ))
++ sif_log(uc, SIF_INFO, "failed to flush SQ, QP %d",
++ ibv_qp->qp_num);
++ }
++#endif
++
+ return ret;
+ }
+ #undef SQS_ACTIVE
+
++bool reliable_qp(enum ibv_qp_type type)
++{
++ return (type == IBV_QPT_RC);
++}
++
+ /* Return bypass mode offset or 0 if invalid for post_sends (see below)
+ * (PSIF will take care of rejecting the post)
+ */
[email protected]@ -225,9 +262,12 @@
+ "next_seq %x length %d", rq->entries, rq->extent,
+ rq->index, rq->m.base, rq->sw->next_seq, rq->sw->length);
+ }
+- /* Notify hw of new entries */
++
++ /* Enforce ordering of new rq entries and tail */
+ wmb();
+ set_psif_rq_sw__tail_indx(&rq->sw->d, rq->sw->next_seq);
++ /* Enforce visibility of rq tail on hw */
++ wmb();
+
+ sif_log(uc, SIF_RCV, "Exit: success");
+ err_post_recv:
+diff -r 7ecef74fde26 src/uverbs.c
+--- a/src/uverbs.c Wed Jul 20 03:09:52 2016 -0700
++++ b/src/uverbs.c Wed Jul 20 04:48:09 2016 -0700
[email protected]@ -388,7 +388,7 @@
+ #if defined(__SVR4) && defined(__sun)
+ cq->m.sz = page_align(cq->m.sz);
+ off64_t offset = mmap_set_cmd(SIF_MAP_CQ, cq->index);
+- cq->m.base = mmap64(NULL, cq->m.sz, PROT_READ, MAP_SHARED, fd, offset);
++ cq->m.base = mmap64(NULL, cq->m.sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, offset);
+ #else
+ off_t offset = mmap_set_cmd(SIF_MAP_CQ, cq->index);
+ cq->m.base = mmap(NULL, cq->m.sz, PROT_READ, MAP_SHARED, fd, offset);