# This patch does the src changes specific to solaris.

# It is developed By solaris PSIF team. We plan to have a common upstream repo

# and submit these changes to it, but do not yet have a target date of doing it.

diff -r 1584d5cbb44d src/cq.c

--- a/src/cq.c	Wed Feb 10 12:40:22 2016 +0530

+++ b/src/cq.c	Wed Mar 09 22:17:00 2016 +0530

@@ -43,7 +43,9 @@

 #include <strings.h>

 #include <pthread.h>

 #include <errno.h>

+#if !(defined(__SVR4) && defined(__sun))

 #include <endian.h>

+#endif

 #include <string.h>

 #include "sif.h"

@@ -57,6 +59,13 @@

 #include "sif_int_user.h"

 #include "cq.h"

+#ifdef _ILP32

+int sif_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc)

+{

+	sif_log(to_sctx(ibcq->context), SIF_INFO, "poll_cq not supported for 32 bit application. Please use 64Bit App");

+	return ENOTSUP;

+}

+#else

 static inline int cq_length(struct sif_cq *cq, u32 head, u32 tail)

 {

     return (int)(tail + cq->entries - head) & cq->mask;

@@ -139,7 +148,7 @@

     qp->sq.sw->head_seq = sq_seq_num;

     sif_log(uc, SIF_CQ,

-            "wr_id 0x%lx on qp/sq %d sq_seq_num %d",

+            "wr_id 0x%"PRIx64" on qp/sq %d sq_seq_num %d",

             wc->wr_id, cqe->qp, sq_seq_num);

     if (uc->debug_mask & SIF_SQSOVRN)

@@ -154,9 +163,11 @@

  */

 struct sif_rq *find_rq(struct sif_qp *qp, struct sif_cq *cq)

 {

+#ifdef HAVE_VERBS_REGISTER_DRIVER

     if (qp->verbs_qp.qp.qp_type == IBV_QPT_XRC_RECV)

         return cq->xsrq;

     else

+#endif

         return qp->rq;

 }

@@ -198,18 +209,23 @@

 	struct ibv_query_qp cmd;

         int ret = 0;

+#ifdef HAVE_VERBS_REGISTER_DRIVER

+        struct ibv_qp *ibv_qp = &qp->verbs_qp.qp;

+#else

+        struct ibv_qp *ibv_qp = &qp->ibv_qp;

+#endif

 	memset(&attr, 0, sizeof(attr));

 	memset(&init_attr, 0, sizeof(init_attr));

-	ret = ibv_cmd_query_qp(&qp->verbs_qp.qp, &attr, IBV_QP_STATE, &init_attr,

+        ret = ibv_cmd_query_qp(ibv_qp, &attr, IBV_QP_STATE, &init_attr,

 	    &cmd, sizeof cmd);

 	if (!ret && attr.qp_state == IBV_QPS_ERR) {

-	    if (flush_qp(&qp->verbs_qp.qp, &attr, FLUSH_RQ))

+            if (flush_qp(ibv_qp, &attr, FLUSH_RQ))

 	    	sif_log(uc, SIF_INFO,"failed to flush RQ, QP %d", wc->qp_num);

     	}

     }

 #endif

     /* TBD: Handle sg list case */

-    sif_log(uc, SIF_CQ,"wr_id 0x%lx queue len %d", wc->wr_id, rq_len);

+    sif_log(uc, SIF_CQ,"wr_id 0x%"PRIx64" queue len %d", wc->wr_id, rq_len);

     return 0;

 }

@@ -253,7 +269,7 @@

          */

         sif_log(uc, log_level,

                 "error compl on cq %d, seq_num %d qpn 0x%x, "

-                "wc_id %Ld status %s(%d), vendor_err 0x%x",

+                "wc_id %"SIFPRIx64" status %s(%d), vendor_err 0x%x",

                 cq->index, lcqe.seq_num, qpn, lcqe.wc_id.rq_id,

                 string_enum_psif_wc_status(lcqe.status),

                 lcqe.status, lcqe.vendor_err);

@@ -282,7 +298,7 @@

 	default:

 		sif_log(uc, SIF_WCE,

 			"unknown opcode on cq %d, seq_num %d qpn 0x%x, "

-			"wc_id %Ld status %s(%d), vendor_err 0x%x",

+                        "wc_id %"SIFPRIx64" status %s(%d), vendor_err 0x%x",

 			cq->index, lcqe.seq_num, qpn, lcqe.wc_id.rq_id,

 			string_enum_psif_wc_status(lcqe.status),

 			lcqe.status, lcqe.vendor_err);

@@ -437,3 +453,4 @@

             npolled, polled_value);

     return !ret ? npolled : ret;

 }

+#endif

diff -r 1584d5cbb44d src/encoding.h

--- a/src/encoding.h	Wed Feb 10 12:40:22 2016 +0530

+++ b/src/encoding.h	Wed Mar 09 22:17:00 2016 +0530

@@ -37,16 +37,25 @@

 #define _SIF_ENCODING_H

 #include <sys/types.h>

+#if !(defined(__SVR4) && defined(__sun))

 #include "kernel_types.h"

+#endif

 #include "psif_hw_data.h"

 #define true 1

 #define false 0

 #define bool int

+#if defined(__SVR4) && defined(__sun)

+#include <sys/atomic.h>

+typedef volatile uint32_t atomic_t;

+#define atomic_inc_return(ptr)  atomic_inc_32_nv(ptr)

+#define atomic_dec_return(ptr)  atomic_dec_32_nv(ptr) 

+#else

 typedef volatile __u32 atomic_t;

 #define atomic_inc_return(ptr)  __sync_add_and_fetch(ptr, 1)

 #define atomic_dec_return(ptr)  __sync_sub_and_fetch(ptr, 1)

+#endif

 #if 1

 #define SIF_INFO	      0x1L

@@ -85,10 +94,17 @@

 #endif

 #define min(x,y) ((x) > (y) ? (y) : (x))

+#if defined(__SVR4) && defined(__sun)

+#define cpu_to_be32 htonl

+#define cpu_to_be64 htonll

+#define be32_to_cpu ntohl

+#define be64_to_cpu ntohll

+#else

 #define cpu_to_be32 htobe32

 #define cpu_to_be64 htobe64

 #define be32_to_cpu be32toh

 #define be64_to_cpu be64toh

+#endif

 #ifdef __x86_64

 #undef wmb

diff -r 1584d5cbb44d src/hwprint.c

--- a/src/hwprint.c	Wed Feb 10 12:40:22 2016 +0530

+++ b/src/hwprint.c	Wed Mar 09 22:17:00 2016 +0530

@@ -34,6 +34,8 @@

  * hwprint.c: User level wrapper for hardware (generated) struct print support

  */

 #include <sys/types.h>

+#if !(defined(__SVR4) && defined(__sun))

 #define OS_PRIx64 "lx"

+#endif

 #include "sif.h"

 #include "psif_hw_print.c"

diff -r 1584d5cbb44d src/mr.c

--- a/src/mr.c	Wed Feb 10 12:40:22 2016 +0530

+++ b/src/mr.c	Wed Mar 09 22:17:00 2016 +0530

@@ -45,7 +45,33 @@

 #include "sif_abi.h"

 #include "mr.h"

 #include <assert.h>

+#include <errno.h>

+#ifdef _ILP32

+struct ibv_mr *sif_reg_mr(struct ibv_pd *ibpd, void *addr,

+                          size_t length, int access)

+{

+	sif_log(to_sctx(ibpd->context), SIF_INFO, "reg_mr not supported for 32-bit. Please use 64-bit Application");

+	errno = ENOTSUP;

+	return NULL;

+}

+

+struct ibv_mr *sif_rereg_mr(struct ibv_mr *ibmr, int flags,

+                            struct ibv_pd *ibpd, void *addr, size_t length,

+                            int access)

+{

+	sif_log(to_sctx(ibpd->context), SIF_INFO, "rereg_mr not supported for 32-bit. Please use 64-bit Application");

+	errno = ENOTSUP;

+	return NULL;

+}

+

+int sif_dereg_mr(struct ibv_mr *ibmr)

+{

+	sif_log(to_sctx(ibmr->context), SIF_INFO,"dereg_mr not supported for 32-bit. Please use 64-bit Application");

+	return ENOTSUP;

+}

+

+#else

 struct ibv_mr *sif_reg_mr(struct ibv_pd *ibpd, void *addr,

                           size_t length, int access)

 {

@@ -103,7 +129,7 @@

             sctx->bypass_mr_ref = malloc(ref_sz);

             if (!sctx->bypass_mr_ref) {

                 sif_log(sctx, SIF_INFO,

-                        "Failed to allocate 0x%lx bytes of storage for bypass MR references\n",

+                        "Failed to allocate 0x%zx bytes of storage for bypass MR references\n",

                         ref_sz);

                 ret = ENOMEM;

                 goto bypass_ref_alloc_failed;

@@ -168,6 +194,38 @@

     free(mr);

     return 0;

 }

+#endif

+

+#if defined(__SVR4) && defined(__sun)

+/*

+ * Libibverbs will not validate the function ptr for non-NULL value for fmr

+ * callbacks.If undefined, application will core dump if it uses any of 

+ * these calls.

+ */

+struct ibv_mr *sif_reg_mr_relaxed(struct ibv_pd *ibpd, void *addr, size_t length, int access)

+{

+    struct sif_context *sctx = to_sctx(ibpd->context);

+

+    sif_log(sctx, SIF_INFO, "sif_reg_relaxed_mr Not Implemented");

+    errno = ENOTSUP;

+

+    return NULL;

+}

+

+int sif_dereg_mr_relaxed(struct ibv_mr *ibmr)

+{

+    struct sif_context *sctx = to_sctx(ibmr->context);

+    sif_log(sctx, SIF_INFO, "sif_dereg_relaxed_mr Not Implemented");

+    return ENOTSUP;

+}

+

+int sif_flush_relaxed_mr(struct ibv_pd *ibpd)

+{

+    struct sif_context *sctx = to_sctx(ibpd->context);

+    sif_log(sctx, SIF_INFO, "sif_flush_relaxed_mr Not Implemented");

+    return ENOTSUP;

+}

+#endif

 struct ibv_mw *sif_alloc_mw(struct ibv_pd *ibpd, enum ibv_mw_type type)

 {

diff -r 1584d5cbb44d src/sif.c

--- a/src/sif.c	Wed Feb 10 12:40:22 2016 +0530

+++ b/src/sif.c	Wed Mar 09 22:17:00 2016 +0530

@@ -44,6 +44,7 @@

 #include <sys/mman.h>

 #include <pthread.h>

 #include <string.h>

+#include <infiniband/verbs.h>

 #ifndef HAVE_IBV_READ_SYSFS_FILE

 #include <sys/types.h>

@@ -76,6 +77,8 @@

     HCA(SUN,  PSIF_VF),

     HCA(SUN,  SIBS_PF),

     HCA(SUN,  SIBS_VF),

+    HCA(SUN,  SIBS2_PF),

+    HCA(SUN,  SIBS2_VF),

 };

 static struct ibv_context_ops sif_ctx_ops = {

@@ -90,6 +93,12 @@

     .rereg_mr      = sif_rereg_mr,

     .dereg_mr      = sif_dereg_mr,

+#if defined(__SVR4) && defined(__sun)

+    .reg_mr_relaxed= sif_reg_mr_relaxed,

+    .dereg_mr_relaxed = sif_dereg_mr_relaxed,

+    .flush_relaxed_mr = sif_flush_relaxed_mr,

+#endif

+

     .alloc_mw      = sif_alloc_mw,

     .bind_mw       = sif_bind_mw,

     .dealloc_mw    = sif_dealloc_mw,

@@ -159,6 +168,16 @@

     struct sif_device *sdev = to_sdev(ibdev);

     char value[20];

     char param_path[IBV_SYSFS_PATH_MAX];

+#if defined(__SVR4) && defined(__sun)

+/*

+ * See CR 22214822,22245798:collect buffer mode disabled for solaris

+ * SQ gives better performance on SPARC mode for

+ * smaller message sizes.

+ */

+    const char *cb_mode;

+    int mode;

+    const char *user_cb_disable;

+#endif

     const char* dbm;

     const char *cfum;

     const char *odm;

@@ -170,6 +189,9 @@

 #ifdef HAVE_VERBS_REGISTER_DRIVER

     context = to_sctx(ibv_ctx);

+#if defined(__SVR4) && defined(__sun)

+    ibv_ctx->device = &v_device->device;

+#endif

 #else

     context = malloc(sizeof(*context));

     if (!context)

@@ -177,23 +199,20 @@

     /* The sif specific part needs to be zeroed out */

     memset(context, 0, sizeof(*context));

     ibv_ctx = &context->ibv_ctx;

+#if defined(__SVR4) && defined(__sun)

+    context->ibv_ctx.device = ibdev;

+#endif

 #endif

     /* Satisfy valgrind */

     memset(&cmd,0,sizeof(cmd));

-    /* Memory is allocated zeroed out by libsif */

-

-    if (!sdev->min_resp_ms) {

-        if (ibv_read_sysfs_file(sdev->verbs_dev.device.ibdev_path, "min_resp_ms",

-                                value, sizeof value) < 0)

-            sdev->min_resp_ms = 500;

-        else

-            sscanf(value, "%li", &sdev->min_resp_ms);

-    }

-

     /* Re-read the current kernel default flags */

+#ifdef HAVE_VERBS_REGISTER_DRIVER

     sprintf(param_path, "%s/device/driver/module/parameters", sdev->verbs_dev.device.ibdev_path);

+#else

+    sprintf(param_path, "%s/device/driver/module/parameters", sdev->ibv_dev.ibdev_path);

+#endif

     if (ibv_read_sysfs_file(param_path, "vendor_flags",

                             value, sizeof value) < 0)

         context->kernel_flags = 0;

@@ -207,8 +226,13 @@

         (SIF_UVERBS_ABI_MINOR_VERSION & 0xff);

     ibv_ctx->cmd_fd = cmd_fd;

+#if defined(__SVR4) && defined(__sun)

+    ret = ibv_cmd_get_context(ibv_ctx, &cmd.ibv_cmd,

+                              sizeof cmd.ibv_cmd, &resp.ibv_resp, sizeof resp);

+#else

     ret = ibv_cmd_get_context(ibv_ctx, &cmd.ibv_cmd,

                               sizeof cmd, &resp.ibv_resp, sizeof resp);

+#endif

     if (ret)

         goto err_free;

@@ -230,6 +254,18 @@

     context->opt_disable_mask = odm ? strtoll(odm, NULL, 0) : 0;

     disa_x_wc = context->opt_disable_mask & SOD_EXPLICIT_WC;

+#if defined(__SVR4) && defined(__sun)

+    user_cb_disable = getenv("SIF_DISABLE_USER_CB");

+    context->user_cb_disable = user_cb_disable ? strtoll(user_cb_disable, NULL, 0) : 0;

+    /*

+     * SQ mode is the default mode on solaris

+     * set env variable SIF_CB_MODE = 1 to change to cb_mode

+     */

+    cb_mode = getenv("SIF_CB_MODE");

+    mode = cb_mode ? strtol(cb_mode, NULL, 0) : 0;

+    context->default_flags |= mode? 0: SQ_mode;

+    sif_log(context, SIF_INFO, "Mode: %s", mode? "CB/SQ": "SQ");

+#endif

     probe_cpu_features();

     sif_log(context, SIF_VERBS, "CPU features detected: %s",

 	    cpu_features_to_str(feature_mnemonics_size, feature_mnemonics));

@@ -369,6 +405,12 @@

     dev->abi_version = abi_version;

     dev->min_resp_ms = 0; /* Will be set by the first allocated context */

+#if defined(__SVR4) && defined(__sun)

+    /*

+     * SIFMT_BYPASS mode is not supported for userspace on Solaris, mr_size is

+     * not required */

+    dev->mr_size = 4096;

+#else

     if (ibv_read_sysfs_file("/sys/module/sif", "parameters/mr_size",

                             value, sizeof value) < 0) {

         fprintf(stderr, "Failed to read mr_size\n");

@@ -376,6 +418,7 @@

         return NULL;

     }

     sscanf(value, "%i", &dev->mr_size);

+#endif

 #ifndef HAVE_VERBS_REGISTER_DRIVER

     dev->ibv_dev.ops = sif_dev_ops;

diff -r 1584d5cbb44d src/sif.h.in

--- a/src/sif.h.in	Wed Feb 10 12:40:22 2016 +0530

+++ b/src/sif.h.in	Wed Mar 09 22:17:00 2016 +0530

@@ -34,6 +34,9 @@

  * sif.h.in: TBD: **insert desc here**

  */

+#if HAVE_CONFIG_H

+#include <config.h>

+#endif /* HAVE_CONFIG_H */

 #ifndef _SIF_H

 #define _SIF_H

 #if defined(__cplusplus)

@@ -54,6 +57,8 @@

 #define PCI_DEVICE_ID_PSIF_VF       0x2089

 #define PCI_DEVICE_ID_SIBS_PF       0x2188

 #define PCI_DEVICE_ID_SIBS_VF       0x2189

+#define PCI_DEVICE_ID_SIBS2_PF      0x2198

+#define PCI_DEVICE_ID_SIBS2_VF      0x2199

 #endif

 #define HIDDEN      __attribute__((visibility ("hidden")))

@@ -64,6 +69,19 @@

 #define SOD_EXPLICIT_WC 0x1

 #define SOD_WC_TO_SQ 0x2

+#if defined(__SVR4) && defined(__sun)

+#define SIF_PAGE_SHIFT	12

+#define SIF_PAGE_SIZE	(1 << SIF_PAGE_SHIFT)

+

+#define        SIF_DOORBELL_IOCTL      30464

+#define container_of(ptr, type, member) ({                      \

+        const typeof( ((type *)0)->member ) *__mptr = (ptr);    \

+        (type *)( (char *)__mptr - offsetof(type,member) );})

+#define SIFPRIx64 PRIx64

+#include "sif_user.h"

+#else

+#define SIFPRIx64 "llx"

+

 enum sif_vendor_flags

 {

     MMU_special   =  0x1,  /* Use special mmu setup in associated mappings

@@ -102,9 +120,10 @@

     FLUSH_SQ,

     FLUSH_RQ

 };

+#endif

 struct sif_device {

-    struct verbs_device   verbs_dev;

+    struct @SIF_GET_STRUCT@_device   @SIF_GET_STRUCT@_dev;

     uint16_t            abi_version;

     uint32_t 		mr_size;  /* Maximal number of MRs in the system */

     uint64_t		min_resp_ms; /* Universal timeout scaling for sif */

@@ -160,7 +179,7 @@

 struct sif_rq {

     struct sif_map m;

-    struct verbs_srq verbs_srq; /* Only used if this RQ is used a SRQ */

+    struct @SIF_GET_STRUCT@_srq @SIF_GET_STRUCT@_srq; /* Only used if this RQ is used a SRQ */

     uint32_t index;

     uint32_t entries;

     uint32_t mask;     /* entries - 1 for modulo using & */

@@ -185,7 +204,7 @@

 };

 struct sif_context {

-    struct verbs_context  verbs_ctx;

+    struct @SIF_GET_STRUCT@_context @SIF_GET_STRUCT@_ctx;

     enum sif_vendor_flags default_flags;

     enum sif_vendor_flags kernel_flags; /* Vendor flags set by the kernel */

     enum sif_mem_type mem_type;

@@ -198,6 +217,9 @@

     struct sif_mr **bypass_mr_ref;

     int bypass_cnt; /* Number of bypass MRs initiated - to avoid slow lookup unless bypass */

     uint64_t debug_mask;

+#if defined(__SVR4) && defined(__sun)

+    uint64_t user_cb_disable;

+#endif

     uint64_t opt_disable_mask;

     void (* fast_copy)(void *dst, const void *src, int len);

 };

@@ -222,7 +244,7 @@

 };

 struct sif_qp {

-    struct verbs_qp   verbs_qp;

+    struct @SIF_GET_STRUCT@_qp @SIF_GET_STRUCT@_qp;

     uint32_t qp_idx;

     uint32_t magic;

     uint8_t  qosl;

@@ -245,7 +267,11 @@

 struct sif_mr {

     struct ibv_mr   ibv_mr;

+#if defined(__SVR4) && defined(__sun)

+    off64_t uv2dma;

+#else

     off_t uv2dma;

+#endif

     enum sif_mem_type mem_type;

 };

@@ -298,12 +324,12 @@

 static inline struct sif_qp *to_sqp(struct ibv_qp *ibqp)

 {

-    return container_of(ibqp, struct sif_qp, verbs_qp.qp);

+    @SIF_GET_QP_CONTAINER@

 }

 static inline struct sif_rq *to_ssrq(struct ibv_srq *ibsrq)

 {

-    return container_of(ibsrq, struct sif_rq, verbs_srq.srq);

+    @SIF_GET_SRQ_CONTAINER@

 }

 static inline struct sif_ah *to_sah(struct ibv_ah *ibah)

@@ -353,6 +379,10 @@

                         struct ibv_pd *pd, void *addr, size_t length,

                         int access);

 int sif_dereg_mr(struct ibv_mr *mr);

+struct ibv_mr *sif_reg_mr_relaxed(struct ibv_pd *pd, void *addr,

+                        size_t length, int access);

+int sif_dereg_mr_relaxed(struct ibv_mr *mr);

+int sif_flush_relaxed_mr(struct ibv_pd *pd);

 struct ibv_mw *sif_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type);

 int sif_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, struct ibv_mw_bind *mw_bind);

 int sif_dealloc_mw(struct ibv_mw *mw);

@@ -366,9 +396,13 @@

 struct ibv_qp *sif_create_qp(struct ibv_pd *pd,

                    struct ibv_qp_init_attr *attr);

+#ifdef HAVE_VERBS_REGISTER_DRIVER

 struct ibv_qp *sif_create_qp_ex(struct ibv_context *context,

                    struct ibv_qp_init_attr_ex *qp_init_attr_ex);

+struct ibv_srq *sif_create_srq_ex(struct ibv_context *context,