Delphix vmxnet3s fixes default tip
authorSebastien Roy <seb@delphix.com>
Sun, 03 May 2015 18:26:55 +0100
changeset 14277 711401aaa206
parent 14276 7b173d1a28b7
Delphix vmxnet3s fixes
usr/src/uts/intel/io/vmxnet3s/vmxnet3_main.c
usr/src/uts/intel/io/vmxnet3s/vmxnet3_rx.c
usr/src/uts/intel/io/vmxnet3s/vmxnet3_solaris.h
usr/src/uts/intel/io/vmxnet3s/vmxnet3s.conf
--- a/usr/src/uts/intel/io/vmxnet3s/vmxnet3_main.c	Thu Apr 30 20:04:18 2015 +0100
+++ b/usr/src/uts/intel/io/vmxnet3s/vmxnet3_main.c	Sun May 03 18:26:55 2015 +0100
@@ -14,7 +14,7 @@
  *
  *********************************************************/
 /*
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <vmxnet3_solaris.h>
@@ -179,26 +179,22 @@
          *val = 0;
          break;
       case MAC_STAT_RBYTES:
-         *val = rxStats->LROBytesRxOK +
-                rxStats->ucastBytesRxOK +
+         *val = rxStats->ucastBytesRxOK +
                 rxStats->mcastBytesRxOK +
                 rxStats->bcastBytesRxOK;
          break;
       case MAC_STAT_IPACKETS:
-         *val = rxStats->LROPktsRxOK +
-                rxStats->ucastPktsRxOK +
+         *val = rxStats->ucastPktsRxOK +
                 rxStats->mcastPktsRxOK +
                 rxStats->bcastPktsRxOK;
          break;
       case MAC_STAT_OBYTES:
-         *val = txStats->TSOBytesTxOK +
-                txStats->ucastBytesTxOK +
+         *val = txStats->ucastBytesTxOK +
                 txStats->mcastBytesTxOK +
                 txStats->bcastBytesTxOK;
          break;
       case MAC_STAT_OPACKETS:
-         *val = txStats->TSOPktsTxOK +
-                txStats->ucastPktsTxOK +
+         *val = txStats->ucastPktsTxOK +
                 txStats->mcastPktsTxOK +
                 txStats->bcastPktsTxOK;
          break;
@@ -959,11 +955,13 @@
  *
  * vmxnet3_change_mtu --
  *
- *    Change the MTU as seen by the driver. Reset the device and tx/rx queues
- *    so that buffers of right size are posted in rx queues.
+ *    Change the MTU as seen by the driver. This is only supported when
+ *    the mac is stopped.
  *
  * Results:
- *    EINVAL for invalid MTUs or other failures. 0 for success.
+ *    EBUSY if the device is enabled.
+ *    EINVAL for invalid MTU values.
+ *    0 on success.
  *
  * Side effects:
  *    None.
@@ -974,8 +972,11 @@
 static int
 vmxnet3_change_mtu(vmxnet3_softc_t *dp, uint32_t new_mtu)
 {
-   int ret = 0, do_reset = 0, macret;
-   ASSERT(dp);
+   int ret;
+
+   if (dp->devEnabled)
+      return EBUSY;
+
    if (new_mtu == dp->cur_mtu) {
       VMXNET3_WARN(dp, "New MTU is same as old mtu : %d.\n", new_mtu);
       return 0;
@@ -987,24 +988,10 @@
       return EINVAL;
    }
 
-   if (dp->devEnabled) {
-      do_reset = 1;
-      vmxnet3_stop(dp);
-      VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
-   }
-
    dp->cur_mtu = new_mtu;
 
-   if (do_reset) {
-      if ((ret = vmxnet3_start(dp)) != 0)
-	 VMXNET3_WARN(dp, "Unable to restart the device: %d", ret);
-   }
-
-   if ((macret = mac_maxsdu_update(dp->mac, new_mtu)) != 0) {
-      VMXNET3_WARN(dp, "Unable to update mac with %d mtu: %d", new_mtu, macret);
-      if (ret == 0)
-         ret = macret;
-   }
+   if ((ret = mac_maxsdu_update(dp->mac, new_mtu)) != 0)
+      VMXNET3_WARN(dp, "Unable to update mac with %d mtu: %d", new_mtu, ret);
 
    return ret;
 }
@@ -1256,7 +1243,7 @@
    vmxnet3_stop(dp);
    VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
    if ((ret = vmxnet3_start(dp)) != DDI_SUCCESS)
-      VMXNET3_WARN(dp, "failed to restart the device: %d", ret);
+      VMXNET3_WARN(dp, "failed to reset the device: %d", ret);
 }
 
 /*
@@ -1306,6 +1293,9 @@
          vmxnet3_refresh_linkstate(dp);
          linkStateChanged = B_TRUE;
       }
+      if (events & VMXNET3_ECR_DIC) {
+         VMXNET3_DEBUG(dp, 1, "device implementation change\n");
+      }
       VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_ECR, events);
    }
 
@@ -1826,12 +1816,5 @@
 void
 vmxnet3_log(int level, vmxnet3_softc_t *dp, char *fmt, ...)
 {
-   va_list ap;
-   char buf[256];
-
-   va_start(ap, fmt);
-   (void) vsnprintf(buf, sizeof (buf), fmt, ap);
-   va_end(ap);
-
-   cmn_err(level, VMXNET3_MODNAME ":%d: %s", dp->instance, buf);
+   dev_err(dp->dip, level, fmt);
 }
--- a/usr/src/uts/intel/io/vmxnet3s/vmxnet3_rx.c	Thu Apr 30 20:04:18 2015 +0100
+++ b/usr/src/uts/intel/io/vmxnet3s/vmxnet3_rx.c	Sun May 03 18:26:55 2015 +0100
@@ -14,7 +14,7 @@
  *
  *********************************************************/
 /*
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2013 by Delphix. All rights reserved.
  */
 
 #include <vmxnet3_solaris.h>
@@ -104,6 +104,42 @@
 /*
  *---------------------------------------------------------------------------
  *
+ * vmxnet3_put_rxpool_buf --
+ *
+ *    Return a rxBuf to the pool.
+ *
+ * Results:
+ *    B_TRUE if there was room in the pool and the rxBuf was returned,
+ *    B_FALSE otherwise.
+ *
+ * Side effects:
+ *    None.
+ *
+ *---------------------------------------------------------------------------
+ */
+static boolean_t
+vmxnet3_put_rxpool_buf(vmxnet3_softc_t *dp, vmxnet3_rxbuf_t *rxBuf)
+{
+   vmxnet3_rxpool_t *rxPool = &dp->rxPool;
+   boolean_t returned = B_FALSE;
+
+   mutex_enter(&dp->rxPoolLock);
+   ASSERT(rxPool->nBufs <= rxPool->nBufsLimit);
+   if (dp->devEnabled && rxPool->nBufs < rxPool->nBufsLimit) {
+      ASSERT((rxPool->listHead == NULL && rxPool->nBufs == 0) ||
+         (rxPool->listHead != NULL && rxPool->nBufs != 0));
+      rxBuf->next = rxPool->listHead;
+      rxPool->listHead = rxBuf;
+      rxPool->nBufs++;
+      returned = B_TRUE;
+   }
+   mutex_exit(&dp->rxPoolLock);
+   return returned;
+}
+
+/*
+ *---------------------------------------------------------------------------
+ *
  * vmxnet3_put_rxbuf --
  *
  *    Return a rxBuf to the pool or free it.
@@ -120,19 +156,44 @@
 vmxnet3_put_rxbuf(vmxnet3_rxbuf_t *rxBuf)
 {
    vmxnet3_softc_t *dp = rxBuf->dp;
-   vmxnet3_rxpool_t *rxPool = &dp->rxPool;
 
    VMXNET3_DEBUG(dp, 5, "free 0x%p\n", rxBuf);
 
+   if (!vmxnet3_put_rxpool_buf(dp, rxBuf))
+      vmxnet3_free_rxbuf(dp, rxBuf);
+}
+
+/*
+ *---------------------------------------------------------------------------
+ *
+ * vmxnet3_get_rxpool_buf --
+ *
+ *    Get an unused rxBuf from the pool.
+ *
+ * Results:
+ *    A rxBuf or NULL if there are no buffers in the pool.
+ *
+ * Side effects:
+ *    None.
+ *
+ *---------------------------------------------------------------------------
+ */
+static vmxnet3_rxbuf_t *
+vmxnet3_get_rxpool_buf(vmxnet3_softc_t *dp)
+{
+   vmxnet3_rxpool_t *rxPool = &dp->rxPool;
+   vmxnet3_rxbuf_t *rxBuf = NULL;
+
    mutex_enter(&dp->rxPoolLock);
-   if (dp->devEnabled && rxPool->nBufs < rxPool->nBufsLimit) {
-      rxBuf->next = rxPool->listHead;
-      rxPool->listHead = rxBuf;
-      mutex_exit(&dp->rxPoolLock);
-   } else {
-      mutex_exit(&dp->rxPoolLock);
-      vmxnet3_free_rxbuf(dp, rxBuf);
+   if (rxPool->listHead) {
+      rxBuf = rxPool->listHead;
+      rxPool->listHead = rxBuf->next;
+      rxPool->nBufs--;
+      ASSERT((rxPool->listHead == NULL && rxPool->nBufs == 0) ||
+         (rxPool->listHead != NULL && rxPool->nBufs != 0));
    }
+   mutex_exit(&dp->rxPoolLock);
+   return rxBuf;
 }
 
 /*
@@ -155,35 +216,24 @@
 vmxnet3_get_rxbuf(vmxnet3_softc_t *dp, boolean_t canSleep)
 {
    vmxnet3_rxbuf_t *rxBuf;
-   vmxnet3_rxpool_t *rxPool = &dp->rxPool;
 
-   mutex_enter(&dp->rxPoolLock);
-   if (rxPool->listHead) {
-      rxBuf = rxPool->listHead;
-      rxPool->listHead = rxBuf->next;
-      mutex_exit(&dp->rxPoolLock);
+   if ((rxBuf = vmxnet3_get_rxpool_buf(dp))) {
       VMXNET3_DEBUG(dp, 5, "alloc 0x%p from pool\n", rxBuf);
-   } else {
-      mutex_exit(&dp->rxPoolLock);
-      rxBuf = vmxnet3_alloc_rxbuf(dp, canSleep);
-      if (!rxBuf) {
-         goto done;
-      }
+   } else if ((rxBuf = vmxnet3_alloc_rxbuf(dp, canSleep))) {
       VMXNET3_DEBUG(dp, 5, "alloc 0x%p from mem\n", rxBuf);
    }
 
-   ASSERT(rxBuf);
-
-   rxBuf->mblk = desballoc((uchar_t *) rxBuf->dma.buf,
-                           rxBuf->dma.bufLen, BPRI_MED,
-                           &rxBuf->freeCB);
-   if (!rxBuf->mblk) {
-      vmxnet3_put_rxbuf(rxBuf);
-      atomic_inc_32(&dp->rx_alloc_failed);
-      rxBuf = NULL;
+   if (rxBuf) {
+      rxBuf->mblk = desballoc((uchar_t *) rxBuf->dma.buf,
+                              rxBuf->dma.bufLen, BPRI_MED,
+                              &rxBuf->freeCB);
+      if (!rxBuf->mblk) {
+         vmxnet3_put_rxbuf(rxBuf);
+         atomic_inc_32(&dp->rx_alloc_failed);
+         rxBuf = NULL;
+      }
    }
 
-done:
    return rxBuf;
 }
 
@@ -287,18 +337,14 @@
 void
 vmxnet3_rxqueue_fini(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq)
 {
-   vmxnet3_rxpool_t *rxPool = &dp->rxPool;
    vmxnet3_rxbuf_t *rxBuf;
    unsigned int i;
 
    ASSERT(!dp->devEnabled);
 
    /* First the rxPool */
-   while (rxPool->listHead) {
-      rxBuf = rxPool->listHead;
-      rxPool->listHead = rxBuf->next;
+   while ((rxBuf = vmxnet3_get_rxpool_buf(dp)))
       vmxnet3_free_rxbuf(dp, rxBuf);
-   }
 
    /* Then the ring */
    for (i = 0; i < rxq->cmdRing.size; i++) {
--- a/usr/src/uts/intel/io/vmxnet3s/vmxnet3_solaris.h	Thu Apr 30 20:04:18 2015 +0100
+++ b/usr/src/uts/intel/io/vmxnet3s/vmxnet3_solaris.h	Sun May 03 18:26:55 2015 +0100
@@ -24,6 +24,7 @@
 #include <sys/types.h>
 #include <sys/conf.h>
 #include <sys/debug.h>
+#include <sys/cmn_err.h>
 #include <sys/stropts.h>
 #include <sys/stream.h>
 #include <sys/strlog.h>
--- a/usr/src/uts/intel/io/vmxnet3s/vmxnet3s.conf	Thu Apr 30 20:04:18 2015 +0100
+++ b/usr/src/uts/intel/io/vmxnet3s/vmxnet3s.conf	Sun May 03 18:26:55 2015 +0100
@@ -7,7 +7,7 @@
 #    Minimum value: 32
 #    Maximum value: 4096
 #
-TxRingSize=256,256,256,256,256,256,256,256,256,256;
+TxRingSize=1024,1024,1024,1024,1024,1024,1024,1024,1024,1024;
 
 # RxRingSize --
 #
@@ -16,7 +16,7 @@
 #    Minimum value: 32
 #    Maximum value: 4096
 #
-RxRingSize=256,256,256,256,256,256,256,256,256,256;
+RxRingSize=1024,1024,1024,1024,1024,1024,1024,1024,1024,1024;
 
 # RxBufPoolLimit --
 #
@@ -27,7 +27,7 @@
 #    Minimum value: 0
 #    Maximum value: RxRingSize * 10
 #
-RxBufPoolLimit=512,512,512,512,512,512,512,512,512,512;
+RxBufPoolLimit=2048,2048,2048,2048,2048,2048,2048,2048,2048,2048;
 
 # EnableLSO --
 #