[gve] Allow for out-of-order buffer consumption

We currently assume that the buffer index is equal to the descriptor
ring index, which is correct only for in-order queues.

Out-of-order queues will include a buffer tag value that is copied
from the descriptor to the completion.  Redefine the data buffers as
being indexed by this tag value (rather than by the descriptor ring
index), and add a circular ring buffer to allow for tags to be reused
in whatever order they are released by the hardware.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

diff --git a/src/drivers/net/gve.c b/src/drivers/net/gve.c
index 80eb2e012e4e83c52de6ac2aabcf23dfcc37fc39..0fb60ac90d100cccb8e066433ded42f4aa7933b4 100644 (file)
--- a/src/drivers/net/gve.c
+++ b/src/drivers/net/gve.c
@@ -124,6 +124,59 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
                EIO_ADMIN_UNAUTH, EIO_ADMIN_RESOURCE,                   \
                EIO_ADMIN_UNAVAIL, EIO_ADMIN_NOTSUP, EIO_ADMIN_UNKNOWN )
 
+/******************************************************************************
+ *
+ * Buffer layout
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get buffer offset (within queue page list allocation)
+ *
+ * @v queue            Descriptor queue
+ * @v tag              Buffer tag
+ * @ret addr           Buffer address within queue page list address space
+ */
+static inline __attribute__ (( always_inline)) size_t
+gve_offset ( struct gve_queue *queue, unsigned int tag ) {
+
+       /* We allocate sufficient pages for the maximum fill level of
+        * buffers, and reuse the buffers in strict rotation as they
+        * are released by the hardware.
+        */
+       assert ( tag < queue->fill );
+       return ( tag * GVE_BUF_SIZE );
+}
+
+/**
+ * Get buffer address (within queue page list address space)
+ *
+ * @v queue            Descriptor queue
+ * @v tag              Buffer tag
+ * @ret addr           Buffer address within queue page list address space
+ */
+static inline __attribute__ (( always_inline)) physaddr_t
+gve_address ( struct gve_queue *queue, unsigned int tag ) {
+
+       /* Pages are allocated as a single contiguous block */
+       return ( queue->qpl.base + gve_offset ( queue, tag ) );
+}
+
+/**
+ * Get buffer address
+ *
+ * @v queue            Descriptor queue
+ * @v tag              Buffer tag
+ * @ret addr           Buffer address
+ */
+static inline __attribute__ (( always_inline )) void *
+gve_buffer ( struct gve_queue *queue, unsigned int tag ) {
+
+       /* Pages are allocated as a single contiguous block */
+       return ( queue->qpl.data + gve_offset ( queue, tag ) );
+}
+
 /******************************************************************************
  *
  * Device reset
@@ -694,14 +747,29 @@ static void gve_create_rx_param ( struct gve_queue *queue, uint32_t qpl,
 static int gve_create_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
        const struct gve_queue_type *type = queue->type;
        union gve_admin_command *cmd;
+       struct gve_buffer *buf;
        unsigned int db_off;
        unsigned int evt_idx;
+       unsigned int tag;
+       unsigned int i;
        uint32_t qpl;
        int rc;
 
        /* Reset queue */
        queue->prod = 0;
        queue->cons = 0;
+       memset ( queue->desc.raw, 0, ( queue->count * type->desc_len ) );
+       memset ( queue->cmplt.raw, 0, ( queue->count * type->cmplt_len ) );
+       for ( i = 0 ; i < queue->fill ; i++ )
+               queue->tag[i] = i;
+
+       /* Pre-populate descriptor offsets */
+       buf = ( queue->desc.raw + type->desc_len - sizeof ( *buf ) );
+       for ( i = 0 ; i < queue->count ; i++ ) {
+               tag = ( i & ( queue->fill - 1 ) );
+               buf->addr = cpu_to_be64 ( gve_address ( queue, tag ) );
+               buf = ( ( ( void * ) buf ) + type->desc_len );
+       }
 
        /* Construct request */
        cmd = gve_admin_command ( gve );
@@ -861,51 +929,6 @@ static void gve_free_qpl ( struct gve_nic *nic __unused,
        dma_ufree ( &qpl->map, qpl->data, len );
 }
 
-/**
- * Get buffer offset (within queue page list allocation)
- *
- * @v queue            Descriptor queue
- * @v index            Buffer index
- * @ret addr           Buffer address within queue page list address space
- */
-static inline __attribute__ (( always_inline)) size_t
-gve_offset ( struct gve_queue *queue, unsigned int index ) {
-
-       /* We allocate sufficient pages for the maximum fill level of
-        * buffers, and reuse the pages in strict rotation as we
-        * progress through the queue.
-        */
-       return ( ( index & ( queue->fill - 1 ) ) * GVE_BUF_SIZE );
-}
-
-/**
- * Get buffer address (within queue page list address space)
- *
- * @v queue            Descriptor queue
- * @v index            Buffer index
- * @ret addr           Buffer address within queue page list address space
- */
-static inline __attribute__ (( always_inline)) physaddr_t
-gve_address ( struct gve_queue *queue, unsigned int index ) {
-
-       /* Pages are allocated as a single contiguous block */
-       return ( queue->qpl.base + gve_offset ( queue, index ) );
-}
-
-/**
- * Get buffer address
- *
- * @v queue            Descriptor queue
- * @v index            Buffer index
- * @ret addr           Buffer address
- */
-static inline __attribute__ (( always_inline )) void *
-gve_buffer ( struct gve_queue *queue, unsigned int index ) {
-
-       /* Pages are allocated as a single contiguous block */
-       return ( queue->qpl.data + gve_offset ( queue, index ) );
-}
-
 /**
  * Calculate next receive sequence number
  *
@@ -944,8 +967,6 @@ static int gve_alloc_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
        size_t desc_len = ( queue->count * type->desc_len );
        size_t cmplt_len = ( queue->count * type->cmplt_len );
        size_t res_len = sizeof ( *queue->res );
-       struct gve_buffer *buf;
-       unsigned int i;
        int rc;
 
        /* Sanity checks */
@@ -1002,13 +1023,6 @@ static int gve_alloc_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
        }
        memset ( queue->res, 0, res_len );
 
-       /* Populate descriptor offsets */
-       buf = ( queue->desc.raw + type->desc_len - sizeof ( *buf ) );
-       for ( i = 0 ; i < queue->count ; i++ ) {
-               buf->addr = cpu_to_be64 ( gve_address ( queue, i ) );
-               buf = ( ( ( void * ) buf ) + type->desc_len );
-       }
-
        return 0;
 
        dma_free ( &queue->res_map, queue->res, res_len );
@@ -1073,9 +1087,6 @@ static int gve_start ( struct gve_nic *gve ) {
                        netdev_tx_complete_err ( netdev, iobuf, -ECANCELED );
        }
 
-       /* Invalidate receive completions */
-       memset ( rx->cmplt.raw, 0, ( rx->count * rx->type->cmplt_len ) );
-
        /* Reset receive sequence */
        gve->seq = gve_next ( 0 );
 
@@ -1309,6 +1320,7 @@ static int gve_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
        struct gve_tx_descriptor *desc;
        unsigned int count;
        unsigned int index;
+       unsigned int tag;
        size_t frag_len;
        size_t offset;
        size_t len;
@@ -1326,20 +1338,24 @@ static int gve_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
        }
 
        /* Copy packet to queue pages and populate descriptors */
-       for ( offset = 0 ; offset < len ; offset += frag_len ) {
+       offset = 0;
+       while ( 1 ) {
+
+               /* Identify next available buffer */
+               index = ( tx->prod++ & ( tx->count - 1 ) );
+               tag = tx->tag[ index % GVE_TX_FILL ];
 
                /* Sanity check */
-               assert ( gve->tx_iobuf[ tx->prod % GVE_TX_FILL ] == NULL );
+               assert ( gve->tx_iobuf[tag] == NULL );
 
                /* Copy packet fragment */
                frag_len = ( len - offset );
                if ( frag_len > GVE_BUF_SIZE )
                        frag_len = GVE_BUF_SIZE;
-               memcpy ( gve_buffer ( tx, tx->prod ),
+               memcpy ( gve_buffer ( tx, tag ),
                         ( iobuf->data + offset ), frag_len );
 
                /* Populate descriptor */
-               index = ( tx->prod++ & ( tx->count - 1 ) );
                desc = &tx->desc.tx[index];
                if ( offset ) {
                        desc->type = GVE_TX_TYPE_CONT;
@@ -1354,13 +1370,19 @@ static int gve_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
                DBGC2 ( gve, "GVE %p TX %#04x %#02x:%#02x len %#04x/%#04x at "
                        "%#08zx\n", gve, index, desc->type, desc->count,
                        be16_to_cpu ( desc->len ), be16_to_cpu ( desc->total ),
-                       gve_offset ( tx, index ) );
+                       gve_offset ( tx, tag ) );
+
+               /* Move to next descriptor */
+               offset += frag_len;
+               if ( offset < len )
+                       continue;
+
+               /* Record I/O buffer against final descriptor */
+               gve->tx_iobuf[tag] = iobuf;
+               break;
        }
        assert ( ( tx->prod - tx->cons ) <= tx->fill );
 
-       /* Record I/O buffer against final descriptor */
-       gve->tx_iobuf[ ( tx->prod - 1U ) % GVE_TX_FILL ] = iobuf;
-
        /* Ring doorbell */
        wmb();
        writel ( bswap_32 ( tx->prod ), tx->db );
@@ -1377,6 +1399,7 @@ static void gve_poll_tx ( struct net_device *netdev ) {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct io_buffer *iobuf;
+       unsigned int tag;
        uint32_t count;
 
        /* Read event counter */
@@ -1385,8 +1408,9 @@ static void gve_poll_tx ( struct net_device *netdev ) {
        /* Process transmit completions */
        while ( count != tx->cons ) {
                DBGC2 ( gve, "GVE %p TX %#04x complete\n", gve, tx->cons );
-               iobuf = gve->tx_iobuf[ tx->cons % GVE_TX_FILL ];
-               gve->tx_iobuf[ tx->cons % GVE_TX_FILL ] = NULL;
+               tag = ( tx->cons % GVE_TX_FILL );
+               iobuf = gve->tx_iobuf[tag];
+               gve->tx_iobuf[tag] = NULL;
                tx->cons++;
                if ( iobuf )
                        netdev_tx_complete ( netdev, iobuf );
@@ -1405,6 +1429,7 @@ static void gve_poll_rx ( struct net_device *netdev ) {
        struct io_buffer *iobuf;
        unsigned int index;
        unsigned int seq;
+       unsigned int tag;
        uint32_t cons;
        size_t total;
        size_t len;
@@ -1427,9 +1452,10 @@ static void gve_poll_rx ( struct net_device *netdev ) {
 
                /* Parse completion */
                len = be16_to_cpu ( cmplt->len );
+               tag = ( index % GVE_RX_FILL );
                DBGC2 ( gve, "GVE %p RX %#04x %#02x:%#02x len %#04zx at "
                        "%#08zx\n", gve, index, cmplt->seq, cmplt->flags,
-                       len, gve_offset ( rx, index ) );
+                       len, gve_offset ( rx, tag ) );
 
                /* Accumulate a complete packet */
                if ( cmplt->flags & GVE_RXF_ERROR ) {
@@ -1445,15 +1471,16 @@ static void gve_poll_rx ( struct net_device *netdev ) {
                iobuf = ( total ? alloc_iob ( total ) : NULL );
                for ( ; rx->cons != cons ; rx->cons++ ) {
 
-                       /* Re-read completion length */
+                       /* Re-read completion */
                        index = ( rx->cons & ( rx->count - 1 ) );
                        cmplt = &rx->cmplt.rx[index];
+                       tag = ( index % GVE_RX_FILL );
 
                        /* Copy data */
                        if ( iobuf ) {
                                len = be16_to_cpu ( cmplt->len );
                                memcpy ( iob_put ( iobuf, len ),
-                                        gve_buffer ( rx, rx->cons ), len );
+                                        gve_buffer ( rx, tag ), len );
                        }
                }
                assert ( ( iobuf == NULL ) || ( iob_len ( iobuf ) == total ) );
@@ -1628,6 +1655,8 @@ static int gve_probe ( struct pci_device *pci ) {
        gve->netdev = netdev;
        gve->tx.type = &gve_tx_type;
        gve->rx.type = &gve_rx_type;
+       gve->tx.tag = gve->tx_tag;
+       gve->rx.tag = gve->rx_tag;
        process_init_stopped ( &gve->startup, &gve_startup_desc,
                               &netdev->refcnt );
        timer_init ( &gve->watchdog, gve_watchdog, &netdev->refcnt );

diff --git a/src/drivers/net/gve.h b/src/drivers/net/gve.h
index d3b7ca760c4490ff8600ea807f1b873eca5ea4a4..2ad713f903cedb724b292637faba8ff51df843d7 100644 (file)
--- a/src/drivers/net/gve.h
+++ b/src/drivers/net/gve.h
@@ -666,6 +666,8 @@ struct gve_queue {
        uint32_t prod;
        /** Consumer counter */
        uint32_t cons;
+       /** Tag ring */
+       uint8_t *tag;
 
        /** Queue page list */
        struct gve_qpl qpl;
@@ -730,8 +732,12 @@ struct gve_nic {
        struct gve_queue tx;
        /** Receive queue */
        struct gve_queue rx;
-       /** Transmit I/O buffers */
+       /** Transmit I/O buffers (indexed by tag) */
        struct io_buffer *tx_iobuf[GVE_TX_FILL];
+       /** Transmit tag ring */
+       uint8_t tx_tag[GVE_TX_FILL];
+       /** Receive tag ring */
+       uint8_t rx_tag[GVE_RX_FILL];
        /** Receive sequence number */
        unsigned int seq;