[riscv] Invalidate data cache on completed RX DMA buffers

The data cache must be invalidated twice for RX DMA buffers: once
before passing ownership to the DMA device (in case the cache happens
to contain dirty data that will be written back at an undefined future
point), and once after receiving ownership from the DMA device (in
case the CPU happens to have speculatively accessed data in the buffer
while it was owned by the hardware).

Only the used portion of the buffer needs to be invalidated after
completion, since we do not care about data within the unused portion.

Update the DMA API to include the used length as an additional
parameter to dma_unmap(), and add the necessary second cache
invalidation pass to the RISC-V DMA API implementation.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

diff --git a/src/arch/riscv/core/riscv_dma.c b/src/arch/riscv/core/riscv_dma.c
index d215fba2c40e4da49e5fc68e0d4a335f1ac0fb41..ab88f2785aacf6225a7df943a536585ebe16500f 100644 (file)
--- a/src/arch/riscv/core/riscv_dma.c
+++ b/src/arch/riscv/core/riscv_dma.c
@@ -44,24 +44,57 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v flags            Mapping flags
  * @ret rc             Return status code
  */
-static int riscv_dma_map ( struct dma_device *dma __unused,
-                          struct dma_mapping *map __unused,
+static int riscv_dma_map ( struct dma_device *dma,
+                          struct dma_mapping *map,
                           void *addr, size_t len, int flags ) {
 
        /* Sanity check: we cannot support bidirectional mappings */
        assert ( ! ( ( flags & DMA_TX ) & ( flags & DMA_RX ) ) );
 
+       /* Populate mapping */
+       map->dma = dma;
+       map->offset = 0;
+       map->token = NULL;
+
        /* Flush cached data to transmit buffers */
        if ( flags & DMA_TX )
                cache_clean ( addr, len );
 
-       /* Invalidate cached data in receive buffers */
-       if ( flags & DMA_RX )
+       /* Invalidate cached data in receive buffers and record address */
+       if ( flags & DMA_RX ) {
                cache_invalidate ( addr, len );
+               map->token = addr;
+       }
+
+       /* Increment mapping count (for debugging) */
+       if ( DBG_LOG )
+               dma->mapped++;
 
        return 0;
 }
 
+/**
+ * Unmap buffer
+ *
+ * @v map              DMA mapping
+ * @v len              Used length
+ */
+static void riscv_dma_unmap ( struct dma_mapping *map, size_t len ) {
+       struct dma_device *dma = map->dma;
+       void *addr = map->token;
+
+       /* Invalidate cached data in receive buffers */
+       if ( addr )
+               cache_invalidate ( addr, len );
+
+       /* Clear mapping */
+       map->dma = NULL;
+
+       /* Decrement mapping count (for debugging) */
+       if ( DBG_LOG )
+               dma->mapped--;
+}
+
 /**
  * Allocate and map DMA-coherent buffer
  *
@@ -98,6 +131,10 @@ static void * riscv_dma_alloc ( struct dma_device *dma,
        DBGC ( dma, "DMA allocated [%#08lx,%#08lx) via %p\n",
               phys, ( phys + len ), caddr );
 
+       /* Increment allocation count (for debugging) */
+       if ( DBG_LOG )
+               dma->allocated++;
+
        return caddr;
 }
 
@@ -111,6 +148,7 @@ static void * riscv_dma_alloc ( struct dma_device *dma,
  */
 static void riscv_dma_free ( struct dma_mapping *map,
                             void *addr, size_t len ) {
+       struct dma_device *dma = map->dma;
 
        /* Sanity check */
        assert ( virt_to_phys ( addr ) == virt_to_phys ( map->token ) );
@@ -121,10 +159,14 @@ static void riscv_dma_free ( struct dma_mapping *map,
        /* Clear mapping */
        map->dma = NULL;
        map->token = NULL;
+
+       /* Decrement allocation count (for debugging) */
+       if ( DBG_LOG )
+               dma->allocated--;
 }
 
 PROVIDE_DMAAPI ( riscv, dma_map, riscv_dma_map );
-PROVIDE_DMAAPI_INLINE ( riscv, dma_unmap );
+PROVIDE_DMAAPI ( riscv, dma_unmap, riscv_dma_unmap );
 PROVIDE_DMAAPI ( riscv, dma_alloc, riscv_dma_alloc );
 PROVIDE_DMAAPI ( riscv, dma_free, riscv_dma_free );
 PROVIDE_DMAAPI ( riscv, dma_umalloc, riscv_dma_alloc );

diff --git a/src/arch/riscv/include/ipxe/riscv_dma.h b/src/arch/riscv/include/ipxe/riscv_dma.h
index 568f28afe928125f6d5d07b2dc8f47cd420580f4..d35904d88b7bc41aebe5a38941d697d740d32170 100644 (file)
--- a/src/arch/riscv/include/ipxe/riscv_dma.h
+++ b/src/arch/riscv/include/ipxe/riscv_dma.h
@@ -15,17 +15,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define DMAAPI_PREFIX_riscv __riscv_
 #endif
 
-/**
- * Unmap buffer
- *
- * @v map              DMA mapping
- */
-static inline __always_inline void
-DMAAPI_INLINE ( riscv, dma_unmap ) ( struct dma_mapping *map __unused ) {
-
-       /* Nothing to do */
-}
-
 /**
  * Set addressable space mask
  *

diff --git a/src/core/dma.c b/src/core/dma.c
index cbc9b8d10f6a80d56f4f5fcaf39cf32a18f5264a..3f3023c4d7162b6c46e93e475adb5d10092c1f67 100644 (file)
--- a/src/core/dma.c
+++ b/src/core/dma.c
@@ -79,13 +79,14 @@ static int dma_op_map ( struct dma_device *dma, struct dma_mapping *map,
  * Unmap buffer
  *
  * @v map              DMA mapping
+ * @v len              Used length
  */
-static void dma_op_unmap ( struct dma_mapping *map ) {
+static void dma_op_unmap ( struct dma_mapping *map, size_t len ) {
        struct dma_device *dma = map->dma;
 
        assert ( dma != NULL );
        assert ( dma->op != NULL );
-       dma->op->unmap ( dma, map );
+       dma->op->unmap ( dma, map, len );
 }
 
 /**

diff --git a/src/drivers/net/intelxl.c b/src/drivers/net/intelxl.c
index 1d4c22a09773500aaf3e62d5ca8be226cbf66bf0..31a3e25774dad6b7136229229f1fbf6267ab6fc9 100644 (file)
--- a/src/drivers/net/intelxl.c
+++ b/src/drivers/net/intelxl.c
@@ -90,7 +90,7 @@ int intelxl_msix_enable ( struct intelxl_nic *intelxl,
 
        pci_msix_disable ( pci, &intelxl->msix.cap );
  err_enable:
-       dma_unmap ( &intelxl->msix.map );
+       dma_unmap ( &intelxl->msix.map, sizeof ( intelxl->msix.msg ) );
  err_map:
        return rc;
 }
@@ -112,7 +112,7 @@ void intelxl_msix_disable ( struct intelxl_nic *intelxl,
        pci_msix_disable ( pci, &intelxl->msix.cap );
 
        /* Unmap dummy target location */
-       dma_unmap ( &intelxl->msix.map );
+       dma_unmap ( &intelxl->msix.map, sizeof ( intelxl->msix.msg ) );
 }
 
 /******************************************************************************

diff --git a/src/include/ipxe/dma.h b/src/include/ipxe/dma.h
index b675df2123ce4c6ed8be5a485a2ab490863a3e22..a6e41c1abbf5fdc663efa5c7c73befc0d4648723 100644 (file)
--- a/src/include/ipxe/dma.h
+++ b/src/include/ipxe/dma.h
@@ -74,8 +74,10 @@ struct dma_operations {
         *
         * @v dma               DMA device
         * @v map               DMA mapping
+        * @v len               Used length
         */
-       void ( * unmap ) ( struct dma_device *dma, struct dma_mapping *map );
+       void ( * unmap ) ( struct dma_device *dma, struct dma_mapping *map,
+                          size_t len );
        /**
         * Allocate and map DMA-coherent buffer
         *
@@ -194,9 +196,11 @@ DMAAPI_INLINE ( flat, dma_map ) ( struct dma_device *dma,
  * Unmap buffer
  *
  * @v map              DMA mapping
+ * @v len              Used length
  */
 static inline __always_inline void
-DMAAPI_INLINE ( flat, dma_unmap ) ( struct dma_mapping *map ) {
+DMAAPI_INLINE ( flat, dma_unmap ) ( struct dma_mapping *map,
+                                   size_t len __unused ) {
 
        /* Decrement mapping count (for debugging) */
        if ( DBG_LOG ) {
@@ -365,8 +369,9 @@ int dma_map ( struct dma_device *dma, struct dma_mapping *map,
  * Unmap buffer
  *
  * @v map              DMA mapping
+ * @v len              Used length
  */
-void dma_unmap ( struct dma_mapping *map );
+void dma_unmap ( struct dma_mapping *map, size_t len );
 
 /**
  * Allocate and map DMA-coherent buffer

diff --git a/src/include/ipxe/iobuf.h b/src/include/ipxe/iobuf.h
index df8d0b8544fea8354ac04202c57845c7d498b174..46b350458f2198b56aa08b885d3c4fe9b38bb309 100644 (file)
--- a/src/include/ipxe/iobuf.h
+++ b/src/include/ipxe/iobuf.h
@@ -276,7 +276,7 @@ static inline __always_inline physaddr_t iob_dma ( struct io_buffer *iobuf ) {
  * @ret rc             Return status code
  */
 static inline __always_inline void iob_unmap ( struct io_buffer *iobuf ) {
-       dma_unmap ( &iobuf->map );
+       dma_unmap ( &iobuf->map, iob_len ( iobuf ) );
 }
 
 extern struct io_buffer * __malloc alloc_iob_raw ( size_t len, size_t align,

diff --git a/src/interface/efi/efi_pci.c b/src/interface/efi/efi_pci.c
index ae0ec4264723a8a486c8f6ae36de66884a3578dc..fbf06300b74c1d8003ad2ab36cf41990a1d79f18 100644 (file)
--- a/src/interface/efi/efi_pci.c
+++ b/src/interface/efi/efi_pci.c
@@ -537,9 +537,10 @@ static int efipci_dma_map ( struct dma_device *dma, struct dma_mapping *map,
  *
  * @v dma              DMA device
  * @v map              DMA mapping
+ * @v len              Used length
  */
 static void efipci_dma_unmap ( struct dma_device *dma,
-                              struct dma_mapping *map ) {
+                              struct dma_mapping *map, size_t len __unused ) {
        struct efi_pci_device *efipci =
                container_of ( dma, struct efi_pci_device, pci.dma );
        EFI_PCI_IO_PROTOCOL *pci_io = efipci->io;
@@ -606,7 +607,7 @@ static void * efipci_dma_alloc ( struct dma_device *dma,
 
        return addr;
 
-       efipci_dma_unmap ( dma, map );
+       efipci_dma_unmap ( dma, map, len );
  err_map:
        pci_io->FreeBuffer ( pci_io, pages, addr );
  err_alloc:
@@ -632,7 +633,7 @@ static void efipci_dma_free ( struct dma_device *dma, struct dma_mapping *map,
        pages = ( ( len + EFI_PAGE_SIZE - 1 ) / EFI_PAGE_SIZE );
 
        /* Unmap buffer */
-       efipci_dma_unmap ( dma, map );
+       efipci_dma_unmap ( dma, map, len );
 
        /* Free buffer */
        pci_io->FreeBuffer ( pci_io, pages, addr );