dma: fix DMA sync for drivers not calling dma_set_mask*()

There are several reports that the DMA sync shortcut broke non-coherent
devices.
dev->dma_need_sync is false after the &device allocation and if a driver
didn't call dma_set_mask*(), it will still be false even if the device
is not DMA-coherent and thus needs synchronizing. Due to historical
reasons, there's still a lot of drivers not calling it.
Invert the boolean, so that the sync will be performed by default and
the shortcut will be enabled only when calling dma_set_mask*().

Reported-by: Steven Price <steven.price@arm.com>
Closes: https://lore.kernel.org/lkml/010686f5-3049-46a1-8230-7752a1b433ff@arm.com
Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
Closes: https://lore.kernel.org/lkml/46160534-5003-4809-a408-6b3a3f4921e9@samsung.com
Fixes: f406c8e4b770. ("dma: avoid redundant calls for sync operations")
Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Tested-by: Steven Price <steven.price@arm.com>
Tested-by: Marek Szyprowski <m.szyprowski@samsung.com>

diff --git a/include/linux/device.h b/include/linux/device.h
index ed95b829f05bbe15da234578488104d63d8a5ea7..d4b50accff26c8ee084485e29774406bcaa6ce01 100644 (file)
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -691,7 +691,7 @@ struct device_physical_location {
  *             and optionall (if the coherent mask is large enough) also
  *             for dma allocations.  This flag is managed by the dma ops
  *             instance from ->dma_supported.
- * @dma_need_sync: The device needs performing DMA sync operations.
+ * @dma_skip_sync: DMA sync operations can be skipped for coherent buffers.
  *
  * At the lowest level, every device in a Linux system is represented by an
  * instance of struct device. The device structure contains the information
@@ -805,7 +805,7 @@ struct device {
        bool                    dma_ops_bypass : 1;
 #endif
 #ifdef CONFIG_DMA_NEED_SYNC
-       bool                    dma_need_sync:1;
+       bool                    dma_skip_sync:1;
 #endif
 };
 

diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 4893cb89cb5246ba3a3a33f4f7c3dcefa470664c..5217b922d29f7a813c867aa3a12fd22326bdc6d3 100644 (file)
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -280,8 +280,8 @@ static inline void dma_reset_need_sync(struct device *dev)
 {
 #ifdef CONFIG_DMA_NEED_SYNC
        /* Reset it only once so that the function can be called on hotpath */
-       if (unlikely(!dev->dma_need_sync))
-               dev->dma_need_sync = true;
+       if (unlikely(dev->dma_skip_sync))
+               dev->dma_skip_sync = false;
 #endif
 }
 

diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index eb4e15893b6c44f596568219b53f1fce902fa93f..f693aafe221f2cc38788aef62b0f5273a7dc80a3 100644 (file)
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -295,7 +295,7 @@ bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr);
 static inline bool dma_dev_need_sync(const struct device *dev)
 {
        /* Always call DMA sync operations when debugging is enabled */
-       return dev->dma_need_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG);
+       return !dev->dma_skip_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG);
 }
 
 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,

diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 3524bc92c37fd4649194be7049c4b02ba2bf6018..3f77c3f8d16d6d2dd2695612ef2deee9e6bd8ed4 100644 (file)
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -392,7 +392,7 @@ bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr)
 
        if (dma_map_direct(dev, ops))
                /*
-                * dma_need_sync could've been reset on first SWIOTLB buffer
+                * dma_skip_sync could've been reset on first SWIOTLB buffer
                 * mapping, but @dma_addr is not necessary an SWIOTLB buffer.
                 * In this case, fall back to more granular check.
                 */
@@ -407,20 +407,20 @@ static void dma_setup_need_sync(struct device *dev)
 
        if (dma_map_direct(dev, ops) || (ops->flags & DMA_F_CAN_SKIP_SYNC))
                /*
-                * dma_need_sync will be reset to %true on first SWIOTLB buffer
+                * dma_skip_sync will be reset to %false on first SWIOTLB buffer
                 * mapping, if any. During the device initialization, it's
                 * enough to check only for the DMA coherence.
                 */
-               dev->dma_need_sync = !dev_is_dma_coherent(dev);
+               dev->dma_skip_sync = dev_is_dma_coherent(dev);
        else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu &&
                 !ops->sync_sg_for_device && !ops->sync_sg_for_cpu)
                /*
                 * Synchronization is not possible when none of DMA sync ops
                 * is set.
                 */
-               dev->dma_need_sync = false;
+               dev->dma_skip_sync = true;
        else
-               dev->dma_need_sync = true;
+               dev->dma_skip_sync = false;
 }
 #else /* !CONFIG_DMA_NEED_SYNC */
 static inline void dma_setup_need_sync(struct device *dev) { }

diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index ae3e593eaadb98198b0267f23f3e1aa58abe60cd..068134697cf1bd012486902c1a10bef519e89bff 100644 (file)
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1409,7 +1409,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
        }
 
        /*
-        * If dma_need_sync wasn't set, reset it on first SWIOTLB buffer
+        * If dma_skip_sync was set, reset it on first SWIOTLB buffer
         * mapping to always sync SWIOTLB buffers.
         */
        dma_reset_need_sync(dev);