arm64: Provide dcache_by_myline_op_nosync helper

dcache_by_myline_op ensures completion of the data cache operations for a
region, while dcache_by_myline_op_nosync only issues them without waiting.
This enables deferred synchronization so completion for multiple regions
can be handled together later.

Cc: Leon Romanovsky <leon@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Ada Couprie Diaz <ada.coupriediaz@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Tangquan Zheng <zhengtangquan@oppo.com>
Tested-by: Xueyuan Chen <xueyuan.chen21@gmail.com>
Signed-off-by: Barry Song <baohua@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com>
Link: https://lore.kernel.org/r/20260228221216.59886-1-21cnbao@gmail.com

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index d3d46e5f7188403887788b654cfad26c4cdc0454..cdbaad41bddbf185bd159dd0e062119c6f397ce2 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -371,14 +371,13 @@ alternative_endif
  * [start, end) with dcache line size explicitly provided.
  *
  *     op:             operation passed to dc instruction
- *     domain:         domain used in dsb instruction
  *     start:          starting virtual address of the region
  *     end:            end virtual address of the region
  *     linesz:         dcache line size
  *     fixup:          optional label to branch to on user fault
  *     Corrupts:       start, end, tmp
  */
-       .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup
+       .macro dcache_by_myline_op_nosync op, start, end, linesz, tmp, fixup
        sub     \tmp, \linesz, #1
        bic     \start, \start, \tmp
 alternative_if ARM64_WORKAROUND_4311569
@@ -412,14 +411,28 @@ alternative_if ARM64_WORKAROUND_4311569
        cbnz    \start, .Ldcache_op\@
        .endif
 alternative_else_nop_endif
-       dsb     \domain
 
        _cond_uaccess_extable .Ldcache_op\@, \fixup
        .endm
 
 /*
  * Macro to perform a data cache maintenance for the interval
- * [start, end)
+ * [start, end) without waiting for completion
+ *
+ *     op:             operation passed to dc instruction
+ *     start:          starting virtual address of the region
+ *     end:            end virtual address of the region
+ *     fixup:          optional label to branch to on user fault
+ *     Corrupts:       start, end, tmp1, tmp2
+ */
+       .macro dcache_by_line_op_nosync op, start, end, tmp1, tmp2, fixup
+       dcache_line_size \tmp1, \tmp2
+       dcache_by_myline_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup
+       .endm
+
+/*
+ * Macro to perform a data cache maintenance for the interval
+ * [start, end) and wait for completion
  *
  *     op:             operation passed to dc instruction
  *     domain:         domain used in dsb instruction
@@ -429,8 +442,8 @@ alternative_else_nop_endif
  *     Corrupts:       start, end, tmp1, tmp2
  */
        .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup
-       dcache_line_size \tmp1, \tmp2
-       dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup
+       dcache_by_line_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup
+       dsb \domain
        .endm
 
 /*

diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index 413f899e4ac639414dcfaf2e5f0a7152fc55b938..6cb4209f5dab56a998c9e3ab95f73ce3099a4f29 100644 (file)
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -64,7 +64,8 @@ SYM_CODE_START(arm64_relocate_new_kernel)
        mov     x19, x13
        copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
        add     x1, x19, #PAGE_SIZE
-       dcache_by_myline_op civac, sy, x19, x1, x15, x20
+       dcache_by_myline_op_nosync civac, x19, x1, x15, x20
+       dsb     sy
        b       .Lnext
 .Ltest_indirection:
        tbz     x16, IND_INDIRECTION_BIT, .Ltest_destination