Skip to content

Commit

Permalink
ARM: 9385/2: mm: Type-annotate all cache assembly routines
Browse files Browse the repository at this point in the history
Tag all references to assembly functions with SYM_TYPED_FUNC_START()
and SYM_FUNC_END() so they also become CFI-safe.

When we add SYM_TYPED_FUNC_START() to assembly calls, a function
prototype signature will be emitted into the object file at
(pc-4) at the call site, so that the KCFI runtime check can compare
this to the expected call. Example:

8011ae38:       a540670c        .word   0xa540670c

8011ae3c <v7_flush_icache_all>:
8011ae3c:       e3a00000        mov     r0, #0
8011ae40:       ee070f11        mcr     15, 0, r0, cr7, cr1, {0}
8011ae44:       e12fff1e        bx      lr

This means no "fallthrough" code can enter a SYM_TYPED_FUNC_START()
call from above it: there will be a function prototype signature
there, so those are consistently converted to a branch or ret lr
depending on context.

Tested-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Sami Tolvanen <samitolvanen@google.com>
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
  • Loading branch information
Linus Walleij authored and Russell King (Oracle) committed Apr 29, 2024
1 parent 6b0ef27 commit 1036b89
Show file tree
Hide file tree
Showing 22 changed files with 544 additions and 373 deletions.
39 changes: 24 additions & 15 deletions arch/arm/mm/cache-fa.S
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include <asm/page.h>

Expand Down Expand Up @@ -39,26 +40,28 @@
*
* Unconditionally clean and invalidate the entire icache.
*/
ENTRY(fa_flush_icache_all)
SYM_TYPED_FUNC_START(fa_flush_icache_all)
mov r0, #0
mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache
ret lr
ENDPROC(fa_flush_icache_all)
SYM_FUNC_END(fa_flush_icache_all)

/*
* flush_user_cache_all()
*
* Clean and invalidate all cache entries in a particular address
* space.
*/
ENTRY(fa_flush_user_cache_all)
/* FALLTHROUGH */
SYM_TYPED_FUNC_START(fa_flush_user_cache_all)
b fa_flush_kern_cache_all
SYM_FUNC_END(fa_flush_user_cache_all)

/*
* flush_kern_cache_all()
*
* Clean and invalidate the entire cache.
*/
ENTRY(fa_flush_kern_cache_all)
SYM_TYPED_FUNC_START(fa_flush_kern_cache_all)
mov ip, #0
mov r2, #VM_EXEC
__flush_whole_cache:
Expand All @@ -69,6 +72,7 @@ __flush_whole_cache:
mcrne p15, 0, ip, c7, c10, 4 @ drain write buffer
mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush
ret lr
SYM_FUNC_END(fa_flush_kern_cache_all)

/*
* flush_user_cache_range(start, end, flags)
Expand All @@ -80,7 +84,7 @@ __flush_whole_cache:
* - end - end address (exclusive, page aligned)
* - flags - vma_area_struct flags describing address space
*/
ENTRY(fa_flush_user_cache_range)
SYM_TYPED_FUNC_START(fa_flush_user_cache_range)
mov ip, #0
sub r3, r1, r0 @ calculate total size
cmp r3, #CACHE_DLIMIT @ total size >= limit?
Expand All @@ -97,6 +101,7 @@ ENTRY(fa_flush_user_cache_range)
mcrne p15, 0, ip, c7, c10, 4 @ data write barrier
mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush
ret lr
SYM_FUNC_END(fa_flush_user_cache_range)

/*
* coherent_kern_range(start, end)
Expand All @@ -108,8 +113,9 @@ ENTRY(fa_flush_user_cache_range)
* - start - virtual start address
* - end - virtual end address
*/
ENTRY(fa_coherent_kern_range)
/* fall through */
SYM_TYPED_FUNC_START(fa_coherent_kern_range)
b fa_coherent_user_range
SYM_FUNC_END(fa_coherent_kern_range)

/*
* coherent_user_range(start, end)
Expand All @@ -121,7 +127,7 @@ ENTRY(fa_coherent_kern_range)
* - start - virtual start address
* - end - virtual end address
*/
ENTRY(fa_coherent_user_range)
SYM_TYPED_FUNC_START(fa_coherent_user_range)
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D entry
mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry
Expand All @@ -133,6 +139,7 @@ ENTRY(fa_coherent_user_range)
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
mcr p15, 0, r0, c7, c5, 4 @ prefetch flush
ret lr
SYM_FUNC_END(fa_coherent_user_range)

/*
* flush_kern_dcache_area(void *addr, size_t size)
Expand All @@ -143,7 +150,7 @@ ENTRY(fa_coherent_user_range)
* - addr - kernel address
* - size - size of region
*/
ENTRY(fa_flush_kern_dcache_area)
SYM_TYPED_FUNC_START(fa_flush_kern_dcache_area)
add r1, r0, r1
1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line
add r0, r0, #CACHE_DLINESIZE
Expand All @@ -153,6 +160,7 @@ ENTRY(fa_flush_kern_dcache_area)
mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
ret lr
SYM_FUNC_END(fa_flush_kern_dcache_area)

/*
* dma_inv_range(start, end)
Expand Down Expand Up @@ -203,7 +211,7 @@ fa_dma_clean_range:
* - start - virtual start address of region
* - end - virtual end address of region
*/
ENTRY(fa_dma_flush_range)
SYM_TYPED_FUNC_START(fa_dma_flush_range)
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D entry
add r0, r0, #CACHE_DLINESIZE
Expand All @@ -212,30 +220,31 @@ ENTRY(fa_dma_flush_range)
mov r0, #0
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
ret lr
SYM_FUNC_END(fa_dma_flush_range)

/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(fa_dma_map_area)
SYM_TYPED_FUNC_START(fa_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq fa_dma_clean_range
bcs fa_dma_inv_range
b fa_dma_flush_range
ENDPROC(fa_dma_map_area)
SYM_FUNC_END(fa_dma_map_area)

/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(fa_dma_unmap_area)
SYM_TYPED_FUNC_START(fa_dma_unmap_area)
ret lr
ENDPROC(fa_dma_unmap_area)
SYM_FUNC_END(fa_dma_unmap_area)

.globl fa_flush_kern_cache_louis
.equ fa_flush_kern_cache_louis, fa_flush_kern_cache_all
Expand Down
51 changes: 30 additions & 21 deletions arch/arm/mm/cache-nop.S
Original file line number Diff line number Diff line change
@@ -1,47 +1,56 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>

#include "proc-macros.S"

ENTRY(nop_flush_icache_all)
SYM_TYPED_FUNC_START(nop_flush_icache_all)
ret lr
ENDPROC(nop_flush_icache_all)
SYM_FUNC_END(nop_flush_icache_all)

.globl nop_flush_kern_cache_all
.equ nop_flush_kern_cache_all, nop_flush_icache_all
SYM_TYPED_FUNC_START(nop_flush_kern_cache_all)
ret lr
SYM_FUNC_END(nop_flush_kern_cache_all)

.globl nop_flush_kern_cache_louis
.equ nop_flush_kern_cache_louis, nop_flush_icache_all

.globl nop_flush_user_cache_all
.equ nop_flush_user_cache_all, nop_flush_icache_all
SYM_TYPED_FUNC_START(nop_flush_user_cache_all)
ret lr
SYM_FUNC_END(nop_flush_user_cache_all)

.globl nop_flush_user_cache_range
.equ nop_flush_user_cache_range, nop_flush_icache_all
SYM_TYPED_FUNC_START(nop_flush_user_cache_range)
ret lr
SYM_FUNC_END(nop_flush_user_cache_range)

.globl nop_coherent_kern_range
.equ nop_coherent_kern_range, nop_flush_icache_all
SYM_TYPED_FUNC_START(nop_coherent_kern_range)
ret lr
SYM_FUNC_END(nop_coherent_kern_range)

ENTRY(nop_coherent_user_range)
SYM_TYPED_FUNC_START(nop_coherent_user_range)
mov r0, 0
ret lr
ENDPROC(nop_coherent_user_range)

.globl nop_flush_kern_dcache_area
.equ nop_flush_kern_dcache_area, nop_flush_icache_all
SYM_FUNC_END(nop_coherent_user_range)

.globl nop_dma_flush_range
.equ nop_dma_flush_range, nop_flush_icache_all
SYM_TYPED_FUNC_START(nop_flush_kern_dcache_area)
ret lr
SYM_FUNC_END(nop_flush_kern_dcache_area)

.globl nop_dma_map_area
.equ nop_dma_map_area, nop_flush_icache_all
SYM_TYPED_FUNC_START(nop_dma_flush_range)
ret lr
SYM_FUNC_END(nop_dma_flush_range)

.globl nop_dma_unmap_area
.equ nop_dma_unmap_area, nop_flush_icache_all
SYM_TYPED_FUNC_START(nop_dma_map_area)
ret lr
SYM_FUNC_END(nop_dma_map_area)

__INITDATA

@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
define_cache_functions nop

SYM_TYPED_FUNC_START(nop_dma_unmap_area)
ret lr
SYM_FUNC_END(nop_dma_unmap_area)
47 changes: 28 additions & 19 deletions arch/arm/mm/cache-v4.S
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include <asm/page.h>
#include "proc-macros.S"
Expand All @@ -15,9 +16,9 @@
*
* Unconditionally clean and invalidate the entire icache.
*/
ENTRY(v4_flush_icache_all)
SYM_TYPED_FUNC_START(v4_flush_icache_all)
ret lr
ENDPROC(v4_flush_icache_all)
SYM_FUNC_END(v4_flush_icache_all)

/*
* flush_user_cache_all()
Expand All @@ -27,21 +28,24 @@ ENDPROC(v4_flush_icache_all)
*
* - mm - mm_struct describing address space
*/
ENTRY(v4_flush_user_cache_all)
/* FALLTHROUGH */
SYM_TYPED_FUNC_START(v4_flush_user_cache_all)
b v4_flush_kern_cache_all
SYM_FUNC_END(v4_flush_user_cache_all)

/*
* flush_kern_cache_all()
*
* Clean and invalidate the entire cache.
*/
ENTRY(v4_flush_kern_cache_all)
SYM_TYPED_FUNC_START(v4_flush_kern_cache_all)
#ifdef CONFIG_CPU_CP15
mov r0, #0
mcr p15, 0, r0, c7, c7, 0 @ flush ID cache
ret lr
#else
/* FALLTHROUGH */
ret lr
#endif
SYM_FUNC_END(v4_flush_kern_cache_all)

/*
* flush_user_cache_range(start, end, flags)
Expand All @@ -53,14 +57,15 @@ ENTRY(v4_flush_kern_cache_all)
* - end - end address (exclusive, may not be aligned)
* - flags - vma_area_struct flags describing address space
*/
ENTRY(v4_flush_user_cache_range)
SYM_TYPED_FUNC_START(v4_flush_user_cache_range)
#ifdef CONFIG_CPU_CP15
mov ip, #0
mcr p15, 0, ip, c7, c7, 0 @ flush ID cache
ret lr
#else
/* FALLTHROUGH */
ret lr
#endif
SYM_FUNC_END(v4_flush_user_cache_range)

/*
* coherent_kern_range(start, end)
Expand All @@ -72,8 +77,9 @@ ENTRY(v4_flush_user_cache_range)
* - start - virtual start address
* - end - virtual end address
*/
ENTRY(v4_coherent_kern_range)
/* FALLTHROUGH */
SYM_TYPED_FUNC_START(v4_coherent_kern_range)
ret lr
SYM_FUNC_END(v4_coherent_kern_range)

/*
* coherent_user_range(start, end)
Expand All @@ -85,9 +91,10 @@ ENTRY(v4_coherent_kern_range)
* - start - virtual start address
* - end - virtual end address
*/
ENTRY(v4_coherent_user_range)
SYM_TYPED_FUNC_START(v4_coherent_user_range)
mov r0, #0
ret lr
SYM_FUNC_END(v4_coherent_user_range)

/*
* flush_kern_dcache_area(void *addr, size_t size)
Expand All @@ -98,8 +105,9 @@ ENTRY(v4_coherent_user_range)
* - addr - kernel address
* - size - region size
*/
ENTRY(v4_flush_kern_dcache_area)
/* FALLTHROUGH */
SYM_TYPED_FUNC_START(v4_flush_kern_dcache_area)
b v4_dma_flush_range
SYM_FUNC_END(v4_flush_kern_dcache_area)

/*
* dma_flush_range(start, end)
Expand All @@ -109,34 +117,35 @@ ENTRY(v4_flush_kern_dcache_area)
* - start - virtual start address
* - end - virtual end address
*/
ENTRY(v4_dma_flush_range)
SYM_TYPED_FUNC_START(v4_dma_flush_range)
#ifdef CONFIG_CPU_CP15
mov r0, #0
mcr p15, 0, r0, c7, c7, 0 @ flush ID cache
#endif
ret lr
SYM_FUNC_END(v4_dma_flush_range)

/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4_dma_unmap_area)
SYM_TYPED_FUNC_START(v4_dma_unmap_area)
teq r2, #DMA_TO_DEVICE
bne v4_dma_flush_range
/* FALLTHROUGH */
ret lr
SYM_FUNC_END(v4_dma_unmap_area)

/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4_dma_map_area)
SYM_TYPED_FUNC_START(v4_dma_map_area)
ret lr
ENDPROC(v4_dma_unmap_area)
ENDPROC(v4_dma_map_area)
SYM_FUNC_END(v4_dma_map_area)

.globl v4_flush_kern_cache_louis
.equ v4_flush_kern_cache_louis, v4_flush_kern_cache_all
Expand Down
Loading

0 comments on commit 1036b89

Please sign in to comment.