From 6216182fb776ae546c5566f21976d116c8650cee Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Thu, 24 Oct 2024 11:19:40 +0100 Subject: [PATCH 01/65] RISC-V: clarify what some RISCV_ISA* config options do During some discussion on IRC yesterday and on Pu's bpf patch [1] I noticed that these RISCV_ISA* Kconfig options are not really clear about their implications. Many of these options have no impact on what userspace is allowed to do, for example an application can use Zbb regardless of whether or not the kernel does. Change the help text to try and clarify whether or not an option affects just the kernel, or also userspace. None of these options actually control whether or not an extension is detected dynamically as that's done regardless of Kconfig options, so drop any text that implies the option is required for dynamic detection, rewording them as "do x when y is detected". Link: https://lore.kernel.org/linux-riscv/20240328-ferocity-repose-c554f75a676c@spud/ [1] Reviewed-by: Andrew Jones Signed-off-by: Conor Dooley Reviewed-by: Charlie Jenkins Reviewed-by: Samuel Holland Link: https://lore.kernel.org/r/20241024-overdue-slogan-0b0f69d3da91@spud Signed-off-by: Alexandre Ghiti --- arch/riscv/Kconfig | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 62545946ecf43..278a38c94c5a6 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -527,7 +527,8 @@ config RISCV_ISA_C help Adds "C" to the ISA subsets that the toolchain is allowed to emit when building Linux, which results in compressed instructions in the - Linux binary. + Linux binary. This option produces a kernel that will not run on + systems that do not support compressed instructions. If you don't know what to do here, say Y. @@ -537,8 +538,8 @@ config RISCV_ISA_SVNAPOT depends on RISCV_ALTERNATIVE default y help - Allow kernel to detect the Svnapot ISA-extension dynamically at boot - time and enable its usage. + Enable support for the Svnapot ISA-extension when it is detected + at boot. The Svnapot extension is used to mark contiguous PTEs as a range of contiguous virtual-to-physical translations for a naturally @@ -556,9 +557,8 @@ config RISCV_ISA_SVPBMT depends on RISCV_ALTERNATIVE default y help - Adds support to dynamically detect the presence of the Svpbmt - ISA-extension (Supervisor-mode: page-based memory types) and - enable its usage. + Add support for the Svpbmt ISA-extension (Supervisor-mode: + page-based memory types) in the kernel when it is detected at boot. The memory type for a page contains a combination of attributes that indicate the cacheability, idempotency, and ordering @@ -577,14 +577,15 @@ config TOOLCHAIN_HAS_V depends on AS_HAS_OPTION_ARCH config RISCV_ISA_V - bool "VECTOR extension support" + bool "Vector extension support" depends on TOOLCHAIN_HAS_V depends on FPU select DYNAMIC_SIGFRAME default y help - Say N here if you want to disable all vector related procedure - in the kernel. + Add support for the Vector extension when it is detected at boot. + When this option is disabled, neither the kernel nor userspace may + use vector procedures. If you don't know what to do here, say Y. @@ -667,8 +668,8 @@ config RISCV_ISA_ZBB depends on RISCV_ALTERNATIVE default y help - Adds support to dynamically detect the presence of the ZBB - extension (basic bit manipulation) and enable its usage. + Add support for enabling optimisations in the kernel when the + Zbb extension is detected at boot. The Zbb extension provides instructions to accelerate a number of bit-specific operations (count bit population, sign extending, @@ -707,9 +708,9 @@ config RISCV_ISA_ZICBOM select RISCV_DMA_NONCOHERENT select DMA_DIRECT_REMAP help - Adds support to dynamically detect the presence of the ZICBOM - extension (Cache Block Management Operations) and enable its - usage. + Add support for the Zicbom extension (Cache Block Management + Operations) and enable its use in the kernel when it is detected + at boot. The Zicbom extension can be used to handle for example non-coherent DMA support on devices that need it. @@ -722,7 +723,7 @@ config RISCV_ISA_ZICBOZ default y help Enable the use of the Zicboz extension (cbo.zero instruction) - when available. + in the kernel when it is detected at boot. The Zicboz extension is used for faster zeroing of memory. @@ -760,8 +761,9 @@ config FPU bool "FPU support" default y help - Say N here if you want to disable all floating-point related procedure - in the kernel. + Add support for floating point operations when an FPU is detected at + boot. When this option is disabled, neither the kernel nor userspace + may use the floating point unit. If you don't know what to do here, say Y. From 9343aaba1f256ff42730db5a61efc32a86149776 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Thu, 24 Oct 2024 11:19:41 +0100 Subject: [PATCH 02/65] RISC-V: separate Zbb optimisations requiring and not requiring toolchain support It seems a bit ridiculous to require toolchain support for BPF to assemble Zbb instructions, so move the dependency on toolchain support for Zbb optimisations out of the Kconfig option and to the callsites. Zbb support has always depended on alternatives, so while adjusting the config options guarding optimisations, remove any checks for whether or not alternatives are enabled. Reviewed-by: Andrew Jones Signed-off-by: Conor Dooley Reviewed-by: Charlie Jenkins Reviewed-by: Samuel Holland Link: https://lore.kernel.org/r/20241024-chump-freebase-d26b6d81af33@spud Signed-off-by: Alexandre Ghiti --- arch/riscv/Kconfig | 4 ++-- arch/riscv/include/asm/arch_hweight.h | 6 +++--- arch/riscv/include/asm/bitops.h | 4 ++-- arch/riscv/include/asm/checksum.h | 3 +-- arch/riscv/lib/csum.c | 21 +++------------------ arch/riscv/lib/strcmp.S | 5 +++-- arch/riscv/lib/strlen.S | 5 +++-- arch/riscv/lib/strncmp.S | 5 +++-- 8 files changed, 20 insertions(+), 33 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 278a38c94c5a6..6ec7a500a25ff 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -664,12 +664,12 @@ config RISCV_ISA_ZBA config RISCV_ISA_ZBB bool "Zbb extension support for bit manipulation instructions" - depends on TOOLCHAIN_HAS_ZBB depends on RISCV_ALTERNATIVE default y help Add support for enabling optimisations in the kernel when the - Zbb extension is detected at boot. + Zbb extension is detected at boot. Some optimisations may + additionally depend on toolchain support for Zbb. The Zbb extension provides instructions to accelerate a number of bit-specific operations (count bit population, sign extending, diff --git a/arch/riscv/include/asm/arch_hweight.h b/arch/riscv/include/asm/arch_hweight.h index 613769b9cdc90..0e7cdbbec8efd 100644 --- a/arch/riscv/include/asm/arch_hweight.h +++ b/arch/riscv/include/asm/arch_hweight.h @@ -19,7 +19,7 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) { -#ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); @@ -50,7 +50,7 @@ static inline unsigned int __arch_hweight8(unsigned int w) #if BITS_PER_LONG == 64 static __always_inline unsigned long __arch_hweight64(__u64 w) { -# ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); @@ -64,7 +64,7 @@ static __always_inline unsigned long __arch_hweight64(__u64 w) return w; legacy: -# endif +#endif return __sw_hweight64(w); } #else /* BITS_PER_LONG == 64 */ diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index fae152ea0508d..410e2235d6589 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -15,7 +15,7 @@ #include #include -#if !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) +#if !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) #include #include #include @@ -175,7 +175,7 @@ static __always_inline int variable_fls(unsigned int x) variable_fls(x_); \ }) -#endif /* !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) */ +#endif /* !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) */ #include #include diff --git a/arch/riscv/include/asm/checksum.h b/arch/riscv/include/asm/checksum.h index 88e6f1499e889..da378856f1d59 100644 --- a/arch/riscv/include/asm/checksum.h +++ b/arch/riscv/include/asm/checksum.h @@ -49,8 +49,7 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) * ZBB only saves three instructions on 32-bit and five on 64-bit so not * worth checking if supported without Alternatives. */ - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { unsigned long fold_temp; asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, diff --git a/arch/riscv/lib/csum.c b/arch/riscv/lib/csum.c index 7fb12c59e5719..9408f50ca59a8 100644 --- a/arch/riscv/lib/csum.c +++ b/arch/riscv/lib/csum.c @@ -40,12 +40,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, uproto = (__force unsigned int)htonl(proto); sum += uproto; - /* - * Zbb support saves 4 instructions, so not worth checking without - * alternatives if supported - */ - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { unsigned long fold_temp; /* @@ -157,12 +152,7 @@ do_csum_with_alignment(const unsigned char *buff, int len) csum = do_csum_common(ptr, end, data); #ifdef CC_HAS_ASM_GOTO_TIED_OUTPUT - /* - * Zbb support saves 6 instructions, so not worth checking without - * alternatives if supported - */ - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { unsigned long fold_temp; /* @@ -244,12 +234,7 @@ do_csum_no_alignment(const unsigned char *buff, int len) end = (const unsigned long *)(buff + len); csum = do_csum_common(ptr, end, data); - /* - * Zbb support saves 6 instructions, so not worth checking without - * alternatives if supported - */ - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { unsigned long fold_temp; /* diff --git a/arch/riscv/lib/strcmp.S b/arch/riscv/lib/strcmp.S index 57a5c00662315..65027e742af15 100644 --- a/arch/riscv/lib/strcmp.S +++ b/arch/riscv/lib/strcmp.S @@ -8,7 +8,8 @@ /* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) - ALTERNATIVE("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) + __ALTERNATIVE_CFG("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) /* * Returns @@ -43,7 +44,7 @@ SYM_FUNC_START(strcmp) * The code was published as part of the bitmanip manual * in Appendix A. */ -#ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) strcmp_zbb: .option push diff --git a/arch/riscv/lib/strlen.S b/arch/riscv/lib/strlen.S index 962983b73251e..eb4d2b7ed22b6 100644 --- a/arch/riscv/lib/strlen.S +++ b/arch/riscv/lib/strlen.S @@ -8,7 +8,8 @@ /* int strlen(const char *s) */ SYM_FUNC_START(strlen) - ALTERNATIVE("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) + __ALTERNATIVE_CFG("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) /* * Returns @@ -33,7 +34,7 @@ SYM_FUNC_START(strlen) /* * Variant of strlen using the ZBB extension if available */ -#ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) strlen_zbb: #ifdef CONFIG_CPU_BIG_ENDIAN diff --git a/arch/riscv/lib/strncmp.S b/arch/riscv/lib/strncmp.S index 7b2d0ff9ed6c7..062000c468c83 100644 --- a/arch/riscv/lib/strncmp.S +++ b/arch/riscv/lib/strncmp.S @@ -8,7 +8,8 @@ /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) - ALTERNATIVE("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) + __ALTERNATIVE_CFG("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) /* * Returns @@ -46,7 +47,7 @@ SYM_FUNC_START(strncmp) /* * Variant of strncmp using the ZBB extension if available */ -#ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) strncmp_zbb: .option push From 03dc00a2b67854604c0aa8cbd32105f8b633451e Mon Sep 17 00:00:00 2001 From: Andrew Bresticker Date: Wed, 8 Jan 2025 05:57:00 -0800 Subject: [PATCH 03/65] riscv: Support huge pfnmaps Use RSW0 as the special bit for pmds and puds, just like for ptes. Also define the {pte,pmd,pud}_pgprot helpers which were previously missing and are needed for the follow_pfnmap APIs. Signed-off-by: Andrew Bresticker Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250108135700.2614848-1-abrestic@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/pgtable.h | 49 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 8b86ce1965dc8..ff35eb16853cd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -62,6 +62,7 @@ config RISCV select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_CFI_CLANG select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU + select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_HUGETLBFS if MMU # LLD >= 14: https://github.com/llvm/llvm-project/issues/50505 select ARCH_SUPPORTS_LTO_CLANG if LLD_VERSION >= 140000 diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 050fdc49b5ad7..b6697dc21dafd 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -341,6 +341,14 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) #define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) +#define pte_pgprot pte_pgprot +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pfn = pte_pfn(pte); + + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} + static inline int pte_present(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); @@ -674,6 +682,11 @@ static inline pmd_t pte_pmd(pte_t pte) return __pmd(pte_val(pte)); } +static inline pud_t pte_pud(pte_t pte) +{ + return __pud(pte_val(pte)); +} + static inline pmd_t pmd_mkhuge(pmd_t pmd) { return pmd; @@ -699,6 +712,18 @@ static inline unsigned long pud_pfn(pud_t pud) return ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT); } +#define pmd_pgprot pmd_pgprot +static inline pgprot_t pmd_pgprot(pmd_t pmd) +{ + return pte_pgprot(pmd_pte(pmd)); +} + +#define pud_pgprot pud_pgprot +static inline pgprot_t pud_pgprot(pud_t pud) +{ + return pte_pgprot(pud_pte(pud)); +} + static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); @@ -768,6 +793,30 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd) return pte_pmd(pte_mkdevmap(pmd_pte(pmd))); } +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP +static inline bool pmd_special(pmd_t pmd) +{ + return pte_special(pmd_pte(pmd)); +} + +static inline pmd_t pmd_mkspecial(pmd_t pmd) +{ + return pte_pmd(pte_mkspecial(pmd_pte(pmd))); +} +#endif + +#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP +static inline bool pud_special(pud_t pud) +{ + return pte_special(pud_pte(pud)); +} + +static inline pud_t pud_mkspecial(pud_t pud) +{ + return pte_pud(pte_mkspecial(pud_pte(pud))); +} +#endif + static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { From d3817d091fe6480de5bf3faba0fc2ce25f8d023e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 30 Aug 2024 10:49:32 +0200 Subject: [PATCH 04/65] riscv: remove useless pc check in stacktrace handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking for pc to be a kernel text address at this location is useless since pc == handle_exception. Remove this check. [ alex: Fix merge conflict ] Signed-off-by: Clément Léger Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240830084934.3690037-1-cleger@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/stacktrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index d4355c770c36a..3fe9e6edef8f1 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -74,7 +74,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, &frame->ra); if (pc >= (unsigned long)handle_exception && pc < (unsigned long)&ret_from_exception_end) { - if (unlikely(!__kernel_text_address(pc) || !fn(arg, pc))) + if (unlikely(!fn(arg, pc))) break; pc = ((struct pt_regs *)sp)->epc; From 4458b8f68dc7ab8309291f1667157d0250938291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miquel=20Sabat=C3=A9=20Sol=C3=A0?= Date: Fri, 13 Sep 2024 07:13:24 +0200 Subject: [PATCH 05/65] riscv: hwprobe: export Zicntr and Zihpm extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zicntr and Zihpm ISA extensions through the hwprobe syscall. [ alex: Fix hwprobe numbering ] Signed-off-by: Miquel Sabaté Solà Acked-by: Jesse Taube Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20240913051324.8176-1-mikisabate@gmail.com Signed-off-by: Alexandre Ghiti --- Documentation/arch/riscv/hwprobe.rst | 6 ++++++ arch/riscv/include/uapi/asm/hwprobe.h | 2 ++ arch/riscv/kernel/sys_hwprobe.c | 2 ++ 3 files changed, 10 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index f273ea15a8e83..35a979dd164a6 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -183,6 +183,9 @@ The following keys are defined: defined in the Atomic Compare-and-Swap (CAS) instructions manual starting from commit 5059e0ca641c ("update to ratified"). + * :c:macro:`RISCV_HWPROBE_EXT_ZICNTR`: The Zicntr extension version 2.0 + is supported as defined in the RISC-V ISA manual. + * :c:macro:`RISCV_HWPROBE_EXT_ZICOND`: The Zicond extension is supported as defined in the RISC-V Integer Conditional (Zicond) operations extension manual starting from commit 95cf1f9 ("Add changes requested by Ved @@ -192,6 +195,9 @@ The following keys are defined: supported as defined in the RISC-V ISA manual starting from commit d8ab5c78c207 ("Zihintpause is ratified"). + * :c:macro:`RISCV_HWPROBE_EXT_ZIHPM`: The Zihpm extension version 2.0 + is supported as defined in the RISC-V ISA manual. + * :c:macro:`RISCV_HWPROBE_EXT_ZVE32X`: The Vector sub-extension Zve32x is supported, as defined by version 1.0 of the RISC-V Vector extension manual. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index c3c1cc951cb94..8cac35cb19d8b 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -73,6 +73,8 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47) #define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48) #define RISCV_HWPROBE_EXT_SUPM (1ULL << 49) +#define RISCV_HWPROBE_EXT_ZICNTR (1ULL << 50) +#define RISCV_HWPROBE_EXT_ZIHPM (1ULL << 51) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index bcd3b816306c2..b35cce0b57aec 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -108,9 +108,11 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZCB); EXT_KEY(ZCMOP); EXT_KEY(ZICBOZ); + EXT_KEY(ZICNTR); EXT_KEY(ZICOND); EXT_KEY(ZIHINTNTL); EXT_KEY(ZIHINTPAUSE); + EXT_KEY(ZIHPM); EXT_KEY(ZIMOP); EXT_KEY(ZKND); EXT_KEY(ZKNE); From d9be2b9b60497a82aeceec3a98d8b37fdd2960f2 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Mon, 13 Jan 2025 15:24:24 +0100 Subject: [PATCH 06/65] riscv: Call secondary mmu notifier when flushing the tlb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is required to allow the IOMMU driver to correctly flush its own TLB. Reviewed-by: Clément Léger Reviewed-by: Samuel Holland Link: https://lore.kernel.org/r/20250113142424.30487-1-alexghiti@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/mm/tlbflush.c | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 9b6e86ce38674..bb77607c87aa2 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -78,10 +79,17 @@ static void __ipi_flush_tlb_range_asid(void *info) local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); } -static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid, +static inline unsigned long get_mm_asid(struct mm_struct *mm) +{ + return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; +} + +static void __flush_tlb_range(struct mm_struct *mm, + const struct cpumask *cmask, unsigned long start, unsigned long size, unsigned long stride) { + unsigned long asid = get_mm_asid(mm); unsigned int cpu; if (cpumask_empty(cmask)) @@ -105,30 +113,26 @@ static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid, } put_cpu(); -} -static inline unsigned long get_mm_asid(struct mm_struct *mm) -{ - return cntx2asid(atomic_long_read(&mm->context.id)); + if (mm) + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + size); } void flush_tlb_mm(struct mm_struct *mm) { - __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm), - 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); + __flush_tlb_range(mm, mm_cpumask(mm), 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); } void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int page_size) { - __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm), - start, end - start, page_size); + __flush_tlb_range(mm, mm_cpumask(mm), start, end - start, page_size); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { - __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), + __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), addr, PAGE_SIZE, PAGE_SIZE); } @@ -161,13 +165,13 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, } } - __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), + __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), start, end - start, stride_size); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - __flush_tlb_range(cpu_online_mask, FLUSH_TLB_NO_ASID, + __flush_tlb_range(NULL, cpu_online_mask, start, end - start, PAGE_SIZE); } @@ -175,7 +179,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), + __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), start, end - start, PMD_SIZE); } #endif @@ -189,7 +193,10 @@ void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, struct mm_struct *mm, unsigned long uaddr) { + unsigned long start = uaddr & PAGE_MASK; + cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + PAGE_SIZE); } void arch_flush_tlb_batched_pending(struct mm_struct *mm) @@ -199,7 +206,7 @@ void arch_flush_tlb_batched_pending(struct mm_struct *mm) void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __flush_tlb_range(&batch->cpumask, FLUSH_TLB_NO_ASID, 0, - FLUSH_TLB_MAX_SIZE, PAGE_SIZE); + __flush_tlb_range(NULL, &batch->cpumask, + 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); cpumask_clear(&batch->cpumask); } From d9708b1931fc0ebb21cd94a56283d09222847749 Mon Sep 17 00:00:00 2001 From: Guo Ren Date: Mon, 16 Dec 2024 20:39:10 -0500 Subject: [PATCH 07/65] riscv: Implement smp_cond_load8/16() with Zawrs RISC-V code uses the queued spinlock implementation, which calls the macros smp_cond_load_acquire for one byte. So, complement the implementation of byte and halfword versions. Signed-off-by: Guo Ren Signed-off-by: Guo Ren Cc: Andrew Jones Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20241217013910.1039923-1-guoren@kernel.org Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/cmpxchg.h | 38 +++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 427c41dde6431..2ec119eb147bb 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -365,16 +365,48 @@ static __always_inline void __cmpwait(volatile void *ptr, { unsigned long tmp; + u32 *__ptr32b; + ulong __s, __val, __mask; + asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", 0, RISCV_ISA_EXT_ZAWRS, 1) : : : : no_zawrs); switch (size) { case 1: - fallthrough; + __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); + __s = ((ulong)(ptr) & 0x3) * BITS_PER_BYTE; + __val = val << __s; + __mask = 0xff << __s; + + asm volatile( + " lr.w %0, %1\n" + " and %0, %0, %3\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(__ptr32b)) + : "r" (__val), "r" (__mask) + : "memory"); + break; case 2: - /* RISC-V doesn't have lr instructions on byte and half-word. */ - goto no_zawrs; + __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); + __s = ((ulong)(ptr) & 0x2) * BITS_PER_BYTE; + __val = val << __s; + __mask = 0xffff << __s; + + asm volatile( + " lr.w %0, %1\n" + " and %0, %0, %3\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(__ptr32b)) + : "r" (__val), "r" (__mask) + : "memory"); + break; case 4: asm volatile( " lr.w %0, %1\n" From 35bc1883733c81ff307c9c73bd00191313e651af Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Thu, 13 Feb 2025 08:38:45 +0800 Subject: [PATCH 08/65] dt-bindings: riscv: add bfloat16 ISA extension description Add description for the BFloat16 precision Floating-Point ISA extension, (Zfbfmin, Zvfbfmin, Zvfbfwma). which was ratified in commit 4dc23d62 ("Added Chapter title to BF16") of the riscv-isa-manual. Signed-off-by: Inochi Amaoto Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20250213003849.147358-2-inochiama@gmail.com Signed-off-by: Alexandre Ghiti --- .../devicetree/bindings/riscv/extensions.yaml | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index 9c7dd7e75e0ca..0a1f1a76d1296 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -329,6 +329,12 @@ properties: instructions, as ratified in commit 056b6ff ("Zfa is ratified") of riscv-isa-manual. + - const: zfbfmin + description: + The standard Zfbfmin extension which provides minimal support for + 16-bit half-precision brain floating-point instructions, as ratified + in commit 4dc23d62 ("Added Chapter title to BF16") of riscv-isa-manual. + - const: zfh description: The standard Zfh extension for 16-bit half-precision binary @@ -525,6 +531,18 @@ properties: in commit 6f702a2 ("Vector extensions are now ratified") of riscv-v-spec. + - const: zvfbfmin + description: + The standard Zvfbfmin extension for minimal support for vectored + 16-bit half-precision brain floating-point instructions, as ratified + in commit 4dc23d62 ("Added Chapter title to BF16") of riscv-isa-manual. + + - const: zvfbfwma + description: + The standard Zvfbfwma extension for vectored half-precision brain + floating-point widening multiply-accumulate instructions, as ratified + in commit 4dc23d62 ("Added Chapter title to BF16") of riscv-isa-manual. + - const: zvfh description: The standard Zvfh extension for vectored half-precision @@ -663,6 +681,33 @@ properties: then: contains: const: zca + # Zfbfmin depends on F + - if: + contains: + const: zfbfmin + then: + contains: + const: f + # Zvfbfmin depends on V or Zve32f + - if: + contains: + const: zvfbfmin + then: + oneOf: + - contains: + const: v + - contains: + const: zve32f + # Zvfbfwma depends on Zfbfmin and Zvfbfmin + - if: + contains: + const: zvfbfwma + then: + allOf: + - contains: + const: zfbfmin + - contains: + const: zvfbfmin allOf: # Zcf extension does not exist on rv64 From e186c28dda11e8d6d829b08b9da2ec7c342edd78 Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Thu, 13 Feb 2025 08:38:46 +0800 Subject: [PATCH 09/65] riscv: add ISA extension parsing for bfloat16 ISA extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parsing for Zfbmin, Zvfbfmin, Zvfbfwma ISA extension which were ratified in 4dc23d62 ("Added Chapter title to BF16") of the riscv-isa-manual. Signed-off-by: Inochi Amaoto Reviewed-by: Clément Léger Link: https://lore.kernel.org/r/20250213003849.147358-3-inochiama@gmail.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/hwcap.h | 3 +++ arch/riscv/kernel/cpufeature.c | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 869da082252a4..14cc29f2a723c 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -100,6 +100,9 @@ #define RISCV_ISA_EXT_ZICCRSE 91 #define RISCV_ISA_EXT_SVADE 92 #define RISCV_ISA_EXT_SVADU 93 +#define RISCV_ISA_EXT_ZFBFMIN 94 +#define RISCV_ISA_EXT_ZVFBFMIN 95 +#define RISCV_ISA_EXT_ZVFBFWMA 96 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index c0916ed318c20..40021459a2574 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -76,6 +76,15 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i } EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); +static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + return 0; + + return -EPROBE_DEFER; +} + static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -136,6 +145,28 @@ static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, return -EPROBE_DEFER; } +static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F)) + return 0; + + return -EPROBE_DEFER; +} + +static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN)) + return 0; + + return -EPROBE_DEFER; +} + static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -341,6 +372,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), + __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), @@ -373,6 +405,9 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_SUPERSET(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts), __RISCV_ISA_EXT_SUPERSET(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts), __RISCV_ISA_EXT_SUPERSET(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts), + __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, + riscv_ext_zvfbfwma_validate), __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), From a4863e002cf0dd6fb2f06796f16d7bc0974e9845 Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Thu, 13 Feb 2025 08:38:47 +0800 Subject: [PATCH 10/65] riscv: hwprobe: export bfloat16 ISA extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zfbmin, Zvfbfmin, Zvfbfwma ISA extension through hwprobe. Signed-off-by: Inochi Amaoto Reviewed-by: Clément Léger Link: https://lore.kernel.org/r/20250213003849.147358-4-inochiama@gmail.com Signed-off-by: Alexandre Ghiti --- Documentation/arch/riscv/hwprobe.rst | 12 ++++++++++++ arch/riscv/include/uapi/asm/hwprobe.h | 3 +++ arch/riscv/kernel/sys_hwprobe.c | 3 +++ 3 files changed, 18 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 955fbcd19ce90..a9cb40e407a48 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -242,6 +242,18 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_SUPM`: The Supm extension is supported as defined in version 1.0 of the RISC-V Pointer Masking extensions. + * :c:macro:`RISCV_HWPROBE_EXT_ZFBFMIN`: The Zfbfmin extension is supported as + defined in the RISC-V ISA manual starting from commit 4dc23d6229de + ("Added Chapter title to BF16"). + + * :c:macro:`RISCV_HWPROBE_EXT_ZVFBFMIN`: The Zvfbfmin extension is supported as + defined in the RISC-V ISA manual starting from commit 4dc23d6229de + ("Added Chapter title to BF16"). + + * :c:macro:`RISCV_HWPROBE_EXT_ZVFBFWMA`: The Zvfbfwma extension is supported as + defined in the RISC-V ISA manual starting from commit 4dc23d6229de + ("Added Chapter title to BF16"). + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: Deprecated. Returns similar values to :c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF`, but the key was mistakenly classified as a bitmask rather than a value. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 3af142b99f778..aecc1c800d54b 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -73,6 +73,9 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47) #define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48) #define RISCV_HWPROBE_EXT_SUPM (1ULL << 49) +#define RISCV_HWPROBE_EXT_ZFBFMIN (1ULL << 50) +#define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 51) +#define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 52) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index cb93adfffc486..bd215f58bd1b7 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -131,6 +131,8 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZVE64D); EXT_KEY(ZVE64F); EXT_KEY(ZVE64X); + EXT_KEY(ZVFBFMIN); + EXT_KEY(ZVFBFWMA); EXT_KEY(ZVFH); EXT_KEY(ZVFHMIN); EXT_KEY(ZVKB); @@ -147,6 +149,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZCD); EXT_KEY(ZCF); EXT_KEY(ZFA); + EXT_KEY(ZFBFMIN); EXT_KEY(ZFH); EXT_KEY(ZFHMIN); } From de70b532f91bbcfa9f4100f1a2cd62810c799239 Mon Sep 17 00:00:00 2001 From: Yunhui Cui Date: Wed, 26 Feb 2025 14:32:04 +0800 Subject: [PATCH 11/65] RISC-V: Enable cbo.clean/flush in usermode Enabling cbo.clean and cbo.flush in user mode makes it more convenient to manage the cache state and achieve better performance. Reviewed-by: Andrew Jones Signed-off-by: Yunhui Cui Link: https://lore.kernel.org/r/20250226063206.71216-2-cuiyunhui@bytedance.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/cpufeature.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 0ff97cf4dc2ab..ef34622902bf9 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -32,6 +32,7 @@ #define NUM_ALPHA_EXTS ('z' - 'a' + 1) static bool any_cpu_has_zicboz; +static bool any_cpu_has_zicbom; unsigned long elf_hwcap __read_mostly; @@ -100,6 +101,8 @@ static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); return -EINVAL; } + + any_cpu_has_zicbom = true; return 0; } @@ -1036,6 +1039,11 @@ void __init riscv_user_isa_enable(void) current->thread.envcfg |= ENVCFG_CBZE; else if (any_cpu_has_zicboz) pr_warn("Zicboz disabled as it is unavailable on some harts\n"); + + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM)) + current->thread.envcfg |= ENVCFG_CBCFE; + else if (any_cpu_has_zicbom) + pr_warn("Zicbom disabled as it is unavailable on some harts\n"); } #ifdef CONFIG_RISCV_ALTERNATIVE From eb10039709402cc1fab1533a1ecc1a54c2152e68 Mon Sep 17 00:00:00 2001 From: Yunhui Cui Date: Wed, 26 Feb 2025 14:32:05 +0800 Subject: [PATCH 12/65] RISC-V: hwprobe: Expose Zicbom extension and its block size Expose Zicbom through hwprobe and also provide a key to extract its respective block size. [ alex: Fix merge conflicts and hwprobe numbering ] Reviewed-by: Andrew Jones Reviewed-by: Samuel Holland Signed-off-by: Yunhui Cui Link: https://lore.kernel.org/r/20250226063206.71216-3-cuiyunhui@bytedance.com Signed-off-by: Alexandre Ghiti --- Documentation/arch/riscv/hwprobe.rst | 6 ++++++ arch/riscv/include/asm/hwprobe.h | 2 +- arch/riscv/include/uapi/asm/hwprobe.h | 2 ++ arch/riscv/kernel/sys_hwprobe.c | 8 +++++++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 67ef406cdb675..2792f12e90baa 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -260,6 +260,9 @@ The following keys are defined: defined in the RISC-V ISA manual starting from commit 4dc23d6229de ("Added Chapter title to BF16"). + * :c:macro:`RISCV_HWPROBE_EXT_ZICBOM`: The Zicbom extension is supported, as + ratified in commit 3dd606f ("Create cmobase-v1.0.pdf") of riscv-CMOs. + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: Deprecated. Returns similar values to :c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF`, but the key was mistakenly classified as a bitmask rather than a value. @@ -321,3 +324,6 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_VENDOR_EXT_XTHEADVECTOR`: The xtheadvector vendor extension is supported in the T-Head ISA extensions spec starting from commit a18c801634 ("Add T-Head VECTOR vendor extension. "). + +* :c:macro:`RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE`: An unsigned int which + represents the size of the Zicbom block in bytes. diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index dd624523981c8..1f690fea0e03d 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -8,7 +8,7 @@ #include -#define RISCV_HWPROBE_MAX_KEY 11 +#define RISCV_HWPROBE_MAX_KEY 12 static inline bool riscv_hwprobe_key_is_valid(__s64 key) { diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 08ab52fe8004e..056decf65b1b7 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -78,6 +78,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZFBFMIN (1ULL << 52) #define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 53) #define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 54) +#define RISCV_HWPROBE_EXT_ZICBOM (1ULL << 55) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) @@ -100,6 +101,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_MISALIGNED_VECTOR_FAST 3 #define RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED 4 #define RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0 11 +#define RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE 12 /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ /* Flags */ diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index 32509111fe0ac..cfe6ceaf30693 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -107,6 +107,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZCA); EXT_KEY(ZCB); EXT_KEY(ZCMOP); + EXT_KEY(ZICBOM); EXT_KEY(ZICBOZ); EXT_KEY(ZICNTR); EXT_KEY(ZICOND); @@ -166,7 +167,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, pair->value &= ~missing; } -static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext) +static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext) { struct riscv_hwprobe pair; @@ -284,6 +285,11 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) pair->value = riscv_cboz_block_size; break; + case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: + pair->value = 0; + if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) + pair->value = riscv_cbom_block_size; + break; case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: pair->value = user_max_virt_addr(); break; From 36dec9e44805f80d32277be76d16c88373bdc20d Mon Sep 17 00:00:00 2001 From: Yunhui Cui Date: Wed, 26 Feb 2025 14:32:06 +0800 Subject: [PATCH 13/65] RISC-V: selftests: Add TEST_ZICBOM into CBO tests Add test for Zicbom and its block size into CBO tests, when Zicbom is present, test that cbo.clean/flush may be issued and works. As the software can't verify the clean/flush functions, we just judged that cbo.clean/flush isn't executed illegally. Reviewed-by: Andrew Jones Reviewed-by: Samuel Holland Signed-off-by: Yunhui Cui Link: https://lore.kernel.org/r/20250226063206.71216-4-cuiyunhui@bytedance.com Signed-off-by: Alexandre Ghiti --- tools/testing/selftests/riscv/hwprobe/cbo.c | 66 +++++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/riscv/hwprobe/cbo.c b/tools/testing/selftests/riscv/hwprobe/cbo.c index a40541bb7c7de..5e96ef785d0dd 100644 --- a/tools/testing/selftests/riscv/hwprobe/cbo.c +++ b/tools/testing/selftests/riscv/hwprobe/cbo.c @@ -50,6 +50,14 @@ static void cbo_clean(char *base) { cbo_insn(base, 1); } static void cbo_flush(char *base) { cbo_insn(base, 2); } static void cbo_zero(char *base) { cbo_insn(base, 4); } +static void test_no_cbo_inval(void *arg) +{ + ksft_print_msg("Testing cbo.inval instruction remain privileged\n"); + illegal_insn = false; + cbo_inval(&mem[0]); + ksft_test_result(illegal_insn, "No cbo.inval\n"); +} + static void test_no_zicbom(void *arg) { ksft_print_msg("Testing Zicbom instructions remain privileged\n"); @@ -61,10 +69,6 @@ static void test_no_zicbom(void *arg) illegal_insn = false; cbo_flush(&mem[0]); ksft_test_result(illegal_insn, "No cbo.flush\n"); - - illegal_insn = false; - cbo_inval(&mem[0]); - ksft_test_result(illegal_insn, "No cbo.inval\n"); } static void test_no_zicboz(void *arg) @@ -81,6 +85,30 @@ static bool is_power_of_2(__u64 n) return n != 0 && (n & (n - 1)) == 0; } +static void test_zicbom(void *arg) +{ + struct riscv_hwprobe pair = { + .key = RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE, + }; + cpu_set_t *cpus = (cpu_set_t *)arg; + __u64 block_size; + long rc; + + rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0); + block_size = pair.value; + ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE && + is_power_of_2(block_size), "Zicbom block size\n"); + ksft_print_msg("Zicbom block size: %llu\n", block_size); + + illegal_insn = false; + cbo_clean(&mem[block_size]); + ksft_test_result(!illegal_insn, "cbo.clean\n"); + + illegal_insn = false; + cbo_flush(&mem[block_size]); + ksft_test_result(!illegal_insn, "cbo.flush\n"); +} + static void test_zicboz(void *arg) { struct riscv_hwprobe pair = { @@ -129,7 +157,7 @@ static void test_zicboz(void *arg) ksft_test_result_pass("cbo.zero check\n"); } -static void check_no_zicboz_cpus(cpu_set_t *cpus) +static void check_no_zicbo_cpus(cpu_set_t *cpus, __u64 cbo) { struct riscv_hwprobe pair = { .key = RISCV_HWPROBE_KEY_IMA_EXT_0, @@ -137,6 +165,7 @@ static void check_no_zicboz_cpus(cpu_set_t *cpus) cpu_set_t one_cpu; int i = 0, c = 0; long rc; + char *cbostr; while (i++ < CPU_COUNT(cpus)) { while (!CPU_ISSET(c, cpus)) @@ -148,10 +177,13 @@ static void check_no_zicboz_cpus(cpu_set_t *cpus) rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&one_cpu, 0); assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0); - if (pair.value & RISCV_HWPROBE_EXT_ZICBOZ) - ksft_exit_fail_msg("Zicboz is only present on a subset of harts.\n" - "Use taskset to select a set of harts where Zicboz\n" - "presence (present or not) is consistent for each hart\n"); + cbostr = cbo == RISCV_HWPROBE_EXT_ZICBOZ ? "Zicboz" : "Zicbom"; + + if (pair.value & cbo) + ksft_exit_fail_msg("%s is only present on a subset of harts.\n" + "Use taskset to select a set of harts where %s\n" + "presence (present or not) is consistent for each hart\n", + cbostr, cbostr); ++c; } } @@ -159,7 +191,9 @@ static void check_no_zicboz_cpus(cpu_set_t *cpus) enum { TEST_ZICBOZ, TEST_NO_ZICBOZ, + TEST_ZICBOM, TEST_NO_ZICBOM, + TEST_NO_CBO_INVAL, }; static struct test_info { @@ -169,7 +203,9 @@ static struct test_info { } tests[] = { [TEST_ZICBOZ] = { .nr_tests = 3, test_zicboz }, [TEST_NO_ZICBOZ] = { .nr_tests = 1, test_no_zicboz }, - [TEST_NO_ZICBOM] = { .nr_tests = 3, test_no_zicbom }, + [TEST_ZICBOM] = { .nr_tests = 3, test_zicbom }, + [TEST_NO_ZICBOM] = { .nr_tests = 2, test_no_zicbom }, + [TEST_NO_CBO_INVAL] = { .nr_tests = 1, test_no_cbo_inval }, }; int main(int argc, char **argv) @@ -189,6 +225,7 @@ int main(int argc, char **argv) assert(rc == 0); tests[TEST_NO_ZICBOZ].enabled = true; tests[TEST_NO_ZICBOM].enabled = true; + tests[TEST_NO_CBO_INVAL].enabled = true; } rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus); @@ -206,7 +243,14 @@ int main(int argc, char **argv) tests[TEST_ZICBOZ].enabled = true; tests[TEST_NO_ZICBOZ].enabled = false; } else { - check_no_zicboz_cpus(&cpus); + check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOZ); + } + + if (pair.value & RISCV_HWPROBE_EXT_ZICBOM) { + tests[TEST_ZICBOM].enabled = true; + tests[TEST_NO_ZICBOM].enabled = false; + } else { + check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOM); } for (i = 0; i < ARRAY_SIZE(tests); ++i) From a4a58f510bd8c28f6191eed5698a5291b420c2d6 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Sat, 9 Nov 2024 09:46:05 +0800 Subject: [PATCH 14/65] riscv: Remove unused TASK_TI_FLAGS Since commit f0bddf50586d ("riscv: entry: Convert to generic entry"), TASK_TI_FLAGS is not used any more, so remove it. Fixes: f0bddf50586d ("riscv: entry: Convert to generic entry") Signed-off-by: Jinjie Ruan Reviewed-by: Charlie Jenkins Link: https://lore.kernel.org/r/20241109014605.2801492-1-ruanjinjie@huawei.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/asm-offsets.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index e89455a6a0e50..16490755304e0 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -36,7 +36,6 @@ void asm_offsets(void) OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]); OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu); - OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags); OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp); From 7f238b12660e53d7905b0d9989866b95a32c2467 Mon Sep 17 00:00:00 2001 From: Chin Yik Ming Date: Thu, 30 Jan 2025 04:38:43 +0800 Subject: [PATCH 15/65] riscv: Simplify base extension checks and direct boolean return Reduce three lines checking to single line using a ternary conditional expression for getting the base extension word. In addition, the test_bit macro function already return a boolean which matches the return type of the caller, so directly return the result of the test_bit macro function. Signed-off-by: Chin Yik Ming Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20250129203843.1136838-1-yikming2222@gmail.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/cpufeature.c | 6 ++---- arch/riscv/kernel/vendor_extensions.c | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index ef34622902bf9..6ec9499e2cf27 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -54,9 +54,7 @@ u32 thead_vlenb_of; */ unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) { - if (!isa_bitmap) - return riscv_isa[0]; - return isa_bitmap[0]; + return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0]; } EXPORT_SYMBOL_GPL(riscv_isa_extension_base); @@ -77,7 +75,7 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i if (bit >= RISCV_ISA_EXT_MAX) return false; - return test_bit(bit, bmap) ? true : false; + return test_bit(bit, bmap); } EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); diff --git a/arch/riscv/kernel/vendor_extensions.c b/arch/riscv/kernel/vendor_extensions.c index a31ff84740ebc..9feb7f67a0a39 100644 --- a/arch/riscv/kernel/vendor_extensions.c +++ b/arch/riscv/kernel/vendor_extensions.c @@ -61,6 +61,6 @@ bool __riscv_isa_vendor_extension_available(int cpu, unsigned long vendor, unsig if (bit >= RISCV_ISA_VENDOR_EXT_MAX) return false; - return test_bit(bit, bmap->isa) ? true : false; + return test_bit(bit, bmap->isa); } EXPORT_SYMBOL_GPL(__riscv_isa_vendor_extension_available); From ffef54ad41101f98ea6dd1dcd71c60bb6b7c8ee9 Mon Sep 17 00:00:00 2001 From: Nick Hu Date: Wed, 19 Feb 2025 19:41:34 +0800 Subject: [PATCH 16/65] riscv: Add stimecmp save and restore If the HW support the SSTC extension, we should save and restore the stimecmp register while cpu non retention suspend. Signed-off-by: Nick Hu Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20250219114135.27764-2-nick.hu@sifive.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/suspend.h | 4 ++++ arch/riscv/kernel/suspend.c | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/riscv/include/asm/suspend.h b/arch/riscv/include/asm/suspend.h index 4ffb022b097f4..dc5782b5fbad0 100644 --- a/arch/riscv/include/asm/suspend.h +++ b/arch/riscv/include/asm/suspend.h @@ -18,6 +18,10 @@ struct suspend_context { unsigned long ie; #ifdef CONFIG_MMU unsigned long satp; + unsigned long stimecmp; +#if __riscv_xlen < 64 + unsigned long stimecmph; +#endif #endif }; diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index 9a8a0dc035b26..24b3f57d467f8 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -30,6 +30,13 @@ void suspend_save_csrs(struct suspend_context *context) */ #ifdef CONFIG_MMU + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) { + context->stimecmp = csr_read(CSR_STIMECMP); +#if __riscv_xlen < 64 + context->stimecmph = csr_read(CSR_STIMECMPH); +#endif + } + context->satp = csr_read(CSR_SATP); #endif } @@ -43,6 +50,13 @@ void suspend_restore_csrs(struct suspend_context *context) csr_write(CSR_IE, context->ie); #ifdef CONFIG_MMU + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) { + csr_write(CSR_STIMECMP, context->stimecmp); +#if __riscv_xlen < 64 + csr_write(CSR_STIMECMPH, context->stimecmph); +#endif + } + csr_write(CSR_SATP, context->satp); #endif } From 70c93b026ed07078e933583591aa9ca6701cd9da Mon Sep 17 00:00:00 2001 From: Nick Hu Date: Wed, 19 Feb 2025 19:41:35 +0800 Subject: [PATCH 17/65] clocksource/drivers/timer-riscv: Stop stimecmp when cpu hotplug Stop the timer when the cpu is going to be offline otherwise the timer interrupt may be pending while performing power-down. Suggested-by: Anup Patel Link: https://lore.kernel.org/lkml/20240829033904.477200-3-nick.hu@sifive.com/T/#u Signed-off-by: Nick Hu Reviewed-by: Anup Patel Acked-by: Daniel Lezcano Link: https://lore.kernel.org/r/20250219114135.27764-3-nick.hu@sifive.com Signed-off-by: Alexandre Ghiti --- drivers/clocksource/timer-riscv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index 48ce50c5f5e68..4d7cf338824a3 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -126,7 +126,13 @@ static int riscv_timer_starting_cpu(unsigned int cpu) static int riscv_timer_dying_cpu(unsigned int cpu) { + /* + * Stop the timer when the cpu is going to be offline otherwise + * the timer interrupt may be pending while performing power-down. + */ + riscv_clock_event_stop(); disable_percpu_irq(riscv_clock_event_irq); + return 0; } From 418af0eafb48bbd972040703e5ff5ba94526b5b5 Mon Sep 17 00:00:00 2001 From: Chin Yik Ming Date: Fri, 15 Nov 2024 05:27:25 +0800 Subject: [PATCH 18/65] riscv: Fix a comment typo in set_mm_asid() s/verion/version Signed-off-by: Chin Yik Ming Reviewed-by: Charlie Jenkins Link: https://lore.kernel.org/r/20241114212725.4172401-1-yikming2222@gmail.com Signed-off-by: Alexandre Ghiti --- arch/riscv/mm/context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 4abe3de23225e..55c20ad1f7444 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -158,7 +158,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu) * * - We get a zero back from the cmpxchg and end up waiting on the * lock. Taking the lock synchronises with the rollover and so - * we are forced to see the updated verion. + * we are forced to see the updated version. * * - We get a valid context back from the cmpxchg then we continue * using old ASID because __flush_context() would have marked ASID From c2db83fe10331861d7feb60615c18a72eaf9d444 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 5 Feb 2025 15:07:17 +0100 Subject: [PATCH 19/65] riscv: defconfig: Disable Renesas SoC support Follow-up to commit e36ddf3226864e09 ("riscv: defconfig: Disable RZ/Five peripheral support") in v6.12-rc1: - Disable ARCH_RENESAS, too, as currently RZ/Five is the sole Renesas RISC-V SoC, - Drop no longer needed explicit disable of USB_XHCI_RCAR, which depends on ARCH_RENESAS. Signed-off-by: Geert Uytterhoeven Reviewed-by: Lad Prabhakar Link: https://lore.kernel.org/r/e8a2fb273c8c68bd6d526b924b4212f397195b28.1738764211.git.geert+renesas@glider.be Signed-off-by: Alexandre Ghiti --- arch/riscv/configs/defconfig | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 0f7dcbe3c45bb..3c8e16d71e175 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -26,7 +26,6 @@ CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set CONFIG_PROFILING=y CONFIG_ARCH_MICROCHIP=y -CONFIG_ARCH_RENESAS=y CONFIG_ARCH_SIFIVE=y CONFIG_ARCH_SOPHGO=y CONFIG_ARCH_SPACEMIT=y @@ -202,7 +201,6 @@ CONFIG_USB=y CONFIG_USB_OTG=y CONFIG_USB_XHCI_HCD=y CONFIG_USB_XHCI_PLATFORM=y -# CONFIG_USB_XHCI_RCAR is not set CONFIG_USB_EHCI_HCD=y CONFIG_USB_EHCI_HCD_PLATFORM=y CONFIG_USB_OHCI_HCD=y From 72770690e02c082efbbbd78d768028cf8dd18b9c Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 5 Feb 2025 15:09:03 +0100 Subject: [PATCH 20/65] riscv: Remove duplicate CLINT_TIMER selections Since commit f862bbf4cdca696e ("riscv: Allow NOMMU kernels to run in S-mode") in v6.10, CLINT_TIMER is selected by the main RISCV symbol when RISCV_M_MODE is enabled. Signed-off-by: Geert Uytterhoeven Reviewed-by: Conor Dooley Acked-by: Conor Dooley Link: https://lore.kernel.org/r/ce55529a42fa232cacd580e38866c60701f91095.1738764474.git.geert+renesas@glider.be Signed-off-by: Alexandre Ghiti --- arch/riscv/Kconfig.socs | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 17606940bb523..8b503e54fa1ba 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -59,7 +59,6 @@ config ARCH_THEAD config ARCH_VIRT bool "QEMU Virt Machine" - select CLINT_TIMER if RISCV_M_MODE select POWER_RESET select POWER_RESET_SYSCON select POWER_RESET_SYSCON_POWEROFF @@ -79,7 +78,6 @@ config ARCH_CANAAN config SOC_CANAAN_K210 bool "Canaan Kendryte K210 SoC" depends on !MMU && ARCH_CANAAN - select CLINT_TIMER if RISCV_M_MODE select ARCH_HAS_RESET_CONTROLLER select PINCTRL select COMMON_CLK From bba547810c66434475d8800b3411c59ef71eafe9 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Mon, 24 Feb 2025 18:42:21 -0800 Subject: [PATCH 21/65] riscv: tracing: Fix __write_overflow_field in ftrace_partial_regs() The size of ®s->a0 is unknown, causing the error: ../include/linux/fortify-string.h:571:25: warning: call to '__write_overflow_field' declared with attribute warning: detected write beyond size of field (1st parameter); maybe use struct_group()? [-Wattribute-warning] Fix this by wrapping the required registers in pt_regs with struct_group() and reference the group when doing the offending memcpy(). Signed-off-by: Charlie Jenkins Reviewed-by: Alexandre Ghiti Tested-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250224-fix_ftrace_partial_regs-v1-1-54b906417e86@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/ftrace.h | 2 +- arch/riscv/include/asm/ptrace.h | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index c4721ce44ca47..ec6db1162021f 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -207,7 +207,7 @@ ftrace_partial_regs(const struct ftrace_regs *fregs, struct pt_regs *regs) { struct __arch_ftrace_regs *afregs = arch_ftrace_regs(fregs); - memcpy(®s->a0, afregs->args, sizeof(afregs->args)); + memcpy(®s->a_regs, afregs->args, sizeof(afregs->args)); regs->epc = afregs->epc; regs->ra = afregs->ra; regs->sp = afregs->sp; diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h index b5b0adcc85c18..2910231977cb7 100644 --- a/arch/riscv/include/asm/ptrace.h +++ b/arch/riscv/include/asm/ptrace.h @@ -23,14 +23,16 @@ struct pt_regs { unsigned long t2; unsigned long s0; unsigned long s1; - unsigned long a0; - unsigned long a1; - unsigned long a2; - unsigned long a3; - unsigned long a4; - unsigned long a5; - unsigned long a6; - unsigned long a7; + struct_group(a_regs, + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + ); unsigned long s2; unsigned long s3; unsigned long s4; From 82e81b89501a9f19a3a0b0d7d9641d86c1956284 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 22 Dec 2024 09:08:25 +0900 Subject: [PATCH 22/65] riscv: migrate to the generic rule for built-in DTB Commit 654102df2ac2 ("kbuild: add generic support for built-in boot DTBs") introduced generic support for built-in DTBs. Select GENERIC_BUILTIN_DTB when built-in DTB support is enabled. To keep consistency across architectures, this commit also renames CONFIG_BUILTIN_DTB_SOURCE to CONFIG_BUILTIN_DTB_NAME. Signed-off-by: Masahiro Yamada Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20241222000836.2578171-1-masahiroy@kernel.org Signed-off-by: Alexandre Ghiti --- arch/riscv/Kbuild | 1 - arch/riscv/Kconfig | 3 ++- arch/riscv/boot/dts/Makefile | 2 -- arch/riscv/configs/nommu_k210_defconfig | 2 +- arch/riscv/configs/nommu_k210_sdcard_defconfig | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/arch/riscv/Kbuild b/arch/riscv/Kbuild index 2c585f7a0b6ef..126fb738fc442 100644 --- a/arch/riscv/Kbuild +++ b/arch/riscv/Kbuild @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y += kernel/ mm/ net/ -obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ obj-$(CONFIG_CRYPTO) += crypto/ obj-y += errata/ obj-$(CONFIG_KVM) += kvm/ diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index ff35eb16853cd..bae5fd79d6901 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -1272,13 +1272,14 @@ config RISCV_ISA_FALLBACK config BUILTIN_DTB bool "Built-in device tree" depends on OF && NONPORTABLE + select GENERIC_BUILTIN_DTB help Build a device tree into the Linux image. This option should be selected if no bootloader is being used. If unsure, say N. -config BUILTIN_DTB_SOURCE +config BUILTIN_DTB_NAME string "Built-in device tree source" depends on BUILTIN_DTB help diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile index bff887d38abe4..64a898da9aee3 100644 --- a/arch/riscv/boot/dts/Makefile +++ b/arch/riscv/boot/dts/Makefile @@ -8,5 +8,3 @@ subdir-y += sophgo subdir-y += spacemit subdir-y += starfive subdir-y += thead - -obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix .dtb.o, $(CONFIG_BUILTIN_DTB_SOURCE)) diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index 87ff5a1233af8..ee18d1e333f20 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -35,7 +35,7 @@ CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y CONFIG_BUILTIN_DTB=y -CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic" +CONFIG_BUILTIN_DTB_NAME="canaan/k210_generic" # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set # CONFIG_GCC_PLUGINS is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig index 95cbd574f291f..e770d81b738e1 100644 --- a/arch/riscv/configs/nommu_k210_sdcard_defconfig +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -27,7 +27,7 @@ CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0 root=/dev/mmcblk0p1 rootwait ro" CONFIG_CMDLINE_FORCE=y CONFIG_BUILTIN_DTB=y -CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic" +CONFIG_BUILTIN_DTB_NAME="canaan/k210_generic" # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set # CONFIG_GCC_PLUGINS is not set From 5f1a58ed91a040d4625d854f9bb3dd4995919202 Mon Sep 17 00:00:00 2001 From: Juhan Jin Date: Thu, 6 Feb 2025 13:28:36 -0600 Subject: [PATCH 23/65] riscv: ftrace: Add parentheses in macro definitions of make_call_t0 and make_call_ra This patch adds parentheses to parameters caller and callee of macros make_call_t0 and make_call_ra. Every existing invocation of these two macros uses a single variable for each argument, so the absence of the parentheses seems okay. However, future invocations might use more complex expressions as arguments. For example, a future invocation might look like this: make_call_t0(a - b, c, call). Without parentheses in the macro definition, the macro invocation expands to: ... unsigned int offset = (unsigned long) c - (unsigned long) a - b; ... which is clearly wrong. The use of parentheses ensures arguments are correctly evaluated and potentially saves future users of make_call_t0 and make_call_ra debugging trouble. Fixes: 6724a76cff85 ("riscv: ftrace: Reduce the detour code size to half") Signed-off-by: Juhan Jin Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/tencent_AE90AA59903A628E87E9F80E563DA5BA5508@qq.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/ftrace.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index ec6db1162021f..9704a73e515a5 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -92,7 +92,7 @@ struct dyn_arch_ftrace { #define make_call_t0(caller, callee, call) \ do { \ unsigned int offset = \ - (unsigned long) callee - (unsigned long) caller; \ + (unsigned long) (callee) - (unsigned long) (caller); \ call[0] = to_auipc_t0(offset); \ call[1] = to_jalr_t0(offset); \ } while (0) @@ -108,7 +108,7 @@ do { \ #define make_call_ra(caller, callee, call) \ do { \ unsigned int offset = \ - (unsigned long) callee - (unsigned long) caller; \ + (unsigned long) (callee) - (unsigned long) (caller); \ call[0] = to_auipc_ra(offset); \ call[1] = to_jalr_ra(offset); \ } while (0) From eac5b138814a69435dae04c41591023477b3a18d Mon Sep 17 00:00:00 2001 From: Zixian Zeng Date: Tue, 14 Jan 2025 00:30:20 +0800 Subject: [PATCH 24/65] riscv: remove redundant CMDLINE_FORCE check Drop redundant CMDLINE_FORCE check as it's already done in function early_init_dt_scan_chosen(). Signed-off-by: Zixian Zeng Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250114-rebund-v1-1-5632b2d54d6c@gmail.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/setup.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 4fe45daa6281e..c174544eefc8e 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -235,11 +235,6 @@ static void __init parse_dtb(void) } else { pr_err("No DTB passed to the kernel\n"); } - -#ifdef CONFIG_CMDLINE_FORCE - strscpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE); - pr_info("Forcing kernel command line to: %s\n", boot_command_line); -#endif } #if defined(CONFIG_RISCV_COMBO_SPINLOCKS) From eb8db421ce83f4acf3ca51e642120f42adea3a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Mon, 17 Feb 2025 08:37:58 +0100 Subject: [PATCH 25/65] riscv: mm: Don't use %pK through printk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restricted pointers ("%pK") are not meant to be used through printk(). It can unintentionally expose security sensitive, raw pointer values. Use regular pointer formatting instead. Link: https://lore.kernel.org/lkml/20250113171731-dc10e3c1-da64-4af0-b767-7c7070468023@linutronix.de/ Signed-off-by: Thomas Weißschuh Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250217-restricted-pointers-riscv-v1-1-72a078076a76@linutronix.de Signed-off-by: Alexandre Ghiti --- arch/riscv/mm/physaddr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c index 18706f457da7e..559d291fac5c6 100644 --- a/arch/riscv/mm/physaddr.c +++ b/arch/riscv/mm/physaddr.c @@ -12,7 +12,7 @@ phys_addr_t __virt_to_phys(unsigned long x) * Boundary checking aginst the kernel linear mapping space. */ WARN(!is_linear_mapping(x) && !is_kernel_mapping(x), - "virt_to_phys used for non-linear address: %pK (%pS)\n", + "virt_to_phys used for non-linear address: %p (%pS)\n", (void *)x, (void *)x); return __va_to_pa_nodebug(x); From 475afa39b123699e910c61ad9a51cedce4a0d310 Mon Sep 17 00:00:00 2001 From: Tingbo Liao Date: Fri, 28 Feb 2025 01:08:01 -0800 Subject: [PATCH 26/65] riscv: Fix the __riscv_copy_vec_words_unaligned implementation Correct the VEC_S macro definition to fix the implementation of vector words copy in the case of unalignment in RISC-V. Fixes: e7c9d66e313b ("RISC-V: Report vector unaligned access speed hwprobe") Reviewed-by: Alexandre Ghiti Signed-off-by: Tingbo Liao Link: https://lore.kernel.org/r/20250228090801.8334-1-tingbo.liao@starfivetech.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/vec-copy-unaligned.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/vec-copy-unaligned.S b/arch/riscv/kernel/vec-copy-unaligned.S index d16f19f1b3b65..7ce4de6f6e694 100644 --- a/arch/riscv/kernel/vec-copy-unaligned.S +++ b/arch/riscv/kernel/vec-copy-unaligned.S @@ -11,7 +11,7 @@ #define WORD_SEW CONCATENATE(e, WORD_EEW) #define VEC_L CONCATENATE(vle, WORD_EEW).v -#define VEC_S CONCATENATE(vle, WORD_EEW).v +#define VEC_S CONCATENATE(vse, WORD_EEW).v /* void __riscv_copy_vec_words_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using word loads and stores. */ From 33981b1c4e499021421686dcfa7b3d23a430d00e Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Fri, 28 Feb 2025 10:06:13 +0100 Subject: [PATCH 27/65] riscv: Fix missing __free_pages() in check_vector_unaligned_access() The locally allocated pages are never freed up, so add the corresponding __free_pages(). Fixes: e7c9d66e313b ("RISC-V: Report vector unaligned access speed hwprobe") Link: https://lore.kernel.org/r/20250228090613.345309-1-alexghiti@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/unaligned_access_speed.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 91f189cf16113..074ac4abd023e 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -349,7 +349,7 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n", cpu); - return; + goto free; } if (word_cycles < byte_cycles) @@ -363,6 +363,9 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus (speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow"); per_cpu(vector_misaligned_access, cpu) = speed; + +free: + __free_pages(page, MISALIGNED_BUFFER_ORDER); } static int riscv_online_cpu_vec(unsigned int cpu) From e8eb8e1bdae94b9e003f5909519fd311d0936890 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Mon, 17 Mar 2025 03:12:13 +0000 Subject: [PATCH 28/65] riscv: fgraph: Select HAVE_FUNCTION_GRAPH_TRACER depends on HAVE_DYNAMIC_FTRACE_WITH_ARGS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, fgraph on riscv relies on the infrastructure of DYNAMIC_FTRACE_WITH_ARGS. However, DYNAMIC_FTRACE_WITH_ARGS may be turned off on riscv, which will cause the enabled fgraph to be abnormal. Therefore, let's select HAVE_FUNCTION_GRAPH_TRACER depends on HAVE_DYNAMIC_FTRACE_WITH_ARGS. Fixes: a3ed4157b7d8 ("fgraph: Replace fgraph_ret_regs with ftrace_regs") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202503160820.dvqMpH0g-lkp@intel.com/ Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Link: https://lore.kernel.org/r/20250317031214.4138436-1-pulehui@huaweicloud.com Signed-off-by: Alexandre Ghiti --- arch/riscv/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index bae5fd79d6901..e2c80e88b5cdd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -150,7 +150,7 @@ config RISCV select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_GRAPH_FUNC select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL - select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_GRAPH_TRACER if HAVE_DYNAMIC_FTRACE_WITH_ARGS select HAVE_FUNCTION_GRAPH_FREGS select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION select HAVE_EBPF_JIT if MMU From 67a5ba8f742f247bc83e46dd2313c142b1383276 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Mon, 17 Mar 2025 03:12:14 +0000 Subject: [PATCH 29/65] riscv: fgraph: Fix stack layout to match __arch_ftrace_regs argument of ftrace_return_to_handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Naresh Kamboju reported a "Bad frame pointer" kernel warning while running LTP trace ftrace_stress_test.sh in riscv. We can reproduce the same issue with the following command: ``` $ cd /sys/kernel/debug/tracing $ echo 'f:myprobe do_nanosleep%return args1=$retval' > dynamic_events $ echo 1 > events/fprobes/enable $ echo 1 > tracing_on $ sleep 1 ``` And we can get the following kernel warning: [ 127.692888] ------------[ cut here ]------------ [ 127.693755] Bad frame pointer: expected ff2000000065be50, received ba34c141e9594000 [ 127.693755] from func do_nanosleep return to ffffffff800ccb16 [ 127.698699] WARNING: CPU: 1 PID: 129 at kernel/trace/fgraph.c:755 ftrace_return_to_handler+0x1b2/0x1be [ 127.699894] Modules linked in: [ 127.700908] CPU: 1 UID: 0 PID: 129 Comm: sleep Not tainted 6.14.0-rc3-g0ab191c74642 #32 [ 127.701453] Hardware name: riscv-virtio,qemu (DT) [ 127.701859] epc : ftrace_return_to_handler+0x1b2/0x1be [ 127.702032] ra : ftrace_return_to_handler+0x1b2/0x1be [ 127.702151] epc : ffffffff8013b5e0 ra : ffffffff8013b5e0 sp : ff2000000065bd10 [ 127.702221] gp : ffffffff819c12f8 tp : ff60000080853100 t0 : 6e00000000000000 [ 127.702284] t1 : 0000000000000020 t2 : 6e7566206d6f7266 s0 : ff2000000065bd80 [ 127.702346] s1 : ff60000081262000 a0 : 000000000000007b a1 : ffffffff81894f20 [ 127.702408] a2 : 0000000000000010 a3 : fffffffffffffffe a4 : 0000000000000000 [ 127.702470] a5 : 0000000000000000 a6 : 0000000000000008 a7 : 0000000000000038 [ 127.702530] s2 : ba34c141e9594000 s3 : 0000000000000000 s4 : ff2000000065bdd0 [ 127.702591] s5 : 00007fff8adcf400 s6 : 000055556dc1d8c0 s7 : 0000000000000068 [ 127.702651] s8 : 00007fff8adf5d10 s9 : 000000000000006d s10: 0000000000000001 [ 127.702710] s11: 00005555737377c8 t3 : ffffffff819d899e t4 : ffffffff819d899e [ 127.702769] t5 : ffffffff819d89a0 t6 : ff2000000065bb18 [ 127.702826] status: 0000000200000120 badaddr: 0000000000000000 cause: 0000000000000003 [ 127.703292] [] ftrace_return_to_handler+0x1b2/0x1be [ 127.703760] [] return_to_handler+0x16/0x26 [ 127.704009] [] return_to_handler+0x0/0x26 [ 127.704057] [] common_nsleep+0x42/0x54 [ 127.704117] [] __riscv_sys_clock_nanosleep+0xba/0x10a [ 127.704176] [] do_trap_ecall_u+0x188/0x218 [ 127.704295] [] handle_exception+0x14a/0x156 [ 127.705436] ---[ end trace 0000000000000000 ]--- The reason is that the stack layout for constructing argument for the ftrace_return_to_handler in the return_to_handler does not match the __arch_ftrace_regs structure of riscv, leading to unexpected results. Fixes: a3ed4157b7d8 ("fgraph: Replace fgraph_ret_regs with ftrace_regs") Reported-by: Linux Kernel Functional Testing Closes: https://lore.kernel.org/all/CA+G9fYvp_oAxeDFj88Tk2rfEZ7jtYKAKSwfYS66=57Db9TBdyA@mail.gmail.com Signed-off-by: Pu Lehui Reviewed-by: Alexandre Ghiti Tested-by: Björn Töpel Reviewed-by: Masami Hiramatsu (Google) Link: https://lore.kernel.org/r/20250317031214.4138436-2-pulehui@huaweicloud.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/mcount.S | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S index 068168046e0ef..da4a4000e57ea 100644 --- a/arch/riscv/kernel/mcount.S +++ b/arch/riscv/kernel/mcount.S @@ -12,8 +12,6 @@ #include #include -#define ABI_SIZE_ON_STACK 80 - .text .macro SAVE_ABI_STATE @@ -28,12 +26,12 @@ * register if a0 was not saved. */ .macro SAVE_RET_ABI_STATE - addi sp, sp, -ABI_SIZE_ON_STACK - REG_S ra, 1*SZREG(sp) - REG_S s0, 8*SZREG(sp) - REG_S a0, 10*SZREG(sp) - REG_S a1, 11*SZREG(sp) - addi s0, sp, ABI_SIZE_ON_STACK + addi sp, sp, -FREGS_SIZE_ON_STACK + REG_S ra, FREGS_RA(sp) + REG_S s0, FREGS_S0(sp) + REG_S a0, FREGS_A0(sp) + REG_S a1, FREGS_A1(sp) + addi s0, sp, FREGS_SIZE_ON_STACK .endm .macro RESTORE_ABI_STATE @@ -43,11 +41,11 @@ .endm .macro RESTORE_RET_ABI_STATE - REG_L ra, 1*SZREG(sp) - REG_L s0, 8*SZREG(sp) - REG_L a0, 10*SZREG(sp) - REG_L a1, 11*SZREG(sp) - addi sp, sp, ABI_SIZE_ON_STACK + REG_L ra, FREGS_RA(sp) + REG_L s0, FREGS_S0(sp) + REG_L a0, FREGS_A0(sp) + REG_L a1, FREGS_A1(sp) + addi sp, sp, FREGS_SIZE_ON_STACK .endm SYM_TYPED_FUNC_START(ftrace_stub) From a65e0f67da24d1bff0dc679a018ced26d1952683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 19 Jun 2024 17:39:08 +0200 Subject: [PATCH 30/65] dt-bindings: riscv: add Zaamo and Zalrsc ISA extension description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for the Zaamo and Zalrsc ISA extension[1]. Link: https://github.com/riscv/riscv-zaamo-zalrsc [1] Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240619153913.867263-2-cleger@rivosinc.com Signed-off-by: Alexandre Ghiti --- .../devicetree/bindings/riscv/extensions.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index f26997c3d34d3..73d8fec29ec2c 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -224,6 +224,12 @@ properties: as ratified at commit 4a69197e5617 ("Update to ratified state") of riscv-svvptc. + - const: zaamo + description: | + The standard Zaamo extension for atomic memory operations as + ratified at commit e87412e621f1 ("integrate Zaamo and Zalrsc text + (#1304)") of the unprivileged ISA specification. + - const: zabha description: | The Zabha extension for Byte and Halfword Atomic Memory Operations @@ -236,6 +242,12 @@ properties: is supported as ratified at commit 5059e0ca641c ("update to ratified") of the riscv-zacas. + - const: zalrsc + description: | + The standard Zalrsc extension for load-reserved/store-conditional as + ratified at commit e87412e621f1 ("integrate Zaamo and Zalrsc text + (#1304)") of the unprivileged ISA specification. + - const: zawrs description: | The Zawrs extension for entering a low-power state or for trapping @@ -718,6 +730,13 @@ properties: const: zfbfmin - contains: const: zvfbfmin + # Zacas depends on Zaamo + - if: + contains: + const: zacas + then: + contains: + const: zaamo allOf: # Zcf extension does not exist on rv64 From 35173b666a16ce631da9d70cd1b376162d9dea53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 19 Jun 2024 17:39:09 +0200 Subject: [PATCH 31/65] riscv: add parsing for Zaamo and Zalrsc extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These 2 new extensions are actually a subset of the A extension which provides atomic memory operations and load-reserved/store-conditional instructions. Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240619153913.867263-3-cleger@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/hwcap.h | 2 ++ arch/riscv/kernel/cpufeature.c | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 14cc29f2a723c..e3cbf203cdde7 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -103,6 +103,8 @@ #define RISCV_ISA_EXT_ZFBFMIN 94 #define RISCV_ISA_EXT_ZVFBFMIN 95 #define RISCV_ISA_EXT_ZVFBFWMA 96 +#define RISCV_ISA_EXT_ZAAMO 97 +#define RISCV_ISA_EXT_ZALRSC 98 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 6ec9499e2cf27..af9c346da96e9 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -182,6 +182,11 @@ static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, return 0; } +static const unsigned int riscv_a_exts[] = { + RISCV_ISA_EXT_ZAAMO, + RISCV_ISA_EXT_ZALRSC, +}; + static const unsigned int riscv_zk_bundled_exts[] = { RISCV_ISA_EXT_ZBKB, RISCV_ISA_EXT_ZBKC, @@ -353,7 +358,7 @@ static const unsigned int riscv_c_exts[] = { const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), - __RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a), + __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts), __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), @@ -373,8 +378,10 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), + __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), + __RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC), __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), From 9d45d1ff90a6888f6138eb7e1f2619ef427831d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 19 Jun 2024 17:39:10 +0200 Subject: [PATCH 32/65] riscv: hwprobe: export Zaamo and Zalrsc extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export the Zaamo and Zalrsc extensions to userspace using hwprobe. Signed-off-by: Clément Léger Reviewed-by: Charlie Jenkins Link: https://lore.kernel.org/r/20240619153913.867263-4-cleger@rivosinc.com Signed-off-by: Alexandre Ghiti --- Documentation/arch/riscv/hwprobe.rst | 8 ++++++++ arch/riscv/include/uapi/asm/hwprobe.h | 2 ++ arch/riscv/kernel/sys_hwprobe.c | 2 ++ 3 files changed, 12 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 2792f12e90baa..53607d962653b 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -245,6 +245,14 @@ The following keys are defined: ratified in commit 98918c844281 ("Merge pull request #1217 from riscv/zawrs") of riscv-isa-manual. + * :c:macro:`RISCV_HWPROBE_EXT_ZAAMO`: The Zaamo extension is supported as + defined in the in the RISC-V ISA manual starting from commit e87412e621f1 + ("integrate Zaamo and Zalrsc text (#1304)"). + + * :c:macro:`RISCV_HWPROBE_EXT_ZALRSC`: The Zalrsc extension is supported as + defined in the in the RISC-V ISA manual starting from commit e87412e621f1 + ("integrate Zaamo and Zalrsc text (#1304)"). + * :c:macro:`RISCV_HWPROBE_EXT_SUPM`: The Supm extension is supported as defined in version 1.0 of the RISC-V Pointer Masking extensions. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 056decf65b1b7..3c2fce939673b 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -79,6 +79,8 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 53) #define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 54) #define RISCV_HWPROBE_EXT_ZICBOM (1ULL << 55) +#define RISCV_HWPROBE_EXT_ZAAMO (1ULL << 56) +#define RISCV_HWPROBE_EXT_ZALRSC (1ULL << 57) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index cfe6ceaf30693..f7380ff9ed17a 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -95,7 +95,9 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, * regardless of the kernel's configuration, as no other checks, besides * presence in the hart_isa bitmap, are made. */ + EXT_KEY(ZAAMO); EXT_KEY(ZACAS); + EXT_KEY(ZALRSC); EXT_KEY(ZAWRS); EXT_KEY(ZBA); EXT_KEY(ZBB); From 2d79608cf6112e787e6f90cf909bdadceed30dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 19 Jun 2024 17:39:11 +0200 Subject: [PATCH 33/65] RISC-V: KVM: Allow Zaamo/Zalrsc extensions for Guest/VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the KVM ISA extension ONE_REG interface to allow KVM user space to detect and enable Zaamo/Zalrsc extensions for Guest/VM. Signed-off-by: Clément Léger Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20240619153913.867263-5-cleger@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/uapi/asm/kvm.h | 2 ++ arch/riscv/kvm/vcpu_onereg.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index f06bc5efcd79a..5f59fd226cc57 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -182,6 +182,8 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_SVVPTC, KVM_RISCV_ISA_EXT_ZABHA, KVM_RISCV_ISA_EXT_ZICCRSE, + KVM_RISCV_ISA_EXT_ZAAMO, + KVM_RISCV_ISA_EXT_ZALRSC, KVM_RISCV_ISA_EXT_MAX, }; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index f6d27b59c6419..f05f12f809206 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -47,8 +47,10 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZAAMO), KVM_ISA_EXT_ARR(ZABHA), KVM_ISA_EXT_ARR(ZACAS), + KVM_ISA_EXT_ARR(ZALRSC), KVM_ISA_EXT_ARR(ZAWRS), KVM_ISA_EXT_ARR(ZBA), KVM_ISA_EXT_ARR(ZBB), @@ -149,8 +151,10 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_SVINVAL: case KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_RISCV_ISA_EXT_ZAAMO: case KVM_RISCV_ISA_EXT_ZABHA: case KVM_RISCV_ISA_EXT_ZACAS: + case KVM_RISCV_ISA_EXT_ZALRSC: case KVM_RISCV_ISA_EXT_ZAWRS: case KVM_RISCV_ISA_EXT_ZBA: case KVM_RISCV_ISA_EXT_ZBB: From 7a9827e777da311672cdba2f4400362b1eebb2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 19 Jun 2024 17:39:12 +0200 Subject: [PATCH 34/65] KVM: riscv: selftests: Add Zaamo/Zalrsc extensions to get-reg-list test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The KVM RISC-V allows Zaamo/Zalrsc extensions for Guest/VM so add these extensions to get-reg-list test. Signed-off-by: Clément Léger Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20240619153913.867263-6-cleger@rivosinc.com Signed-off-by: Alexandre Ghiti --- tools/testing/selftests/kvm/riscv/get-reg-list.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index 8515921dfdbf2..569f2d67c9b81 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -53,8 +53,10 @@ bool filter_reg(__u64 reg) case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVPBMT: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZAAMO: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZABHA: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZACAS: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZALRSC: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZAWRS: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZBA: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZBB: @@ -434,8 +436,10 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off) KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZAAMO), KVM_ISA_EXT_ARR(ZABHA), KVM_ISA_EXT_ARR(ZACAS), + KVM_ISA_EXT_ARR(ZALRSC), KVM_ISA_EXT_ARR(ZAWRS), KVM_ISA_EXT_ARR(ZBA), KVM_ISA_EXT_ARR(ZBB), @@ -974,8 +978,10 @@ KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL); KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT); KVM_ISA_EXT_SIMPLE_CONFIG(svpbmt, SVPBMT); KVM_ISA_EXT_SIMPLE_CONFIG(svvptc, SVVPTC); +KVM_ISA_EXT_SIMPLE_CONFIG(zaamo, ZAAMO); KVM_ISA_EXT_SIMPLE_CONFIG(zabha, ZABHA); KVM_ISA_EXT_SIMPLE_CONFIG(zacas, ZACAS); +KVM_ISA_EXT_SIMPLE_CONFIG(zalrsc, ZALRSC); KVM_ISA_EXT_SIMPLE_CONFIG(zawrs, ZAWRS); KVM_ISA_EXT_SIMPLE_CONFIG(zba, ZBA); KVM_ISA_EXT_SIMPLE_CONFIG(zbb, ZBB); @@ -1045,8 +1051,10 @@ struct vcpu_reg_list *vcpu_configs[] = { &config_svnapot, &config_svpbmt, &config_svvptc, + &config_zaamo, &config_zabha, &config_zacas, + &config_zalrsc, &config_zawrs, &config_zba, &config_zbb, From a00e022be5315c5a1f47521a1cc6d3b71c8e9c44 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:16 +0100 Subject: [PATCH 35/65] riscv: Annotate unaligned access init functions Several functions used in unaligned access probing are only run at init time. Annotate them appropriately. Fixes: f413aae96cda ("riscv: Set unaligned access speed at compile time") Reviewed-by: Alexandre Ghiti Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-11-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/cpufeature.h | 4 ++-- arch/riscv/kernel/traps_misaligned.c | 8 ++++---- arch/riscv/kernel/unaligned_access_speed.c | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 569140d6e6399..19defdc2002d8 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -63,7 +63,7 @@ void __init riscv_user_isa_enable(void); #define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \ _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate) -bool check_unaligned_access_emulated_all_cpus(void); +bool __init check_unaligned_access_emulated_all_cpus(void); #if defined(CONFIG_RISCV_SCALAR_MISALIGNED) void check_unaligned_access_emulated(struct work_struct *work __always_unused); void unaligned_emulation_finish(void); @@ -76,7 +76,7 @@ static inline bool unaligned_ctl_available(void) } #endif -bool check_vector_unaligned_access_emulated_all_cpus(void); +bool __init check_vector_unaligned_access_emulated_all_cpus(void); #if defined(CONFIG_RISCV_VECTOR_MISALIGNED) void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused); DECLARE_PER_CPU(long, vector_misaligned_access); diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c index 7cc108aed74e8..aacbd9d7196e7 100644 --- a/arch/riscv/kernel/traps_misaligned.c +++ b/arch/riscv/kernel/traps_misaligned.c @@ -605,7 +605,7 @@ void check_vector_unaligned_access_emulated(struct work_struct *work __always_un kernel_vector_end(); } -bool check_vector_unaligned_access_emulated_all_cpus(void) +bool __init check_vector_unaligned_access_emulated_all_cpus(void) { int cpu; @@ -625,7 +625,7 @@ bool check_vector_unaligned_access_emulated_all_cpus(void) return true; } #else -bool check_vector_unaligned_access_emulated_all_cpus(void) +bool __init check_vector_unaligned_access_emulated_all_cpus(void) { return false; } @@ -659,7 +659,7 @@ void check_unaligned_access_emulated(struct work_struct *work __always_unused) } } -bool check_unaligned_access_emulated_all_cpus(void) +bool __init check_unaligned_access_emulated_all_cpus(void) { int cpu; @@ -684,7 +684,7 @@ bool unaligned_ctl_available(void) return unaligned_ctl; } #else -bool check_unaligned_access_emulated_all_cpus(void) +bool __init check_unaligned_access_emulated_all_cpus(void) { return false; } diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 91f189cf16113..b7a8ff7ba6df4 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -121,7 +121,7 @@ static int check_unaligned_access(void *param) return 0; } -static void check_unaligned_access_nonboot_cpu(void *param) +static void __init check_unaligned_access_nonboot_cpu(void *param) { unsigned int cpu = smp_processor_id(); struct page **pages = param; @@ -175,7 +175,7 @@ static void set_unaligned_access_static_branches(void) modify_unaligned_access_branches(&fast_and_online, num_online_cpus()); } -static int lock_and_set_unaligned_access_static_branch(void) +static int __init lock_and_set_unaligned_access_static_branch(void) { cpus_read_lock(); set_unaligned_access_static_branches(); @@ -218,7 +218,7 @@ static int riscv_offline_cpu(unsigned int cpu) } /* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int check_unaligned_access_speed_all_cpus(void) +static int __init check_unaligned_access_speed_all_cpus(void) { unsigned int cpu; unsigned int cpu_count = num_possible_cpus(); @@ -264,7 +264,7 @@ static int check_unaligned_access_speed_all_cpus(void) return 0; } #else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */ -static int check_unaligned_access_speed_all_cpus(void) +static int __init check_unaligned_access_speed_all_cpus(void) { return 0; } @@ -379,7 +379,7 @@ static int riscv_online_cpu_vec(unsigned int cpu) } /* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) { schedule_on_each_cpu(check_vector_unaligned_access); @@ -393,13 +393,13 @@ static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unuse return 0; } #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) { return 0; } #endif -static int check_unaligned_access_all_cpus(void) +static int __init check_unaligned_access_all_cpus(void) { bool all_cpus_emulated, all_cpus_vec_unsupported; From 5af72a818612332a11171b16f27a62ec0a0f91d7 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:17 +0100 Subject: [PATCH 36/65] riscv: Fix riscv_online_cpu_vec We shouldn't probe when we already know vector is unsupported and we should probe when we see we don't yet know whether it's supported. Furthermore, we should ensure we've set the access type to unsupported when we don't have vector at all. Fixes: e7c9d66e313b ("RISC-V: Report vector unaligned access speed hwprobe") Reviewed-by: Alexandre Ghiti Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-12-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/unaligned_access_speed.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index b7a8ff7ba6df4..161964cf2abce 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -367,10 +367,12 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus static int riscv_online_cpu_vec(unsigned int cpu) { - if (!has_vector()) + if (!has_vector()) { + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; return 0; + } - if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED) + if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) return 0; check_vector_unaligned_access_emulated(NULL); From e6d0adf2eb5bb3244cb21a7a15899aa058bd384f Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:18 +0100 Subject: [PATCH 37/65] riscv: Fix check_unaligned_access_all_cpus check_vector_unaligned_access_emulated_all_cpus(), like its name suggests, will return true when all cpus emulate unaligned vector accesses. If the function returned false it may have been because vector isn't supported at all (!has_vector()) or because at least one cpu doesn't emulate unaligned vector accesses. Since false may be returned for two cases, checking for it isn't sufficient when attempting to determine if we should proceed with the vector speed check. Move the !has_vector() functionality to check_unaligned_access_all_cpus() in order for check_vector_unaligned_access_emulated_all_cpus() to return false for a single case. Fixes: e7c9d66e313b ("RISC-V: Report vector unaligned access speed hwprobe") Reviewed-by: Alexandre Ghiti Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-13-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/traps_misaligned.c | 6 ------ arch/riscv/kernel/unaligned_access_speed.c | 11 +++++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c index aacbd9d7196e7..4354c87c0376f 100644 --- a/arch/riscv/kernel/traps_misaligned.c +++ b/arch/riscv/kernel/traps_misaligned.c @@ -609,12 +609,6 @@ bool __init check_vector_unaligned_access_emulated_all_cpus(void) { int cpu; - if (!has_vector()) { - for_each_online_cpu(cpu) - per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; - return false; - } - schedule_on_each_cpu(check_vector_unaligned_access_emulated); for_each_online_cpu(cpu) diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 161964cf2abce..02b485dc4bc42 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -403,13 +403,16 @@ static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __alway static int __init check_unaligned_access_all_cpus(void) { - bool all_cpus_emulated, all_cpus_vec_unsupported; + bool all_cpus_emulated; + int cpu; all_cpus_emulated = check_unaligned_access_emulated_all_cpus(); - all_cpus_vec_unsupported = check_vector_unaligned_access_emulated_all_cpus(); - if (!all_cpus_vec_unsupported && - IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { + if (!has_vector()) { + for_each_online_cpu(cpu) + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; + } else if (!check_vector_unaligned_access_emulated_all_cpus() && + IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { kthread_run(vec_check_unaligned_access_speed_all_cpus, NULL, "vec_check_unaligned_access_speed_all_cpus"); } From 813d39baee3229d31420af61460b97f4fafdd352 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:19 +0100 Subject: [PATCH 38/65] riscv: Change check_unaligned_access_speed_all_cpus to void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The return value of check_unaligned_access_speed_all_cpus() is always zero, so make the function void so we don't need to concern ourselves with it. The change also allows us to tidy up check_unaligned_access_all_cpus() a bit. Reviewed-by: Clément Léger Reviewed-by: Alexandre Ghiti Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-14-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/unaligned_access_speed.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 02b485dc4bc42..780f1c5f512a4 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -218,7 +218,7 @@ static int riscv_offline_cpu(unsigned int cpu) } /* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int __init check_unaligned_access_speed_all_cpus(void) +static void __init check_unaligned_access_speed_all_cpus(void) { unsigned int cpu; unsigned int cpu_count = num_possible_cpus(); @@ -226,7 +226,7 @@ static int __init check_unaligned_access_speed_all_cpus(void) if (!bufs) { pr_warn("Allocation failure, not measuring misaligned performance\n"); - return 0; + return; } /* @@ -261,12 +261,10 @@ static int __init check_unaligned_access_speed_all_cpus(void) } kfree(bufs); - return 0; } #else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */ -static int __init check_unaligned_access_speed_all_cpus(void) +static void __init check_unaligned_access_speed_all_cpus(void) { - return 0; } #endif @@ -403,10 +401,10 @@ static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __alway static int __init check_unaligned_access_all_cpus(void) { - bool all_cpus_emulated; int cpu; - all_cpus_emulated = check_unaligned_access_emulated_all_cpus(); + if (!check_unaligned_access_emulated_all_cpus()) + check_unaligned_access_speed_all_cpus(); if (!has_vector()) { for_each_online_cpu(cpu) @@ -417,9 +415,6 @@ static int __init check_unaligned_access_all_cpus(void) NULL, "vec_check_unaligned_access_speed_all_cpus"); } - if (!all_cpus_emulated) - return check_unaligned_access_speed_all_cpus(); - return 0; } From 05ee21f0fcb8ca29bf42bd6cbce109f2e49c167f Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:20 +0100 Subject: [PATCH 39/65] riscv: Fix set up of cpu hotplug callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CPU hotplug callbacks should be set up even if we detected all current cpus emulate misaligned accesses, since we want to ensure our expectations of all cpus emulating is maintained. Fixes: 6e5ce7f2eae3 ("riscv: Decouple emulated unaligned accesses from access speed") Fixes: e7c9d66e313b ("RISC-V: Report vector unaligned access speed hwprobe") Reviewed-by: Clément Léger Reviewed-by: Alexandre Ghiti Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-15-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/unaligned_access_speed.c | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 780f1c5f512a4..c9d3237649bbc 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -247,13 +247,6 @@ static void __init check_unaligned_access_speed_all_cpus(void) /* Check core 0. */ smp_call_on_cpu(0, check_unaligned_access, bufs[0], true); - /* - * Setup hotplug callbacks for any new CPUs that come online or go - * offline. - */ - cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", - riscv_online_cpu, riscv_offline_cpu); - out: for_each_cpu(cpu, cpu_online_mask) { if (bufs[cpu]) @@ -383,13 +376,6 @@ static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __alway { schedule_on_each_cpu(check_vector_unaligned_access); - /* - * Setup hotplug callbacks for any new CPUs that come online or go - * offline. - */ - cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", - riscv_online_cpu_vec, NULL); - return 0; } #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ @@ -415,6 +401,19 @@ static int __init check_unaligned_access_all_cpus(void) NULL, "vec_check_unaligned_access_speed_all_cpus"); } + /* + * Setup hotplug callbacks for any new CPUs that come online or go + * offline. + */ +#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", + riscv_online_cpu, riscv_offline_cpu); +#endif +#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", + riscv_online_cpu_vec, NULL); +#endif + return 0; } From 2744ec472de31141ad354907ff98843dd6040917 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:21 +0100 Subject: [PATCH 40/65] riscv: Fix set up of vector cpu hotplug callback Whether or not we have RISCV_PROBE_VECTOR_UNALIGNED_ACCESS we need to set up a cpu hotplug callback to check if we have vector at all, since, when we don't have vector, we need to set vector_misaligned_access to unsupported rather than leave it the default of unknown. Fixes: e7c9d66e313b ("RISC-V: Report vector unaligned access speed hwprobe") Reviewed-by: Alexandre Ghiti Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-16-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/unaligned_access_speed.c | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index c9d3237649bbc..d9d4ca1fadc7f 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -356,6 +356,20 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus per_cpu(vector_misaligned_access, cpu) = speed; } +/* Measure unaligned access speed on all CPUs present at boot in parallel. */ +static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +{ + schedule_on_each_cpu(check_vector_unaligned_access); + + return 0; +} +#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ +static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +{ + return 0; +} +#endif + static int riscv_online_cpu_vec(unsigned int cpu) { if (!has_vector()) { @@ -363,27 +377,16 @@ static int riscv_online_cpu_vec(unsigned int cpu) return 0; } +#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) return 0; check_vector_unaligned_access_emulated(NULL); check_vector_unaligned_access(NULL); - return 0; -} - -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - schedule_on_each_cpu(check_vector_unaligned_access); +#endif return 0; } -#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - return 0; -} -#endif static int __init check_unaligned_access_all_cpus(void) { @@ -409,10 +412,8 @@ static int __init check_unaligned_access_all_cpus(void) cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", riscv_online_cpu, riscv_offline_cpu); #endif -#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", riscv_online_cpu_vec, NULL); -#endif return 0; } From aecb09e091dc143345a9e4b282d0444554445f4b Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:22 +0100 Subject: [PATCH 41/65] riscv: Add parameter for skipping access speed tests Allow skipping scalar and vector unaligned access speed tests. This is useful for testing alternative code paths and to skip the tests in environments where they run too slowly. All CPUs must have the same unaligned access speed. The code movement is because we now need the scalar cpu hotplug callback to always run, so we need to bring it and its supporting functions out of CONFIG_RISCV_PROBE_UNALIGNED_ACCESS. Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-17-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/unaligned_access_speed.c | 187 +++++++++++++-------- 1 file changed, 121 insertions(+), 66 deletions(-) diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index d9d4ca1fadc7f..18e3345495444 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -24,8 +24,12 @@ DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; -#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS +static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; +static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; + static cpumask_t fast_misaligned_access; + +#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS static int check_unaligned_access(void *param) { int cpu = smp_processor_id(); @@ -130,6 +134,50 @@ static void __init check_unaligned_access_nonboot_cpu(void *param) check_unaligned_access(pages[cpu]); } +/* Measure unaligned access speed on all CPUs present at boot in parallel. */ +static void __init check_unaligned_access_speed_all_cpus(void) +{ + unsigned int cpu; + unsigned int cpu_count = num_possible_cpus(); + struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL); + + if (!bufs) { + pr_warn("Allocation failure, not measuring misaligned performance\n"); + return; + } + + /* + * Allocate separate buffers for each CPU so there's no fighting over + * cache lines. + */ + for_each_cpu(cpu, cpu_online_mask) { + bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); + if (!bufs[cpu]) { + pr_warn("Allocation failure, not measuring misaligned performance\n"); + goto out; + } + } + + /* Check everybody except 0, who stays behind to tend jiffies. */ + on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1); + + /* Check core 0. */ + smp_call_on_cpu(0, check_unaligned_access, bufs[0], true); + +out: + for_each_cpu(cpu, cpu_online_mask) { + if (bufs[cpu]) + __free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER); + } + + kfree(bufs); +} +#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */ +static void __init check_unaligned_access_speed_all_cpus(void) +{ +} +#endif + DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key); static void modify_unaligned_access_branches(cpumask_t *mask, int weight) @@ -188,21 +236,29 @@ arch_initcall_sync(lock_and_set_unaligned_access_static_branch); static int riscv_online_cpu(unsigned int cpu) { - static struct page *buf; - /* We are already set since the last check */ - if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) + if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { + goto exit; + } else if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { + per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param; goto exit; - - check_unaligned_access_emulated(NULL); - buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); - if (!buf) { - pr_warn("Allocation failure, not measuring misaligned performance\n"); - return -ENOMEM; } - check_unaligned_access(buf); - __free_pages(buf, MISALIGNED_BUFFER_ORDER); +#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS + { + static struct page *buf; + + check_unaligned_access_emulated(NULL); + buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); + if (!buf) { + pr_warn("Allocation failure, not measuring misaligned performance\n"); + return -ENOMEM; + } + + check_unaligned_access(buf); + __free_pages(buf, MISALIGNED_BUFFER_ORDER); + } +#endif exit: set_unaligned_access_static_branches(); @@ -217,50 +273,6 @@ static int riscv_offline_cpu(unsigned int cpu) return 0; } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static void __init check_unaligned_access_speed_all_cpus(void) -{ - unsigned int cpu; - unsigned int cpu_count = num_possible_cpus(); - struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL); - - if (!bufs) { - pr_warn("Allocation failure, not measuring misaligned performance\n"); - return; - } - - /* - * Allocate separate buffers for each CPU so there's no fighting over - * cache lines. - */ - for_each_cpu(cpu, cpu_online_mask) { - bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); - if (!bufs[cpu]) { - pr_warn("Allocation failure, not measuring misaligned performance\n"); - goto out; - } - } - - /* Check everybody except 0, who stays behind to tend jiffies. */ - on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1); - - /* Check core 0. */ - smp_call_on_cpu(0, check_unaligned_access, bufs[0], true); - -out: - for_each_cpu(cpu, cpu_online_mask) { - if (bufs[cpu]) - __free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER); - } - - kfree(bufs); -} -#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */ -static void __init check_unaligned_access_speed_all_cpus(void) -{ -} -#endif - #ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS static void check_vector_unaligned_access(struct work_struct *work __always_unused) { @@ -372,8 +384,8 @@ static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __alway static int riscv_online_cpu_vec(unsigned int cpu) { - if (!has_vector()) { - per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; + if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { + per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; return 0; } @@ -388,30 +400,73 @@ static int riscv_online_cpu_vec(unsigned int cpu) return 0; } +static const char * const speed_str[] __initconst = { NULL, NULL, "slow", "fast", "unsupported" }; + +static int __init set_unaligned_scalar_speed_param(char *str) +{ + if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW])) + unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_FAST])) + unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED])) + unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED; + else + return -EINVAL; + + return 1; +} +__setup("unaligned_scalar_speed=", set_unaligned_scalar_speed_param); + +static int __init set_unaligned_vector_speed_param(char *str) +{ + if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW])) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_FAST])) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED])) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; + else + return -EINVAL; + + return 1; +} +__setup("unaligned_vector_speed=", set_unaligned_vector_speed_param); + static int __init check_unaligned_access_all_cpus(void) { int cpu; - if (!check_unaligned_access_emulated_all_cpus()) + if (unaligned_scalar_speed_param == RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN && + !check_unaligned_access_emulated_all_cpus()) { check_unaligned_access_speed_all_cpus(); - - if (!has_vector()) { + } else { + pr_info("scalar unaligned access speed set to '%s' by command line\n", + speed_str[unaligned_scalar_speed_param]); for_each_online_cpu(cpu) - per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; - } else if (!check_vector_unaligned_access_emulated_all_cpus() && - IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { + per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param; + } + + if (!has_vector()) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; + + if (unaligned_vector_speed_param == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN && + !check_vector_unaligned_access_emulated_all_cpus() && + IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { kthread_run(vec_check_unaligned_access_speed_all_cpus, NULL, "vec_check_unaligned_access_speed_all_cpus"); + } else { + pr_info("vector unaligned access speed set to '%s' by command line\n", + speed_str[unaligned_vector_speed_param]); + for_each_online_cpu(cpu) + per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; } /* * Setup hotplug callbacks for any new CPUs that come online or go * offline. */ -#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", riscv_online_cpu, riscv_offline_cpu); -#endif cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", riscv_online_cpu_vec, NULL); From 9fe58530a8cd1402aaa35cad19143777a6f65393 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Tue, 4 Mar 2025 13:00:23 +0100 Subject: [PATCH 42/65] Documentation/kernel-parameters: Add riscv unaligned speed parameters Document riscv parameters used to select scalar and vector unaligned access speeds. Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20250304120014.143628-18-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti --- Documentation/admin-guide/kernel-parameters.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index fb8752b42ec85..9e3c5fecfa52e 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7477,6 +7477,22 @@ Note that genuine overcurrent events won't be reported either. + unaligned_scalar_speed= + [RISCV] + Format: {slow | fast | unsupported} + Allow skipping scalar unaligned access speed tests. This + is useful for testing alternative code paths and to skip + the tests in environments where they run too slowly. All + CPUs must have the same scalar unaligned access speed. + + unaligned_vector_speed= + [RISCV] + Format: {slow | fast | unsupported} + Allow skipping vector unaligned access speed tests. This + is useful for testing alternative code paths and to skip + the tests in environments where they run too slowly. All + CPUs must have the same vector unaligned access speed. + unknown_nmi_panic [X86] Cause panic on unknown NMI. From afa8a93932aa63b107d81bd438454760d8c7c8a3 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Wed, 19 Mar 2025 11:35:19 -0700 Subject: [PATCH 43/65] riscv: Move nop definition to insn-def.h We have duplicated the definition of the nop instruction in ftrace.h and in jump_label.c. Move this definition into the generic file insn-def.h so that they can share the definition with each other and with future files. Signed-off-by: Charlie Jenkins Reviewed-by: Alexandre Ghiti Tested-by: Alexandre Ghiti Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20250319-runtime_const_riscv-v10-1-745b31a11d65@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/ftrace.h | 1 - arch/riscv/include/asm/insn-def.h | 3 +++ arch/riscv/kernel/ftrace.c | 6 +++--- arch/riscv/kernel/jump_label.c | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index c4721ce44ca47..b7f361a50f644 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -79,7 +79,6 @@ struct dyn_arch_ftrace { #define AUIPC_RA (0x00000097) #define JALR_T0 (0x000282e7) #define AUIPC_T0 (0x00000297) -#define NOP4 (0x00000013) #define to_jalr_t0(offset) \ (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0) diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index 9a913010cdd93..71060a2f838e2 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -199,5 +199,8 @@ #define RISCV_PAUSE ".4byte 0x100000f" #define ZAWRS_WRS_NTO ".4byte 0x00d00073" #define ZAWRS_WRS_STO ".4byte 0x01d00073" +#define RISCV_NOP4 ".4byte 0x00000013" + +#define RISCV_INSN_NOP4 _AC(0x00000013, U) #endif /* __ASM_INSN_DEF_H */ diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 3524db5e4fa01..674dcdfae7a14 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -36,7 +36,7 @@ static int ftrace_check_current_call(unsigned long hook_pos, unsigned int *expected) { unsigned int replaced[2]; - unsigned int nops[2] = {NOP4, NOP4}; + unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4}; /* we expect nops at the hook position */ if (!expected) @@ -68,7 +68,7 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, bool enable, bool ra) { unsigned int call[2]; - unsigned int nops[2] = {NOP4, NOP4}; + unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4}; if (ra) make_call_ra(hook_pos, target, call); @@ -97,7 +97,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { - unsigned int nops[2] = {NOP4, NOP4}; + unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4}; if (patch_insn_write((void *)rec->ip, nops, MCOUNT_INSN_SIZE)) return -EPERM; diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c index 654ed159c830b..b4c1a6a3fbd28 100644 --- a/arch/riscv/kernel/jump_label.c +++ b/arch/riscv/kernel/jump_label.c @@ -11,8 +11,8 @@ #include #include #include +#include -#define RISCV_INSN_NOP 0x00000013U #define RISCV_INSN_JAL 0x0000006fU bool arch_jump_label_transform_queue(struct jump_entry *entry, @@ -33,7 +33,7 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry, (((u32)offset & GENMASK(10, 1)) << (21 - 1)) | (((u32)offset & GENMASK(20, 20)) << (31 - 20)); } else { - insn = RISCV_INSN_NOP; + insn = RISCV_INSN_NOP4; } if (early_boot_irqs_disabled) { From a44fb5722199de8338d991db5ad3d509192179bb Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Wed, 19 Mar 2025 11:35:20 -0700 Subject: [PATCH 44/65] riscv: Add runtime constant support Implement the runtime constant infrastructure for riscv. Use this infrastructure to generate constants to be used by the d_hash() function. This is the riscv variant of commit 94a2bc0f611c ("arm64: add 'runtime constant' support") and commit e3c92e81711d ("runtime constants: add x86 architecture support"). [ alex: Remove trailing whitespace ] Signed-off-by: Charlie Jenkins Reviewed-by: Alexandre Ghiti Tested-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250319-runtime_const_riscv-v10-2-745b31a11d65@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/Kconfig | 22 ++ arch/riscv/include/asm/asm.h | 1 + arch/riscv/include/asm/runtime-const.h | 265 +++++++++++++++++++++++++ arch/riscv/kernel/vmlinux.lds.S | 3 + 4 files changed, 291 insertions(+) create mode 100644 arch/riscv/include/asm/runtime-const.h diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7612c52e9b1e3..c123f7c0579c1 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -783,6 +783,28 @@ config RISCV_ISA_ZBC If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBKB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbkb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbkb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_HAS_OPTION_ARCH + +config RISCV_ISA_ZBKB + bool "Zbkb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBKB + depends on RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBKB + extension (bit manipulation for cryptography) and enable its usage. + + The Zbkb extension provides instructions to accelerate a number + of common cryptography operations (pack, zip, etc). + + If you don't know what to do here, say Y. + config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on MMU diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index 776354895b81e..a8a2af6dfe9d2 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -27,6 +27,7 @@ #define REG_ASM __REG_SEL(.dword, .word) #define SZREG __REG_SEL(8, 4) #define LGREG __REG_SEL(3, 2) +#define SRLI __REG_SEL(srliw, srli) #if __SIZEOF_POINTER__ == 8 #ifdef __ASSEMBLY__ diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h new file mode 100644 index 0000000000000..ea2e49c7149cd --- /dev/null +++ b/arch/riscv/include/asm/runtime-const.h @@ -0,0 +1,265 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_RUNTIME_CONST_H +#define _ASM_RISCV_RUNTIME_CONST_H + +#include +#include +#include +#include +#include +#include + +#include + +#ifdef CONFIG_32BIT +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + "lui %[__ret],0x89abd\n\t" \ + "addi %[__ret],%[__ret],-0x211\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret)); \ + __ret; \ +}) +#else +/* + * Loading 64-bit constants into a register from immediates is a non-trivial + * task on riscv64. To get it somewhat performant, load 32 bits into two + * different registers and then combine the results. + * + * If the processor supports the Zbkb extension, we can combine the final + * "slli,slli,srli,add" into the single "pack" instruction. If the processor + * doesn't support Zbkb but does support the Zbb extension, we can + * combine the final "slli,srli,add" into one instruction "add.uw". + */ +#define RISCV_RUNTIME_CONST_64_PREAMBLE \ + ".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + "lui %[__ret],0x89abd\n\t" \ + "lui %[__tmp],0x1234\n\t" \ + "addiw %[__ret],%[__ret],-0x211\n\t" \ + "addiw %[__tmp],%[__tmp],0x567\n\t" \ + +#define RISCV_RUNTIME_CONST_64_BASE \ + "slli %[__tmp],%[__tmp],32\n\t" \ + "slli %[__ret],%[__ret],32\n\t" \ + "srli %[__ret],%[__ret],32\n\t" \ + "add %[__ret],%[__ret],%[__tmp]\n\t" \ + +#define RISCV_RUNTIME_CONST_64_ZBA \ + ".option push\n\t" \ + ".option arch,+zba\n\t" \ + "slli %[__tmp],%[__tmp],32\n\t" \ + "add.uw %[__ret],%[__ret],%[__tmp]\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".option pop\n\t" \ + +#define RISCV_RUNTIME_CONST_64_ZBKB \ + ".option push\n\t" \ + ".option arch,+zbkb\n\t" \ + "pack %[__ret],%[__ret],%[__tmp]\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".option pop\n\t" \ + +#define RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + ".option pop\n\t" \ + ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + +#if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_RISCV_ISA_ZBKB) +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + ALTERNATIVE_2( \ + RISCV_RUNTIME_CONST_64_BASE, \ + RISCV_RUNTIME_CONST_64_ZBA, \ + 0, RISCV_ISA_EXT_ZBA, 1, \ + RISCV_RUNTIME_CONST_64_ZBKB, \ + 0, RISCV_ISA_EXT_ZBKB, 1 \ + ) \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#elif defined(CONFIG_RISCV_ISA_ZBA) +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + ALTERNATIVE( \ + RISCV_RUNTIME_CONST_64_BASE, \ + RISCV_RUNTIME_CONST_64_ZBA, \ + 0, RISCV_ISA_EXT_ZBA, 1 \ + ) \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#elif defined(CONFIG_RISCV_ISA_ZBKB) +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + ALTERNATIVE( \ + RISCV_RUNTIME_CONST_64_BASE, \ + RISCV_RUNTIME_CONST_64_ZBKB, \ + 0, RISCV_ISA_EXT_ZBKB, 1 \ + ) \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#else +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + RISCV_RUNTIME_CONST_64_BASE \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#endif +#endif + +#define runtime_const_shift_right_32(val, sym) \ +({ \ + u32 __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + SRLI " %[__ret],%[__val],12\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_shift_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret) \ + : [__val] "r" (val)); \ + __ret; \ +}) + +#define runtime_const_init(type, sym) do { \ + extern s32 __start_runtime_##type##_##sym[]; \ + extern s32 __stop_runtime_##type##_##sym[]; \ + \ + runtime_const_fixup(__runtime_fixup_##type, \ + (unsigned long)(sym), \ + __start_runtime_##type##_##sym, \ + __stop_runtime_##type##_##sym); \ +} while (0) + +static inline void __runtime_fixup_caches(void *where, unsigned int insns) +{ + /* On riscv there are currently only cache-wide flushes so va is ignored. */ + __always_unused uintptr_t va = (uintptr_t)where; + + flush_icache_range(va, va + 4 * insns); +} + +/* + * The 32-bit immediate is stored in a lui+addi pairing. + * lui holds the upper 20 bits of the immediate in the first 20 bits of the instruction. + * addi holds the lower 12 bits of the immediate in the first 12 bits of the instruction. + */ +static inline void __runtime_fixup_32(__le16 *lui_parcel, __le16 *addi_parcel, unsigned int val) +{ + unsigned int lower_immediate, upper_immediate; + u32 lui_insn, addi_insn, addi_insn_mask; + __le32 lui_res, addi_res; + + /* Mask out upper 12 bit of addi */ + addi_insn_mask = 0x000fffff; + + lui_insn = (u32)le16_to_cpu(lui_parcel[0]) | (u32)le16_to_cpu(lui_parcel[1]) << 16; + addi_insn = (u32)le16_to_cpu(addi_parcel[0]) | (u32)le16_to_cpu(addi_parcel[1]) << 16; + + lower_immediate = sign_extend32(val, 11); + upper_immediate = (val - lower_immediate); + + if (upper_immediate & 0xfffff000) { + /* replace upper 20 bits of lui with upper immediate */ + lui_insn &= 0x00000fff; + lui_insn |= upper_immediate & 0xfffff000; + } else { + /* replace lui with nop if immediate is small enough to fit in addi */ + lui_insn = RISCV_INSN_NOP4; + /* + * lui is being skipped, so do a load instead of an add. A load + * is performed by adding with the x0 register. Setting rs to + * zero with the following mask will accomplish this goal. + */ + addi_insn_mask &= 0x07fff; + } + + if (lower_immediate & 0x00000fff) { + /* replace upper 12 bits of addi with lower 12 bits of val */ + addi_insn &= addi_insn_mask; + addi_insn |= (lower_immediate & 0x00000fff) << 20; + } else { + /* replace addi with nop if lower_immediate is empty */ + addi_insn = RISCV_INSN_NOP4; + } + + addi_res = cpu_to_le32(addi_insn); + lui_res = cpu_to_le32(lui_insn); + mutex_lock(&text_mutex); + patch_insn_write(addi_parcel, &addi_res, sizeof(addi_res)); + patch_insn_write(lui_parcel, &lui_res, sizeof(lui_res)); + mutex_unlock(&text_mutex); +} + +static inline void __runtime_fixup_ptr(void *where, unsigned long val) +{ +#ifdef CONFIG_32BIT + __runtime_fixup_32(where, where + 4, val); + __runtime_fixup_caches(where, 2); +#else + __runtime_fixup_32(where, where + 8, val); + __runtime_fixup_32(where + 4, where + 12, val >> 32); + __runtime_fixup_caches(where, 4); +#endif +} + +/* + * Replace the least significant 5 bits of the srli/srliw immediate that is + * located at bits 20-24 + */ +static inline void __runtime_fixup_shift(void *where, unsigned long val) +{ + __le16 *parcel = where; + __le32 res; + u32 insn; + + insn = (u32)le16_to_cpu(parcel[0]) | (u32)le16_to_cpu(parcel[1]) << 16; + + insn &= 0xfe0fffff; + insn |= (val & 0b11111) << 20; + + res = cpu_to_le32(insn); + mutex_lock(&text_mutex); + patch_text_nosync(where, &res, sizeof(insn)); + mutex_unlock(&text_mutex); +} + +static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), + unsigned long val, s32 *start, s32 *end) +{ + while (start < end) { + fn(*start + (void *)start, val); + start++; + } +} + +#endif /* _ASM_RISCV_RUNTIME_CONST_H */ diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 002ca58dd998c..61bd5ba6680a7 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -97,6 +97,9 @@ SECTIONS { EXIT_DATA } + + RUNTIME_CONST_VARIABLES + PERCPU_SECTION(L1_CACHE_BYTES) .rel.dyn : { From 9324571e9eea231321acf0a3d0fbc85a6e0f6ff6 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 12 Mar 2025 13:11:44 +0000 Subject: [PATCH 45/65] RISC-V: add vector extension validation checks Using Clement's new validation callbacks, support checking that dependencies have been satisfied for the vector extensions. From the kernel's perfective, it's not required to differentiate between the conditions for all the various vector subsets - it's the firmware's job to not report impossible combinations. Instead, the kernel only has to check that the correct config options are enabled and to enforce its requirement of the d extension being present for FPU support. Since vector will now be disabled proactively, there's no need to clear the bit in elf_hwcap in riscv_fill_hwcap() any longer. Signed-off-by: Conor Dooley Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250312-eclair-affluent-55b098c3602b@spud Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/cpufeature.h | 3 ++ arch/riscv/kernel/cpufeature.c | 60 +++++++++++++++++++---------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 569140d6e6399..5d9427ccbc7a6 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -56,6 +56,9 @@ void __init riscv_user_isa_enable(void); #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \ ARRAY_SIZE(_bundled_exts), NULL) +#define __RISCV_ISA_EXT_BUNDLE_VALIDATE(_name, _bundled_exts, _validate) \ + _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \ + ARRAY_SIZE(_bundled_exts), _validate) /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 40ac72e407b68..76a3b34d7a707 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -109,6 +109,38 @@ static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, return 0; } +static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + return 0; +} + +static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + if (!IS_ENABLED(CONFIG_FPU)) + return -EINVAL; + + /* + * The kernel doesn't support systems that don't implement both of + * F and D, so if any of the vector extensions that do floating point + * are to be usable, both floating point extensions need to be usable. + * + * Since this function validates vector only, and v/Zve* are probed + * after f/d, there's no need for a deferral here. + */ + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + return -EINVAL; + + return 0; +} + static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -326,12 +358,10 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), - __RISCV_ISA_EXT_SUPERSET(v, RISCV_ISA_EXT_v, riscv_v_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate), __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), - __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, - riscv_ext_zicbom_validate), - __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, - riscv_ext_zicboz_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), @@ -372,11 +402,11 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), - __RISCV_ISA_EXT_SUPERSET(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts), - __RISCV_ISA_EXT_DATA(zve32x, RISCV_ISA_EXT_ZVE32X), - __RISCV_ISA_EXT_SUPERSET(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts), - __RISCV_ISA_EXT_SUPERSET(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts), - __RISCV_ISA_EXT_SUPERSET(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), @@ -960,16 +990,6 @@ void __init riscv_fill_hwcap(void) riscv_v_setup_vsize(); } - if (elf_hwcap & COMPAT_HWCAP_ISA_V) { - /* - * ISA string in device tree might have 'v' flag, but - * CONFIG_RISCV_ISA_V is disabled in kernel. - * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled. - */ - if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) - elf_hwcap &= ~COMPAT_HWCAP_ISA_V; - } - memset(print_str, 0, sizeof(print_str)); for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) if (riscv_isa[0] & BIT_MASK(i)) From 38077ec8fc11fa62e85a3a7630cfa9f2b8fd9f65 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 12 Mar 2025 13:11:45 +0000 Subject: [PATCH 46/65] RISC-V: add vector crypto extension validation checks Using Clement's new validation callbacks, support checking that dependencies have been satisfied for the vector crpyto extensions. Currently riscv_isa_extension_available() will return true on systems that support the extensions but vector itself has been disabled by the kernel, adding validation callbacks will prevent such a scenario from occuring and make the behaviour of the extension detection functions more consistent with user expectations - it's not expected to have to check for vector AND the specific crypto extension. The Unpriv spec states: | The Zvknhb and Zvbc Vector Crypto Extensions --and accordingly the | composite extensions Zvkn, Zvknc, Zvkng, and Zvksc-- require a Zve64x | base, or application ("V") base Vector Extension. All of the other | Vector Crypto Extensions can be built on any embedded (Zve*) or | application ("V") base Vector Extension. While this could be used as the basis for checking that the correct base for individual crypto extensions, but that's not really the kernel's job in my opinion and it is sufficient to leave that sort of precision to the dt-bindings. The kernel only needs to make sure that vector, in some form, is available. Link: https://github.com/riscv/riscv-isa-manual/blob/main/src/vector-crypto.adoc#extensions-overview Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20250312-entertain-shaking-b664142c2f99@spud Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/cpufeature.c | 49 +++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 76a3b34d7a707..654dd4cfc479a 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -141,6 +141,23 @@ static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data return 0; } +static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + /* + * It isn't the kernel's job to check that the binding is correct, so + * it should be enough to check that any of the vector extensions are + * enabled, which in-turn means that vector is usable in this kernel + */ + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X)) + return -EPROBE_DEFER; + + return 0; +} + static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -400,8 +417,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), - __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), - __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate), @@ -409,20 +426,20 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), - __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), - __RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG), - __RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts), - __RISCV_ISA_EXT_BUNDLE(zvknc, riscv_zvknc_bundled_exts), - __RISCV_ISA_EXT_DATA(zvkned, RISCV_ISA_EXT_ZVKNED), - __RISCV_ISA_EXT_BUNDLE(zvkng, riscv_zvkng_bundled_exts), - __RISCV_ISA_EXT_DATA(zvknha, RISCV_ISA_EXT_ZVKNHA), - __RISCV_ISA_EXT_DATA(zvknhb, RISCV_ISA_EXT_ZVKNHB), - __RISCV_ISA_EXT_BUNDLE(zvks, riscv_zvks_bundled_exts), - __RISCV_ISA_EXT_BUNDLE(zvksc, riscv_zvksc_bundled_exts), - __RISCV_ISA_EXT_DATA(zvksed, RISCV_ISA_EXT_ZVKSED), - __RISCV_ISA_EXT_DATA(zvksh, RISCV_ISA_EXT_ZVKSH), - __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts), - __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), From 12e7fbb6a84e6c90a61330d152319e870bc01c46 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 12 Mar 2025 13:11:46 +0000 Subject: [PATCH 47/65] RISC-V: add f & d extension validation checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using Clement's new validation callbacks, support checking that dependencies have been satisfied for the floating point extensions. The check for "d" might be slightly confusingly shorter than that of "f", despite "d" depending on "f". This is because the requirement that a hart supporting double precision must also support single precision, should be validated by dt-bindings etc, not the kernel but lack of support for single precision only is a limitation of the kernel. Tested-by: Clément Léger Reviewed-by: Clément Léger Signed-off-by: Conor Dooley Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250312-reptile-platinum-62ee0f444a32@spud Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/cpufeature.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 654dd4cfc479a..4024da3fc1ddf 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -109,6 +109,33 @@ static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, return 0; } +static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_FPU)) + return -EINVAL; + + /* + * Due to extension ordering, d is checked before f, so no deferral + * is required. + */ + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) { + pr_warn_once("This kernel does not support systems with F but not D\n"); + return -EINVAL; + } + + return 0; +} + +static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_FPU)) + return -EINVAL; + + return 0; +} + static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -371,8 +398,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), __RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a), - __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), - __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), + __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate), __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate), From 534d813a06202c565b4a7e75a3e710db7155e6d3 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 12 Mar 2025 13:11:47 +0000 Subject: [PATCH 48/65] dt-bindings: riscv: d requires f MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the specifications, the d extension for double-precision floating point operations depends on the f extension for single-precision floating point. Add that requirement to the bindings. This differs from the Linux implementation, where single-precious only is not supported. Reviewed-by: Clément Léger Acked-by: Krzysztof Kozlowski Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20250312-perpetual-daunting-ad489c9a857a@spud Signed-off-by: Alexandre Ghiti --- Documentation/devicetree/bindings/riscv/extensions.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index a63b994e07636..ebb252275ddd3 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -639,6 +639,12 @@ properties: https://github.com/T-head-Semi/thead-extension-spec/blob/95358cb2cca9489361c61d335e03d3134b14133f/xtheadvector.adoc. allOf: + - if: + contains: + const: d + then: + contains: + const: f # Zcb depends on Zca - if: contains: From e9f1d61a5e186092d3b8eaa411bb4a76622bf854 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 12 Mar 2025 13:11:48 +0000 Subject: [PATCH 49/65] dt-bindings: riscv: add vector sub-extension dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Section 33.18.2. Zve*: Vector Extensions for Embedded Processors in [1] says: | The Zve32f and Zve64x extensions depend on the Zve32x extension. The Zve64f extension depends | on the Zve32f and Zve64x extensions. The Zve64d extension depends on the Zve64f extension | The Zve32x extension depends on the Zicsr extension. The Zve32f and Zve64f extensions depend | upon the F extension | The Zve64d extension depends upon the D extension Apply these rules to the bindings to help prevent invalid combinations. Link: https://github.com/riscv/riscv-isa-manual/releases/tag/riscv-isa-release-698e64a-2024-09-09 [1] Reviewed-by: Clément Léger Acked-by: Krzysztof Kozlowski Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20250312-banking-crestless-58f3259a5018@spud Signed-off-by: Alexandre Ghiti --- .../devicetree/bindings/riscv/extensions.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index ebb252275ddd3..02065664f819e 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -680,6 +680,52 @@ properties: contains: const: zca + - if: + contains: + const: zve32x + then: + contains: + const: zicsr + + - if: + contains: + const: zve32f + then: + allOf: + - contains: + const: f + - contains: + const: zve32x + + - if: + contains: + const: zve64x + then: + contains: + const: zve32x + + - if: + contains: + const: zve64f + then: + allOf: + - contains: + const: f + - contains: + const: zve32f + - contains: + const: zve64x + + - if: + contains: + const: zve64d + then: + allOf: + - contains: + const: d + - contains: + const: zve64f + allOf: # Zcf extension does not exist on rv64 - if: From a0d857205756af45abaf63ca15b2640f707d5e73 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 12 Mar 2025 13:11:49 +0000 Subject: [PATCH 50/65] dt-bindings: riscv: document vector crypto requirements The Unpriv spec states: | The Zvknhb and Zvbc Vector Crypto Extensions --and accordingly the | composite extensions Zvkn, Zvknc, Zvkng, and Zvksc-- require a Zve64x | base, or application ("V") base Vector Extension. All of the other | Vector Crypto Extensions can be built on any embedded (Zve*) or | application ("V") base Vector Extension. Enforce the minimum requirement via schema. Link: https://github.com/riscv/riscv-isa-manual/blob/main/src/vector-crypto.adoc#extensions-overview Acked-by: Krzysztof Kozlowski Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20250312-flask-relay-b36ee622b2c8@spud Signed-off-by: Alexandre Ghiti --- .../devicetree/bindings/riscv/extensions.yaml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index 02065664f819e..9aeb9d4731cac 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -726,6 +726,39 @@ properties: - contains: const: zve64f + - if: + contains: + anyOf: + - const: zvbc + - const: zvkn + - const: zvknc + - const: zvkng + - const: zvknhb + - const: zvksc + then: + contains: + anyOf: + - const: v + - const: zve64x + + - if: + contains: + anyOf: + - const: zvbb + - const: zvkb + - const: zvkg + - const: zvkned + - const: zvknha + - const: zvksed + - const: zvksh + - const: zvks + - const: zvkt + then: + contains: + anyOf: + - const: v + - const: zve32x + allOf: # Zcf extension does not exist on rv64 - if: From bb58e1579f431d42469b6aed0f03eff383ba6db5 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Wed, 26 Mar 2025 15:45:07 -0700 Subject: [PATCH 51/65] RISC-V: errata: Use medany for relocatable builds We're trying to mix non-PIC/PIE objects into the otherwise-PIE relocatable kernels, to avoid GOT/PLT references during early boot alternative resolution (which happens before the GOT/PLT are set up). riscv64-unknown-linux-gnu-ld: arch/riscv/errata/sifive/errata.o: relocation R_RISCV_HI20 against `tlb_flush_all_threshold' can not be used when making a shared object; recompile with -fPIC riscv64-unknown-linux-gnu-ld: arch/riscv/errata/thead/errata.o: relocation R_RISCV_HI20 against `riscv_cbom_block_size' can not be used when making a shared object; recompile with -fPIC Fixes: 8dc2a7e8027f ("riscv: Fix relocatable kernels with early alternatives using -fno-pie") Link: https://lore.kernel.org/r/20250326224506.27165-2-palmer@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/errata/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/riscv/errata/Makefile b/arch/riscv/errata/Makefile index f0da9d7b39c37..bc6c77ba837d2 100644 --- a/arch/riscv/errata/Makefile +++ b/arch/riscv/errata/Makefile @@ -1,5 +1,9 @@ ifdef CONFIG_RELOCATABLE -KBUILD_CFLAGS += -fno-pie +# We can't use PIC/PIE when handling early-boot errata parsing, as the kernel +# doesn't have a GOT setup at that point. So instead just use medany: it's +# usually position-independent, so it should be good enough for the errata +# handling. +KBUILD_CFLAGS += -fno-pie -mcmodel=medany endif ifdef CONFIG_RISCV_ALTERNATIVE_EARLY From bffada8201fc9933ba0974b76b6068d6b4557ef4 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 26 Oct 2024 10:13:53 -0700 Subject: [PATCH 52/65] riscv: Remove duplicate CONFIG_PAGE_OFFSET definition This definition is already provided by include/generated/autoconf.h, so it does not need to be provided on the command line. Signed-off-by: Samuel Holland Reviewed-by: Jesse Taube Link: https://lore.kernel.org/r/20241026171441.3047904-2-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 13fbc0f942387..600df90bc141e 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -98,7 +98,6 @@ KBUILD_AFLAGS += -march=$(riscv-march-y) CC_FLAGS_FPU := -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)([^v_]*)v?/\1\2/') KBUILD_CFLAGS += -mno-save-restore -KBUILD_CFLAGS += -DCONFIG_PAGE_OFFSET=$(CONFIG_PAGE_OFFSET) ifeq ($(CONFIG_CMODEL_MEDLOW),y) KBUILD_CFLAGS += -mcmodel=medlow From 2c0391b29b27f315c1b4c29ffde66f50b29fab99 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 26 Oct 2024 10:13:54 -0700 Subject: [PATCH 53/65] riscv: Allow NOMMU kernels to access all of RAM NOMMU kernels currently cannot access memory below the kernel link address. Remove this restriction by setting PAGE_OFFSET to the actual start of RAM, as determined from the devicetree. The kernel link address must be a constant, so keep using CONFIG_PAGE_OFFSET for that purpose. Signed-off-by: Samuel Holland Reviewed-by: Jesse Taube Link: https://lore.kernel.org/r/20241026171441.3047904-3-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/page.h | 12 ++++-------- arch/riscv/include/asm/pgtable.h | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 125f5ecd95652..6409fd78ae6fa 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -24,12 +24,9 @@ * When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#ifdef CONFIG_64BIT #ifdef CONFIG_MMU +#ifdef CONFIG_64BIT #define PAGE_OFFSET kernel_map.page_offset -#else -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) -#endif /* * By default, CONFIG_PAGE_OFFSET value corresponds to SV57 address space so * define the PAGE_OFFSET value for SV48 and SV39. @@ -39,6 +36,9 @@ #else #define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) #endif /* CONFIG_64BIT */ +#else +#define PAGE_OFFSET ((unsigned long)phys_ram_base) +#endif /* CONFIG_MMU */ #ifndef __ASSEMBLY__ @@ -95,11 +95,7 @@ typedef struct page *pgtable_t; #define MIN_MEMBLOCK_ADDR 0 #endif -#ifdef CONFIG_MMU #define ARCH_PFN_OFFSET (PFN_DOWN((unsigned long)phys_ram_base)) -#else -#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) -#endif /* CONFIG_MMU */ struct kernel_mapping { unsigned long page_offset; diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 050fdc49b5ad7..eb7b25ef556ec 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -12,7 +12,7 @@ #include #ifndef CONFIG_MMU -#define KERNEL_LINK_ADDR PAGE_OFFSET +#define KERNEL_LINK_ADDR _AC(CONFIG_PAGE_OFFSET, UL) #define KERN_VIRT_SIZE (UL(-1)) #else From 51b766c79a3d741fb97419c3da1c58fce5e66f0e Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 26 Oct 2024 10:13:55 -0700 Subject: [PATCH 54/65] riscv: Support CONFIG_RELOCATABLE on NOMMU Move relocate_kernel() out of the CONFIG_MMU block so it can be called from the NOMMU version of setup_vm(). Set some offsets in kernel_map so relocate_kernel() does not need to be modified. Relocatable NOMMU kernels can be loaded to any physical memory address; they no longer depend on CONFIG_PAGE_OFFSET. Signed-off-by: Samuel Holland Link: https://lore.kernel.org/r/20241026171441.3047904-4-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 2 +- arch/riscv/include/asm/pgtable.h | 4 ++ arch/riscv/mm/init.c | 82 +++++++++++++++++--------------- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7612c52e9b1e3..62fb6d9560e5d 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -1075,7 +1075,7 @@ config PARAVIRT_TIME_ACCOUNTING config RELOCATABLE bool "Build a relocatable kernel" - depends on MMU && 64BIT && !XIP_KERNEL + depends on 64BIT && !XIP_KERNEL select MODULE_SECTIONS if MODULES help This builds a kernel as a Position Independent Executable (PIE), diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index eb7b25ef556ec..babd951222db1 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -12,7 +12,11 @@ #include #ifndef CONFIG_MMU +#ifdef CONFIG_RELOCATABLE +#define KERNEL_LINK_ADDR UL(0) +#else #define KERNEL_LINK_ADDR _AC(CONFIG_PAGE_OFFSET, UL) +#endif #define KERN_VIRT_SIZE (UL(-1)) #else diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 15b2eda4c364b..82d14f94c996c 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -323,6 +323,44 @@ static void __init setup_bootmem(void) hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); } +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +static void __init relocate_kernel(void) +{ + Elf64_Rela *rela = (Elf64_Rela *)&__rela_dyn_start; + /* + * This holds the offset between the linked virtual address and the + * relocated virtual address. + */ + uintptr_t reloc_offset = kernel_map.virt_addr - KERNEL_LINK_ADDR; + /* + * This holds the offset between kernel linked virtual address and + * physical address. + */ + uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr; + + for ( ; rela < (Elf64_Rela *)&__rela_dyn_end; rela++) { + Elf64_Addr addr = (rela->r_offset - va_kernel_link_pa_offset); + Elf64_Addr relocated_addr = rela->r_addend; + + if (rela->r_info != R_RISCV_RELATIVE) + continue; + + /* + * Make sure to not relocate vdso symbols like rt_sigreturn + * which are linked from the address 0 in vmlinux since + * vdso symbol addresses are actually used as an offset from + * mm->context.vdso in VDSO_OFFSET macro. + */ + if (relocated_addr >= KERNEL_LINK_ADDR) + relocated_addr += reloc_offset; + + *(Elf64_Addr *)addr = relocated_addr; + } +} +#endif /* CONFIG_RELOCATABLE */ + #ifdef CONFIG_MMU struct pt_alloc_ops pt_ops __meminitdata; @@ -893,44 +931,6 @@ static __init void set_satp_mode(uintptr_t dtb_pa) #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." #endif -#ifdef CONFIG_RELOCATABLE -extern unsigned long __rela_dyn_start, __rela_dyn_end; - -static void __init relocate_kernel(void) -{ - Elf64_Rela *rela = (Elf64_Rela *)&__rela_dyn_start; - /* - * This holds the offset between the linked virtual address and the - * relocated virtual address. - */ - uintptr_t reloc_offset = kernel_map.virt_addr - KERNEL_LINK_ADDR; - /* - * This holds the offset between kernel linked virtual address and - * physical address. - */ - uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr; - - for ( ; rela < (Elf64_Rela *)&__rela_dyn_end; rela++) { - Elf64_Addr addr = (rela->r_offset - va_kernel_link_pa_offset); - Elf64_Addr relocated_addr = rela->r_addend; - - if (rela->r_info != R_RISCV_RELATIVE) - continue; - - /* - * Make sure to not relocate vdso symbols like rt_sigreturn - * which are linked from the address 0 in vmlinux since - * vdso symbol addresses are actually used as an offset from - * mm->context.vdso in VDSO_OFFSET macro. - */ - if (relocated_addr >= KERNEL_LINK_ADDR) - relocated_addr += reloc_offset; - - *(Elf64_Addr *)addr = relocated_addr; - } -} -#endif /* CONFIG_RELOCATABLE */ - #ifdef CONFIG_XIP_KERNEL static void __init create_kernel_page_table(pgd_t *pgdir, __always_unused bool early) @@ -1378,6 +1378,12 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) { dtb_early_va = (void *)dtb_pa; dtb_early_pa = dtb_pa; + +#ifdef CONFIG_RELOCATABLE + kernel_map.virt_addr = (uintptr_t)_start; + kernel_map.phys_addr = (uintptr_t)_start; + relocate_kernel(); +#endif } static inline void setup_vm_final(void) From d073a571e68f42414f8f06f01b59f52224538a83 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 26 Oct 2024 10:13:56 -0700 Subject: [PATCH 55/65] asm-generic: Always define Elf_Rel and Elf_Rela These definitions are useful for relocating the kernel image as well, regardless of the type of relocations used for modules. Signed-off-by: Samuel Holland Link: https://lore.kernel.org/r/20241026171441.3047904-5-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- include/asm-generic/module.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/include/asm-generic/module.h b/include/asm-generic/module.h index 98e1541b72b70..a8622501b9756 100644 --- a/include/asm-generic/module.h +++ b/include/asm-generic/module.h @@ -19,12 +19,8 @@ struct mod_arch_specific #define Elf_Dyn Elf64_Dyn #define Elf_Ehdr Elf64_Ehdr #define Elf_Addr Elf64_Addr -#ifdef CONFIG_MODULES_USE_ELF_REL #define Elf_Rel Elf64_Rel -#endif -#ifdef CONFIG_MODULES_USE_ELF_RELA #define Elf_Rela Elf64_Rela -#endif #define ELF_R_TYPE(X) ELF64_R_TYPE(X) #define ELF_R_SYM(X) ELF64_R_SYM(X) @@ -36,12 +32,8 @@ struct mod_arch_specific #define Elf_Dyn Elf32_Dyn #define Elf_Ehdr Elf32_Ehdr #define Elf_Addr Elf32_Addr -#ifdef CONFIG_MODULES_USE_ELF_REL #define Elf_Rel Elf32_Rel -#endif -#ifdef CONFIG_MODULES_USE_ELF_RELA #define Elf_Rela Elf32_Rela -#endif #define ELF_R_TYPE(X) ELF32_R_TYPE(X) #define ELF_R_SYM(X) ELF32_R_SYM(X) #endif From ea2bde36a46d5724c1b44d80cc9fafbd73c2ecf9 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 26 Oct 2024 10:13:57 -0700 Subject: [PATCH 56/65] riscv: Support CONFIG_RELOCATABLE on riscv32 When adjusted to use the correctly-sized ELF types, relocate_kernel() works on riscv32 as well. The caveat about crossing an intermediate page table boundary does not apply to riscv32, since for Sv32 the early kernel mapping uses only PGD entries. Since KASLR is not yet supported on riscv32, this option is mostly useful for NOMMU. Signed-off-by: Samuel Holland Link: https://lore.kernel.org/r/20241026171441.3047904-6-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 2 +- arch/riscv/mm/init.c | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 62fb6d9560e5d..7d3d457045eec 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -1075,7 +1075,7 @@ config PARAVIRT_TIME_ACCOUNTING config RELOCATABLE bool "Build a relocatable kernel" - depends on 64BIT && !XIP_KERNEL + depends on !XIP_KERNEL select MODULE_SECTIONS if MODULES help This builds a kernel as a Position Independent Executable (PIE), diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 82d14f94c996c..d1494d48a70d6 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -20,15 +20,13 @@ #include #include #include -#ifdef CONFIG_RELOCATABLE -#include -#endif #include #include #include #include #include +#include #include #include #include @@ -328,7 +326,7 @@ extern unsigned long __rela_dyn_start, __rela_dyn_end; static void __init relocate_kernel(void) { - Elf64_Rela *rela = (Elf64_Rela *)&__rela_dyn_start; + Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; /* * This holds the offset between the linked virtual address and the * relocated virtual address. @@ -340,9 +338,9 @@ static void __init relocate_kernel(void) */ uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr; - for ( ; rela < (Elf64_Rela *)&__rela_dyn_end; rela++) { - Elf64_Addr addr = (rela->r_offset - va_kernel_link_pa_offset); - Elf64_Addr relocated_addr = rela->r_addend; + for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { + Elf_Addr addr = (rela->r_offset - va_kernel_link_pa_offset); + Elf_Addr relocated_addr = rela->r_addend; if (rela->r_info != R_RISCV_RELATIVE) continue; @@ -356,7 +354,7 @@ static void __init relocate_kernel(void) if (relocated_addr >= KERNEL_LINK_ADDR) relocated_addr += reloc_offset; - *(Elf64_Addr *)addr = relocated_addr; + *(Elf_Addr *)addr = relocated_addr; } } #endif /* CONFIG_RELOCATABLE */ @@ -1174,7 +1172,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) * makes the kernel cross over a PUD_SIZE boundary, raise a bug * since a part of the kernel would not get mapped. */ - BUG_ON(PUD_SIZE - (kernel_map.virt_addr & (PUD_SIZE - 1)) < kernel_map.size); + if (IS_ENABLED(CONFIG_64BIT)) + BUG_ON(PUD_SIZE - (kernel_map.virt_addr & (PUD_SIZE - 1)) < kernel_map.size); relocate_kernel(); #endif From e1cf2d009b00fd890dbbcb8b79613ff538732559 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 26 Oct 2024 10:13:58 -0700 Subject: [PATCH 57/65] riscv: Remove CONFIG_PAGE_OFFSET The current definition of CONFIG_PAGE_OFFSET is problematic for a couple of reasons: 1) The value is misleading for normal 64-bit kernels, where it is overridden at runtime if Sv48 or Sv39 is chosen. This is especially the case for XIP kernels, which always use Sv39. 2) The option is not user-visible, but for NOMMU kernels it must be a valid RAM address, and for !RELOCATABLE it must additionally be the exact address where the kernel is loaded. Fix both of these by removing the option. 1) For MMU kernels, drop the indirection through Kconfig. Additionally, for XIP, drop the indirection through kernel_map. 2) For NOMMU kernels, use the user-visible physical RAM base if provided. Otherwise, force the kernel to be relocatable. Signed-off-by: Samuel Holland Reviewed-by: Jesse Taube Link: https://lore.kernel.org/r/20241026171441.3047904-7-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 8 +------- arch/riscv/include/asm/page.h | 15 ++++++++------- arch/riscv/include/asm/pgtable.h | 2 +- arch/riscv/mm/init.c | 8 ++------ 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7d3d457045eec..551b9a9b243fd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -201,6 +201,7 @@ config RISCV select PCI_DOMAINS_GENERIC if PCI select PCI_ECAM if (ACPI && PCI) select PCI_MSI if PCI + select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED select RISCV_ALTERNATIVE if !XIP_KERNEL select RISCV_APLIC select RISCV_IMSIC @@ -288,13 +289,6 @@ config MMU Select if you want MMU-based virtualised addressing space support by paged memory management. If unsure, say 'Y'. -config PAGE_OFFSET - hex - default 0x80000000 if !MMU && RISCV_M_MODE - default 0x80200000 if !MMU - default 0xc0000000 if 32BIT - default 0xff60000000000000 if 64BIT - config KASAN_SHADOW_OFFSET hex depends on KASAN_GENERIC diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 6409fd78ae6fa..572a141ddecdb 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -26,15 +26,16 @@ */ #ifdef CONFIG_MMU #ifdef CONFIG_64BIT -#define PAGE_OFFSET kernel_map.page_offset -/* - * By default, CONFIG_PAGE_OFFSET value corresponds to SV57 address space so - * define the PAGE_OFFSET value for SV48 and SV39. - */ +#define PAGE_OFFSET_L5 _AC(0xff60000000000000, UL) #define PAGE_OFFSET_L4 _AC(0xffffaf8000000000, UL) #define PAGE_OFFSET_L3 _AC(0xffffffd600000000, UL) +#ifdef CONFIG_XIP_KERNEL +#define PAGE_OFFSET PAGE_OFFSET_L3 #else -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_map.page_offset +#endif /* CONFIG_XIP_KERNEL */ +#else +#define PAGE_OFFSET _AC(0xc0000000, UL) #endif /* CONFIG_64BIT */ #else #define PAGE_OFFSET ((unsigned long)phys_ram_base) @@ -98,7 +99,6 @@ typedef struct page *pgtable_t; #define ARCH_PFN_OFFSET (PFN_DOWN((unsigned long)phys_ram_base)) struct kernel_mapping { - unsigned long page_offset; unsigned long virt_addr; unsigned long virt_offset; uintptr_t phys_addr; @@ -112,6 +112,7 @@ struct kernel_mapping { uintptr_t xiprom; uintptr_t xiprom_sz; #else + unsigned long page_offset; unsigned long va_kernel_pa_offset; #endif }; diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index babd951222db1..ddebdfb4519ad 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -15,7 +15,7 @@ #ifdef CONFIG_RELOCATABLE #define KERNEL_LINK_ADDR UL(0) #else -#define KERNEL_LINK_ADDR _AC(CONFIG_PAGE_OFFSET, UL) +#define KERNEL_LINK_ADDR _AC(CONFIG_PHYS_RAM_BASE, UL) #endif #define KERN_VIRT_SIZE (UL(-1)) #else diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index d1494d48a70d6..37af6513a50f5 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -859,6 +859,8 @@ static __init void set_satp_mode(uintptr_t dtb_pa) uintptr_t set_satp_mode_pmd = ((unsigned long)set_satp_mode) & PMD_MASK; u64 satp_mode_cmdline = __pi_set_satp_mode_from_cmdline(dtb_pa); + kernel_map.page_offset = PAGE_OFFSET_L5; + if (satp_mode_cmdline == SATP_MODE_57) { disable_pgtable_l5(); } else if (satp_mode_cmdline == SATP_MODE_48) { @@ -1106,11 +1108,6 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; #ifdef CONFIG_XIP_KERNEL -#ifdef CONFIG_64BIT - kernel_map.page_offset = PAGE_OFFSET_L3; -#else - kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); -#endif kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); @@ -1125,7 +1122,6 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.va_kernel_xip_data_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr + (uintptr_t)&_sdata - (uintptr_t)&_start; #else - kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); kernel_map.phys_addr = (uintptr_t)(&_start); kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; From 95c18b7ccdd1b2e6704651b66565339ada318ba2 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Mon, 31 Mar 2025 11:45:24 -0700 Subject: [PATCH 58/65] riscv: Add norvc after .option arch in runtime const .option arch clobbers .option norvc. Prevent gas from emitting compressed instructions in the runtime const alternative blocks by setting .option norvc after .option arch. This issue starts appearing on gcc 15, which adds zca to the march. Reported by: Klara Modin Signed-off-by: Charlie Jenkins Fixes: a44fb5722199 ("riscv: Add runtime constant support") Closes: https://lore.kernel.org/all/cc8f3525-20b7-445b-877b-2add28a160a2@gmail.com/ Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20250331-fix_runtime_const_norvc-v1-1-89bc62687ab8@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/runtime-const.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index ea2e49c7149cd..1ba0985a7ffea 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -56,6 +56,7 @@ #define RISCV_RUNTIME_CONST_64_ZBA \ ".option push\n\t" \ ".option arch,+zba\n\t" \ + ".option norvc\n\t" \ "slli %[__tmp],%[__tmp],32\n\t" \ "add.uw %[__ret],%[__ret],%[__tmp]\n\t" \ "nop\n\t" \ @@ -65,6 +66,7 @@ #define RISCV_RUNTIME_CONST_64_ZBKB \ ".option push\n\t" \ ".option arch,+zbkb\n\t" \ + ".option norvc\n\t" \ "pack %[__ret],%[__ret],%[__tmp]\n\t" \ "nop\n\t" \ "nop\n\t" \ From a3313375e88e0075b9048eba15e5eb4dbf93f60e Mon Sep 17 00:00:00 2001 From: Yunhui Cui Date: Mon, 3 Mar 2025 16:34:24 +0800 Subject: [PATCH 59/65] riscv: print hartid on bringup Firmware randomly releases cores, so CPU numbers don't linearly map to hartids. When the system has an exception, we care more about hartids. Adding "dyndbg="file smpboot.c +p" loglevel=8" to the cmdline can output the hartid. Signed-off-by: Yunhui Cui Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250303083424.14309-1-cuiyunhui@bytedance.com Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/smp.c | 2 ++ arch/riscv/kernel/smpboot.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index d58b5e7512861..e650dec448176 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -48,6 +48,8 @@ EXPORT_SYMBOL_GPL(__cpuid_to_hartid_map); void __init smp_setup_processor_id(void) { cpuid_to_hartid_map(0) = boot_cpu_hartid; + + pr_info("Booting Linux on hartid %lu\n", boot_cpu_hartid); } static DEFINE_PER_CPU_READ_MOSTLY(int, ipi_dummy_dev); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index e36d20205bd7d..601a321e0f176 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -231,6 +231,10 @@ asmlinkage __visible void smp_callin(void) riscv_ipi_enable(); numa_add_cpu(curr_cpuid); + + pr_debug("CPU%u: Booted secondary hartid %lu\n", curr_cpuid, + cpuid_to_hartid_map(curr_cpuid)); + set_cpu_online(curr_cpuid, true); /* From 83d78ac677b9fdd8ea763507c6fe02d6bf415f3a Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Mon, 17 Mar 2025 08:25:51 +0100 Subject: [PATCH 60/65] riscv: Fix hugetlb retrieval of number of ptes in case of !present pte Ryan sent a fix [1] for arm64 that applies to riscv too: in some hugetlb functions, we must not use the pte value to get the size of a mapping because the pte may not be present. So use the already present size parameter for huge_pte_clear() and the newly introduced size parameter for huge_ptep_get_and_clear(). And make sure to gather A/D bits only on present ptes. Fixes: 82a1a1f3bfb6 ("riscv: mm: support Svnapot in hugetlb page") Link: https://lore.kernel.org/all/20250217140419.1702389-1-ryan.roberts@arm.com/ [1] Link: https://lore.kernel.org/r/20250317072551.572169-1-alexghiti@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/mm/hugetlbpage.c | 76 ++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index b4a78a4b35cff..375dd96bb4a0d 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -148,22 +148,25 @@ unsigned long hugetlb_mask_last_page(struct hstate *h) static pte_t get_clear_contig(struct mm_struct *mm, unsigned long addr, pte_t *ptep, - unsigned long pte_num) + unsigned long ncontig) { - pte_t orig_pte = ptep_get(ptep); - unsigned long i; - - for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) { - pte_t pte = ptep_get_and_clear(mm, addr, ptep); - - if (pte_dirty(pte)) - orig_pte = pte_mkdirty(orig_pte); - - if (pte_young(pte)) - orig_pte = pte_mkyoung(orig_pte); + pte_t pte, tmp_pte; + bool present; + + pte = ptep_get_and_clear(mm, addr, ptep); + present = pte_present(pte); + while (--ncontig) { + ptep++; + addr += PAGE_SIZE; + tmp_pte = ptep_get_and_clear(mm, addr, ptep); + if (present) { + if (pte_dirty(tmp_pte)) + pte = pte_mkdirty(pte); + if (pte_young(tmp_pte)) + pte = pte_mkyoung(pte); + } } - - return orig_pte; + return pte; } static pte_t get_clear_contig_flush(struct mm_struct *mm, @@ -212,6 +215,26 @@ static void clear_flush(struct mm_struct *mm, flush_tlb_range(&vma, saddr, addr); } +static int num_contig_ptes_from_size(unsigned long sz, size_t *pgsize) +{ + unsigned long hugepage_shift; + + if (sz >= PGDIR_SIZE) + hugepage_shift = PGDIR_SHIFT; + else if (sz >= P4D_SIZE) + hugepage_shift = P4D_SHIFT; + else if (sz >= PUD_SIZE) + hugepage_shift = PUD_SHIFT; + else if (sz >= PMD_SIZE) + hugepage_shift = PMD_SHIFT; + else + hugepage_shift = PAGE_SHIFT; + + *pgsize = 1 << hugepage_shift; + + return sz >> hugepage_shift; +} + /* * When dealing with NAPOT mappings, the privileged specification indicates that * "if an update needs to be made, the OS generally should first mark all of the @@ -226,22 +249,10 @@ void set_huge_pte_at(struct mm_struct *mm, pte_t pte, unsigned long sz) { - unsigned long hugepage_shift, pgsize; + size_t pgsize; int i, pte_num; - if (sz >= PGDIR_SIZE) - hugepage_shift = PGDIR_SHIFT; - else if (sz >= P4D_SIZE) - hugepage_shift = P4D_SHIFT; - else if (sz >= PUD_SIZE) - hugepage_shift = PUD_SHIFT; - else if (sz >= PMD_SIZE) - hugepage_shift = PMD_SHIFT; - else - hugepage_shift = PAGE_SHIFT; - - pte_num = sz >> hugepage_shift; - pgsize = 1 << hugepage_shift; + pte_num = num_contig_ptes_from_size(sz, &pgsize); if (!pte_present(pte)) { for (i = 0; i < pte_num; i++, ptep++, addr += pgsize) @@ -295,13 +306,14 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long sz) { + size_t pgsize; pte_t orig_pte = ptep_get(ptep); int pte_num; if (!pte_napot(orig_pte)) return ptep_get_and_clear(mm, addr, ptep); - pte_num = napot_pte_num(napot_cont_order(orig_pte)); + pte_num = num_contig_ptes_from_size(sz, &pgsize); return get_clear_contig(mm, addr, ptep, pte_num); } @@ -351,6 +363,7 @@ void huge_pte_clear(struct mm_struct *mm, pte_t *ptep, unsigned long sz) { + size_t pgsize; pte_t pte = ptep_get(ptep); int i, pte_num; @@ -359,8 +372,9 @@ void huge_pte_clear(struct mm_struct *mm, return; } - pte_num = napot_pte_num(napot_cont_order(pte)); - for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) + pte_num = num_contig_ptes_from_size(sz, &pgsize); + + for (i = 0; i < pte_num; i++, addr += pgsize, ptep++) pte_clear(mm, addr, ptep); } From 79ba5c1c7767a539f42c6f6db46961b0bec2bc03 Mon Sep 17 00:00:00 2001 From: Ignacio Encinas Date: Thu, 6 Mar 2025 20:49:27 +0100 Subject: [PATCH 61/65] selftests: riscv: fix v_exec_initval_nolibc.c Vector registers are zero initialized by the kernel. Stop accepting "all ones" as a clean value. Note that this was not working as expected given that value == 0xff can be assumed to be always false by the compiler as value's range is [-128, 127]. Both GCC (-Wtype-limits) and clang (-Wtautological-constant-out-of-range-compare) warn about this. Reviewed-by: Charlie Jenkins Tested-by: Charlie Jenkins Signed-off-by: Ignacio Encinas Link: https://lore.kernel.org/r/20250306-fix-v_exec_initval_nolibc-v2-1-97f9dc8a7faf@iencinas.com Signed-off-by: Alexandre Ghiti --- .../selftests/riscv/vector/v_exec_initval_nolibc.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/riscv/vector/v_exec_initval_nolibc.c b/tools/testing/selftests/riscv/vector/v_exec_initval_nolibc.c index 35c0812e32de0..4dde05e45a041 100644 --- a/tools/testing/selftests/riscv/vector/v_exec_initval_nolibc.c +++ b/tools/testing/selftests/riscv/vector/v_exec_initval_nolibc.c @@ -6,7 +6,7 @@ * the values. To further ensure consistency, this file is compiled without * libc and without auto-vectorization. * - * To be "clean" all values must be either all ones or all zeroes. + * To be "clean" all values must be all zeroes. */ #define __stringify_1(x...) #x @@ -14,9 +14,8 @@ int main(int argc, char **argv) { - char prev_value = 0, value; + char value = 0; unsigned long vl; - int first = 1; if (argc > 2 && strcmp(argv[2], "x")) asm volatile ( @@ -44,14 +43,11 @@ int main(int argc, char **argv) "vsrl.vi " __stringify(register) ", " __stringify(register) ", 8\n\t" \ ".option pop\n\t" \ : "=r" (value)); \ - if (first) { \ - first = 0; \ - } else if (value != prev_value || !(value == 0x00 || value == 0xff)) { \ + if (value != 0x00) { \ printf("Register " __stringify(register) \ " values not clean! value: %u\n", value); \ exit(-1); \ } \ - prev_value = value; \ } \ }) From 28093cfef5dd62f4cbd537f2bdf6f0bf85309c45 Mon Sep 17 00:00:00 2001 From: Yao Zi Date: Wed, 26 Mar 2025 05:14:46 +0000 Subject: [PATCH 62/65] riscv/kexec_file: Handle R_RISCV_64 in purgatory relocator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 58ff537109ac ("riscv: Omit optimized string routines when using KASAN") introduced calls to EXPORT_SYMBOL() in assembly string routines, which result in R_RISCV_64 relocations against .export_symbol section. As these rountines are reused by RISC-V purgatory and our relocator doesn't recognize these relocations, this fails kexec-file-load with dmesg like [ 11.344251] kexec_image: Unknown rela relocation: 2 [ 11.345972] kexec_image: Error loading purgatory ret=-8 Let's support R_RISCV_64 relocation to fix kexec on 64-bit RISC-V. 32-bit variant isn't covered since KEXEC_FILE and KEXEC_PURGATORY isn't available. Fixes: 58ff537109ac ("riscv: Omit optimized string routines when using KASAN") Signed-off-by: Yao Zi Tested-by: Björn Töpel Reviewed-by: Björn Töpel Link: https://lore.kernel.org/r/20250326051445.55131-2-ziyao@disroot.org Signed-off-by: Alexandre Ghiti --- arch/riscv/kernel/elf_kexec.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/kernel/elf_kexec.c b/arch/riscv/kernel/elf_kexec.c index 3c37661801f95..e783a72d051f4 100644 --- a/arch/riscv/kernel/elf_kexec.c +++ b/arch/riscv/kernel/elf_kexec.c @@ -468,6 +468,9 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, case R_RISCV_ALIGN: case R_RISCV_RELAX: break; + case R_RISCV_64: + *(u64 *)loc = val; + break; default: pr_err("Unknown rela relocation: %d\n", r_type); return -ENOEXEC; From 3f7023171df43641a8a8a1c9a12124501e589010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= Date: Fri, 28 Mar 2025 09:53:11 +0100 Subject: [PATCH 63/65] riscv/purgatory: 4B align purgatory_start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a crashkernel is launched on RISC-V, the entry to purgatory is done by trapping via the stvec CSR. From riscv_kexec_norelocate(): | ... | /* | * Switch to physical addressing | * This will also trigger a jump to CSR_STVEC | * which in this case is the address of the new | * kernel. | */ | csrw CSR_STVEC, a2 | csrw CSR_SATP, zero stvec requires that the address is 4B aligned, which was not the case, e.g.: | Loaded purgatory at 0xffffc000 | kexec_file: kexec_file_load: type:1, start:0xffffd232 head:0x4 flags:0x6 The address 0xffffd232 not 4B aligned. Correct by adding proper function alignment. With this change, crashkernels loaded with kexec-file will be able to properly enter the purgatory. Fixes: 736e30af583fb ("RISC-V: Add purgatory") Signed-off-by: Björn Töpel Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20250328085313.1193815-1-bjorn@kernel.org Signed-off-by: Alexandre Ghiti --- arch/riscv/purgatory/entry.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/purgatory/entry.S b/arch/riscv/purgatory/entry.S index 0e6ca6d5ae4b4..c5db2f072c341 100644 --- a/arch/riscv/purgatory/entry.S +++ b/arch/riscv/purgatory/entry.S @@ -12,6 +12,7 @@ .text +.align 2 SYM_CODE_START(purgatory_start) lla sp, .Lstack From 8a2f20ac8e14c1dd29d3214016a27e244f266b39 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Fri, 28 Mar 2025 12:54:22 +0100 Subject: [PATCH 64/65] riscv: Make sure toolchain supports zba before using zba instructions Old toolchain like gcc 8.5.0 does not support zba, so we must check that the toolchain supports this extension before using it in the kernel. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202503281836.8pntHm6I-lkp@intel.com/ Link: https://lore.kernel.org/r/20250328115422.253670-1-alexghiti@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/Kconfig | 8 ++++++++ arch/riscv/include/asm/runtime-const.h | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 0d8def968a7ef..ae6303f15b287 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -735,6 +735,14 @@ config TOOLCHAIN_HAS_VECTOR_CRYPTO def_bool $(as-instr, .option arch$(comma) +v$(comma) +zvkb) depends on AS_HAS_OPTION_ARCH +config TOOLCHAIN_HAS_ZBA + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zba) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zba) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_HAS_OPTION_ARCH + config RISCV_ISA_ZBA bool "Zba extension support for bit manipulation instructions" default y diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index ea2e49c7149cd..c07d049fdd5d2 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -77,7 +77,8 @@ ".long 1b - .\n\t" \ ".popsection" \ -#if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_RISCV_ISA_ZBKB) +#if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) \ + && defined(CONFIG_RISCV_ISA_ZBKB) #define runtime_const_ptr(sym) \ ({ \ typeof(sym) __ret, __tmp; \ @@ -93,7 +94,7 @@ : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ __ret; \ }) -#elif defined(CONFIG_RISCV_ISA_ZBA) +#elif defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) #define runtime_const_ptr(sym) \ ({ \ typeof(sym) __ret, __tmp; \ From 6ee928185aeb0ff085c73ae2ee163d436eba8352 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Mon, 31 Mar 2025 11:45:24 -0700 Subject: [PATCH 65/65] riscv: Add norvc after .option arch in runtime const .option arch clobbers .option norvc. Prevent gas from emitting compressed instructions in the runtime const alternative blocks by setting .option norvc after .option arch. This issue starts appearing on gcc 15, which adds zca to the march. Reported by: Klara Modin Signed-off-by: Charlie Jenkins Fixes: a44fb5722199 ("riscv: Add runtime constant support") Closes: https://lore.kernel.org/all/cc8f3525-20b7-445b-877b-2add28a160a2@gmail.com/ Tested-by: Klara Modin Link: https://lore.kernel.org/r/20250331-fix_runtime_const_norvc-v1-1-89bc62687ab8@rivosinc.com Signed-off-by: Alexandre Ghiti --- arch/riscv/include/asm/runtime-const.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index c07d049fdd5d2..451fd76b88115 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -56,6 +56,7 @@ #define RISCV_RUNTIME_CONST_64_ZBA \ ".option push\n\t" \ ".option arch,+zba\n\t" \ + ".option norvc\n\t" \ "slli %[__tmp],%[__tmp],32\n\t" \ "add.uw %[__ret],%[__ret],%[__tmp]\n\t" \ "nop\n\t" \ @@ -65,6 +66,7 @@ #define RISCV_RUNTIME_CONST_64_ZBKB \ ".option push\n\t" \ ".option arch,+zbkb\n\t" \ + ".option norvc\n\t" \ "pack %[__ret],%[__ret],%[__tmp]\n\t" \ "nop\n\t" \ "nop\n\t" \