From: Will Deacon Date: Mon, 25 Jul 2022 09:59:15 +0000 (+0100) Subject: Merge branch 'for-next/boot' into for-next/core X-Git-Tag: v6.0-rc1~203^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f96d67a8af7a39f7ffaac464d8bccc4c720e52c2;p=thirdparty%2Flinux.git Merge branch 'for-next/boot' into for-next/core * for-next/boot: (34 commits) arm64: fix KASAN_INLINE arm64: Add an override for ID_AA64SMFR0_EL1.FA64 arm64: Add the arm64.nosve command line option arm64: Add the arm64.nosme command line option arm64: Expose a __check_override primitive for oddball features arm64: Allow the idreg override to deal with variable field width arm64: Factor out checking of a feature against the override into a macro arm64: Allow sticky E2H when entering EL1 arm64: Save state of HCR_EL2.E2H before switch to EL1 arm64: Rename the VHE switch to "finalise_el2" arm64: mm: fix booting with 52-bit address space arm64: head: remove __PHYS_OFFSET arm64: lds: use PROVIDE instead of conditional definitions arm64: setup: drop early FDT pointer helpers arm64: head: avoid relocating the kernel twice for KASLR arm64: kaslr: defer initialization to initcall where permitted arm64: head: record CPU boot mode after enabling the MMU arm64: head: populate kernel page tables with MMU and caches on arm64: head: factor out TTBR1 assignment into a macro arm64: idreg-override: use early FDT mapping in ID map ... --- f96d67a8af7a39f7ffaac464d8bccc4c720e52c2 diff --cc arch/arm64/include/asm/assembler.h index dc422fa437c2f,b2584709c3324..5846145be523c --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@@ -462,9 -476,21 +476,21 @@@ alternative_endi dsb ish isb - _cond_extable .Licache_op\@, \fixup + _cond_uaccess_extable .Licache_op\@, \fixup .endm + /* + * load_ttbr1 - install @pgtbl as a TTBR1 page table + * pgtbl preserved + * tmp1/tmp2 clobbered, either may overlap with pgtbl + */ + .macro load_ttbr1, pgtbl, tmp1, tmp2 + phys_to_ttbr \tmp1, \pgtbl + offset_ttbr1 \tmp1, \tmp2 + msr ttbr1_el1, \tmp1 + isb + .endm + /* * To prevent the possibility of old and new partial table walks being visible * in the tlb, switch the ttbr to a zero page when we invalidate the old diff --cc arch/arm64/kernel/hyp-stub.S index 43d212618834d,3dcc3272ce166..12c7fad02ae51 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@@ -73,8 -97,61 +97,61 @@@ SYM_CODE_START_LOCAL(elx_sync eret SYM_CODE_END(elx_sync) - // nVHE? No way! Give me the real thing! - SYM_CODE_START_LOCAL(mutate_to_vhe) + SYM_CODE_START_LOCAL(__finalise_el2) + check_override id_aa64pfr0 ID_AA64PFR0_SVE_SHIFT .Linit_sve .Lskip_sve + + .Linit_sve: /* SVE register access */ + mrs x0, cptr_el2 // Disable SVE traps + bic x0, x0, #CPTR_EL2_TZ + msr cptr_el2, x0 + isb + mov x1, #ZCR_ELx_LEN_MASK // SVE: Enable full vector + msr_s SYS_ZCR_EL2, x1 // length for EL1. + + .Lskip_sve: + check_override id_aa64pfr1 ID_AA64PFR1_SME_SHIFT .Linit_sme .Lskip_sme + + .Linit_sme: /* SME register access and priority mapping */ + mrs x0, cptr_el2 // Disable SME traps + bic x0, x0, #CPTR_EL2_TSM + msr cptr_el2, x0 + isb + + mrs x1, sctlr_el2 + orr x1, x1, #SCTLR_ELx_ENTP2 // Disable TPIDR2 traps + msr sctlr_el2, x1 + isb + + mov x0, #0 // SMCR controls + + // Full FP in SM? + mrs_s x1, SYS_ID_AA64SMFR0_EL1 - __check_override id_aa64smfr0 ID_AA64SMFR0_FA64_SHIFT 1 .Linit_sme_fa64 .Lskip_sme_fa64 ++ __check_override id_aa64smfr0 ID_AA64SMFR0_EL1_FA64_SHIFT 1 .Linit_sme_fa64 .Lskip_sme_fa64 + + .Linit_sme_fa64: + orr x0, x0, SMCR_ELx_FA64_MASK + .Lskip_sme_fa64: + + orr x0, x0, #SMCR_ELx_LEN_MASK // Enable full SME vector + msr_s SYS_SMCR_EL2, x0 // length for EL1. + + mrs_s x1, SYS_SMIDR_EL1 // Priority mapping supported? + ubfx x1, x1, #SMIDR_EL1_SMPS_SHIFT, #1 + cbz x1, .Lskip_sme + + msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal + + mrs x1, id_aa64mmfr1_el1 // HCRX_EL2 present? + ubfx x1, x1, #ID_AA64MMFR1_HCX_SHIFT, #4 + cbz x1, .Lskip_sme + + mrs_s x1, SYS_HCRX_EL2 + orr x1, x1, #HCRX_EL2_SMPME_MASK // Enable priority mapping + msr_s SYS_HCRX_EL2, x1 + + .Lskip_sme: + + // nVHE? No way! Give me the real thing! // Sanity check: MMU *must* be off mrs x1, sctlr_el2 tbnz x1, #0, 1f diff --cc arch/arm64/kernel/idreg-override.c index 21b3d03089cab,7206fd0ed9eb4..1b0542c69738e --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@@ -53,8 -98,9 +98,9 @@@ static const struct ftr_set_desc pfr1 _ .name = "id_aa64pfr1", .override = &id_aa64pfr1_override, .fields = { - { "bt", ID_AA64PFR1_BT_SHIFT }, - { "mte", ID_AA64PFR1_MTE_SHIFT}, - FIELD("bt", ID_AA64PFR1_BT_SHIFT, NULL), ++ FIELD("bt", ID_AA64PFR1_BT_SHIFT, NULL ), + FIELD("mte", ID_AA64PFR1_MTE_SHIFT, NULL), + FIELD("sme", ID_AA64PFR1_SME_SHIFT, pfr1_sme_filter), {} }, }; @@@ -63,10 -109,10 +109,10 @@@ static const struct ftr_set_desc isar1 .name = "id_aa64isar1", .override = &id_aa64isar1_override, .fields = { - { "gpi", ID_AA64ISAR1_EL1_GPI_SHIFT }, - { "gpa", ID_AA64ISAR1_EL1_GPA_SHIFT }, - { "api", ID_AA64ISAR1_EL1_API_SHIFT }, - { "apa", ID_AA64ISAR1_EL1_APA_SHIFT }, - FIELD("gpi", ID_AA64ISAR1_GPI_SHIFT, NULL), - FIELD("gpa", ID_AA64ISAR1_GPA_SHIFT, NULL), - FIELD("api", ID_AA64ISAR1_API_SHIFT, NULL), - FIELD("apa", ID_AA64ISAR1_APA_SHIFT, NULL), ++ FIELD("gpi", ID_AA64ISAR1_EL1_GPI_SHIFT, NULL), ++ FIELD("gpa", ID_AA64ISAR1_EL1_GPA_SHIFT, NULL), ++ FIELD("api", ID_AA64ISAR1_EL1_API_SHIFT, NULL), ++ FIELD("apa", ID_AA64ISAR1_EL1_APA_SHIFT, NULL), {} }, }; @@@ -75,8 -121,18 +121,18 @@@ static const struct ftr_set_desc isar2 .name = "id_aa64isar2", .override = &id_aa64isar2_override, .fields = { - { "gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT }, - { "apa3", ID_AA64ISAR2_EL1_APA3_SHIFT }, - FIELD("gpa3", ID_AA64ISAR2_GPA3_SHIFT, NULL), - FIELD("apa3", ID_AA64ISAR2_APA3_SHIFT, NULL), ++ FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL), ++ FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL), + {} + }, + }; + + static const struct ftr_set_desc smfr0 __initconst = { + .name = "id_aa64smfr0", + .override = &id_aa64smfr0_override, + .fields = { + /* FA64 is a one bit field... :-/ */ - { "fa64", ID_AA64SMFR0_FA64_SHIFT, 1, }, ++ { "fa64", ID_AA64SMFR0_EL1_FA64_SHIFT, 1, }, {} }, }; diff --cc arch/arm64/mm/proc.S index 972ce8d7f2c58,9eb490effb7f1..7837a69524c53 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@@ -249,14 -222,14 +249,16 @@@ SYM_FUNC_END(idmap_cpu_replace_ttbr1 * * Called exactly once from stop_machine context by each CPU found during boot. */ - __idmap_kpti_flag: - .long 1 + .pushsection ".data", "aw", %progbits + SYM_DATA(__idmap_kpti_flag, .long 1) + .popsection + SYM_FUNC_START(idmap_kpti_install_ng_mappings) cpu .req w0 + temp_pte .req x0 num_cpus .req w1 - swapper_pa .req x2 + pte_flags .req x1 + temp_pgd_phys .req x2 swapper_ttb .req x3 flag_ptr .req x4 cur_pgdp .req x5 @@@ -269,11 -243,11 +271,11 @@@ cur_ptep .req x14 end_ptep .req x15 pte .req x16 + valid .req x17 + mov x5, x3 // preserve temp_pte arg mrs swapper_ttb, ttbr1_el1 - adr flag_ptr, __idmap_kpti_flag - restore_ttbr1 swapper_ttb + adr_l flag_ptr, __idmap_kpti_flag cbnz cpu, __idmap_kpti_secondary