Skip to content

Commit f96d67a

Browse files
willdeaconwilldeacon
committed
Merge branch 'for-next/boot' into for-next/core
* for-next/boot: (34 commits) arm64: fix KASAN_INLINE arm64: Add an override for ID_AA64SMFR0_EL1.FA64 arm64: Add the arm64.nosve command line option arm64: Add the arm64.nosme command line option arm64: Expose a __check_override primitive for oddball features arm64: Allow the idreg override to deal with variable field width arm64: Factor out checking of a feature against the override into a macro arm64: Allow sticky E2H when entering EL1 arm64: Save state of HCR_EL2.E2H before switch to EL1 arm64: Rename the VHE switch to "finalise_el2" arm64: mm: fix booting with 52-bit address space arm64: head: remove __PHYS_OFFSET arm64: lds: use PROVIDE instead of conditional definitions arm64: setup: drop early FDT pointer helpers arm64: head: avoid relocating the kernel twice for KASLR arm64: kaslr: defer initialization to initcall where permitted arm64: head: record CPU boot mode after enabling the MMU arm64: head: populate kernel page tables with MMU and caches on arm64: head: factor out TTBR1 assignment into a macro arm64: idreg-override: use early FDT mapping in ID map ...
2 parents 9286773 + 1191b62 commit f96d67a

24 files changed

+750
-612
lines changed

‎Documentation/admin-guide/kernel-parameters.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,12 @@
400400
arm64.nomte [ARM64] Unconditionally disable Memory Tagging Extension
401401
support
402402

403+
arm64.nosve [ARM64] Unconditionally disable Scalable Vector
404+
Extension support
405+
406+
arm64.nosme [ARM64] Unconditionally disable Scalable Matrix
407+
Extension support
408+
403409
ataflop= [HW,M68k]
404410

405411
atarimouse= [HW,MOUSE] Atari Mouse

‎Documentation/virt/kvm/arm/hyp-abi.rst

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,13 @@ these functions (see arch/arm{,64}/include/asm/virt.h):
6060

6161
* ::
6262

63-
x0 = HVC_VHE_RESTART (arm64 only)
63+
x0 = HVC_FINALISE_EL2 (arm64 only)
6464

65-
Attempt to upgrade the kernel's exception level from EL1 to EL2 by enabling
66-
the VHE mode. This is conditioned by the CPU supporting VHE, the EL2 MMU
67-
being off, and VHE not being disabled by any other means (command line
68-
option, for example).
65+
Finish configuring EL2 depending on the command-line options,
66+
including an attempt to upgrade the kernel's exception level from
67+
EL1 to EL2 by enabling the VHE mode. This is conditioned by the CPU
68+
supporting VHE, the EL2 MMU being off, and VHE not being disabled by
69+
any other means (command line option, for example).
6970

7071
Any other value of r0/x0 triggers a hypervisor-specific handling,
7172
which is not documented here.

‎arch/arm64/include/asm/assembler.h

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,20 @@ alternative_cb_end
359359
bfi \valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH
360360
.endm
361361

362+
/*
363+
* idmap_get_t0sz - get the T0SZ value needed to cover the ID map
364+
*
365+
* Calculate the maximum allowed value for TCR_EL1.T0SZ so that the
366+
* entire ID map region can be mapped. As T0SZ == (64 - #bits used),
367+
* this number conveniently equals the number of leading zeroes in
368+
* the physical address of _end.
369+
*/
370+
.macro idmap_get_t0sz, reg
371+
adrp \reg, _end
372+
orr \reg, \reg, #(1 << VA_BITS_MIN) - 1
373+
clz \reg, \reg
374+
.endm
375+
362376
/*
363377
* tcr_compute_pa_size - set TCR.(I)PS to the highest supported
364378
* ID_AA64MMFR0_EL1.PARange value
@@ -465,6 +479,18 @@ alternative_endif
465479
_cond_uaccess_extable .Licache_op\@, \fixup
466480
.endm
467481

482+
/*
483+
* load_ttbr1 - install @pgtbl as a TTBR1 page table
484+
* pgtbl preserved
485+
* tmp1/tmp2 clobbered, either may overlap with pgtbl
486+
*/
487+
.macro load_ttbr1, pgtbl, tmp1, tmp2
488+
phys_to_ttbr \tmp1, \pgtbl
489+
offset_ttbr1 \tmp1, \tmp2
490+
msr ttbr1_el1, \tmp1
491+
isb
492+
.endm
493+
468494
/*
469495
* To prevent the possibility of old and new partial table walks being visible
470496
* in the tlb, switch the ttbr to a zero page when we invalidate the old
@@ -478,10 +504,7 @@ alternative_endif
478504
isb
479505
tlbi vmalle1
480506
dsb nsh
481-
phys_to_ttbr \tmp, \page_table
482-
offset_ttbr1 \tmp, \tmp2
483-
msr ttbr1_el1, \tmp
484-
isb
507+
load_ttbr1 \page_table, \tmp, \tmp2
485508
.endm
486509

487510
/*

‎arch/arm64/include/asm/cpufeature.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -908,7 +908,10 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
908908
}
909909

910910
extern struct arm64_ftr_override id_aa64mmfr1_override;
911+
extern struct arm64_ftr_override id_aa64pfr0_override;
911912
extern struct arm64_ftr_override id_aa64pfr1_override;
913+
extern struct arm64_ftr_override id_aa64zfr0_override;
914+
extern struct arm64_ftr_override id_aa64smfr0_override;
912915
extern struct arm64_ftr_override id_aa64isar1_override;
913916
extern struct arm64_ftr_override id_aa64isar2_override;
914917

‎arch/arm64/include/asm/el2_setup.h

Lines changed: 0 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -129,64 +129,6 @@
129129
msr cptr_el2, x0 // Disable copro. traps to EL2
130130
.endm
131131

132-
/* SVE register access */
133-
.macro __init_el2_nvhe_sve
134-
mrs x1, id_aa64pfr0_el1
135-
ubfx x1, x1, #ID_AA64PFR0_SVE_SHIFT, #4
136-
cbz x1, .Lskip_sve_\@
137-
138-
bic x0, x0, #CPTR_EL2_TZ // Also disable SVE traps
139-
msr cptr_el2, x0 // Disable copro. traps to EL2
140-
isb
141-
mov x1, #ZCR_ELx_LEN_MASK // SVE: Enable full vector
142-
msr_s SYS_ZCR_EL2, x1 // length for EL1.
143-
.Lskip_sve_\@:
144-
.endm
145-
146-
/* SME register access and priority mapping */
147-
.macro __init_el2_nvhe_sme
148-
mrs x1, id_aa64pfr1_el1
149-
ubfx x1, x1, #ID_AA64PFR1_SME_SHIFT, #4
150-
cbz x1, .Lskip_sme_\@
151-
152-
bic x0, x0, #CPTR_EL2_TSM // Also disable SME traps
153-
msr cptr_el2, x0 // Disable copro. traps to EL2
154-
isb
155-
156-
mrs x1, sctlr_el2
157-
orr x1, x1, #SCTLR_ELx_ENTP2 // Disable TPIDR2 traps
158-
msr sctlr_el2, x1
159-
isb
160-
161-
mov x1, #0 // SMCR controls
162-
163-
mrs_s x2, SYS_ID_AA64SMFR0_EL1
164-
ubfx x2, x2, #ID_AA64SMFR0_EL1_FA64_SHIFT, #1 // Full FP in SM?
165-
cbz x2, .Lskip_sme_fa64_\@
166-
167-
orr x1, x1, SMCR_ELx_FA64_MASK
168-
.Lskip_sme_fa64_\@:
169-
170-
orr x1, x1, #SMCR_ELx_LEN_MASK // Enable full SME vector
171-
msr_s SYS_SMCR_EL2, x1 // length for EL1.
172-
173-
mrs_s x1, SYS_SMIDR_EL1 // Priority mapping supported?
174-
ubfx x1, x1, #SMIDR_EL1_SMPS_SHIFT, #1
175-
cbz x1, .Lskip_sme_\@
176-
177-
msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal
178-
179-
mrs x1, id_aa64mmfr1_el1 // HCRX_EL2 present?
180-
ubfx x1, x1, #ID_AA64MMFR1_HCX_SHIFT, #4
181-
cbz x1, .Lskip_sme_\@
182-
183-
mrs_s x1, SYS_HCRX_EL2
184-
orr x1, x1, #HCRX_EL2_SMPME_MASK // Enable priority mapping
185-
msr_s SYS_HCRX_EL2, x1
186-
187-
.Lskip_sme_\@:
188-
.endm
189-
190132
/* Disable any fine grained traps */
191133
.macro __init_el2_fgt
192134
mrs x1, id_aa64mmfr0_el1
@@ -250,8 +192,6 @@
250192
__init_el2_hstr
251193
__init_el2_nvhe_idregs
252194
__init_el2_nvhe_cptr
253-
__init_el2_nvhe_sve
254-
__init_el2_nvhe_sme
255195
__init_el2_fgt
256196
__init_el2_nvhe_prepare_eret
257197
.endm

‎arch/arm64/include/asm/kernel-pgtable.h

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#ifndef __ASM_KERNEL_PGTABLE_H
99
#define __ASM_KERNEL_PGTABLE_H
1010

11+
#include <asm/boot.h>
1112
#include <asm/pgtable-hwdef.h>
1213
#include <asm/sparsemem.h>
1314

@@ -35,10 +36,8 @@
3536
*/
3637
#if ARM64_KERNEL_USES_PMD_MAPS
3738
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
38-
#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT) - 1)
3939
#else
4040
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
41-
#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT))
4241
#endif
4342

4443

@@ -87,7 +86,14 @@
8786
+ EARLY_PUDS((vstart), (vend)) /* each PUD needs a next level page table */ \
8887
+ EARLY_PMDS((vstart), (vend))) /* each PMD needs a next level page table */
8988
#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR, _end))
90-
#define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
89+
90+
/* the initial ID map may need two extra pages if it needs to be extended */
91+
#if VA_BITS < 48
92+
#define INIT_IDMAP_DIR_SIZE ((INIT_IDMAP_DIR_PAGES + 2) * PAGE_SIZE)
93+
#else
94+
#define INIT_IDMAP_DIR_SIZE (INIT_IDMAP_DIR_PAGES * PAGE_SIZE)
95+
#endif
96+
#define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE)
9197

9298
/* Initial memory map size */
9399
#if ARM64_KERNEL_USES_PMD_MAPS
@@ -107,9 +113,11 @@
107113
#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
108114

109115
#if ARM64_KERNEL_USES_PMD_MAPS
110-
#define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
116+
#define SWAPPER_RW_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
117+
#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PMD_SECT_RDONLY)
111118
#else
112-
#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
119+
#define SWAPPER_RW_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
120+
#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PTE_RDONLY)
113121
#endif
114122

115123
/*

‎arch/arm64/include/asm/memory.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,11 @@
174174
#include <linux/types.h>
175175
#include <asm/bug.h>
176176

177+
#if VA_BITS > 48
177178
extern u64 vabits_actual;
179+
#else
180+
#define vabits_actual ((u64)VA_BITS)
181+
#endif
178182

179183
extern s64 memstart_addr;
180184
/* PHYS_OFFSET - the physical address of the start of memory. */

‎arch/arm64/include/asm/mmu_context.h

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,7 @@ static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm)
6060
* TCR_T0SZ(VA_BITS), unless system RAM is positioned very high in
6161
* physical memory, in which case it will be smaller.
6262
*/
63-
extern u64 idmap_t0sz;
64-
extern u64 idmap_ptrs_per_pgd;
63+
extern int idmap_t0sz;
6564

6665
/*
6766
* Ensure TCR.T0SZ is set to the provided value.
@@ -106,13 +105,18 @@ static inline void cpu_uninstall_idmap(void)
106105
cpu_switch_mm(mm->pgd, mm);
107106
}
108107

109-
static inline void cpu_install_idmap(void)
108+
static inline void __cpu_install_idmap(pgd_t *idmap)
110109
{
111110
cpu_set_reserved_ttbr0();
112111
local_flush_tlb_all();
113112
cpu_set_idmap_tcr_t0sz();
114113

115-
cpu_switch_mm(lm_alias(idmap_pg_dir), &init_mm);
114+
cpu_switch_mm(lm_alias(idmap), &init_mm);
115+
}
116+
117+
static inline void cpu_install_idmap(void)
118+
{
119+
__cpu_install_idmap(idmap_pg_dir);
116120
}
117121

118122
/*
@@ -143,7 +147,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz)
143147
* Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
144148
* avoiding the possibility of conflicting TLB entries being allocated.
145149
*/
146-
static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp)
150+
static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
147151
{
148152
typedef void (ttbr_replace_func)(phys_addr_t);
149153
extern ttbr_replace_func idmap_cpu_replace_ttbr1;
@@ -166,7 +170,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp)
166170

167171
replace_phys = (void *)__pa_symbol(function_nocfi(idmap_cpu_replace_ttbr1));
168172

169-
cpu_install_idmap();
173+
__cpu_install_idmap(idmap);
170174
replace_phys(ttbr1);
171175
cpu_uninstall_idmap();
172176
}

‎arch/arm64/include/asm/virt.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@
3636
#define HVC_RESET_VECTORS 2
3737

3838
/*
39-
* HVC_VHE_RESTART - Upgrade the CPU from EL1 to EL2, if possible
39+
* HVC_FINALISE_EL2 - Upgrade the CPU from EL1 to EL2, if possible
4040
*/
41-
#define HVC_VHE_RESTART 3
41+
#define HVC_FINALISE_EL2 3
4242

4343
/* Max number of HYP stub hypercalls */
4444
#define HVC_STUB_HCALL_NR 4
@@ -49,6 +49,13 @@
4949
#define BOOT_CPU_MODE_EL1 (0xe11)
5050
#define BOOT_CPU_MODE_EL2 (0xe12)
5151

52+
/*
53+
* Flags returned together with the boot mode, but not preserved in
54+
* __boot_cpu_mode. Used by the idreg override code to work out the
55+
* boot state.
56+
*/
57+
#define BOOT_CPU_FLAG_E2H BIT_ULL(32)
58+
5259
#ifndef __ASSEMBLY__
5360

5461
#include <asm/ptrace.h>

‎arch/arm64/kernel/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ obj-$(CONFIG_ACPI) += acpi.o
6464
obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o
6565
obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o
6666
obj-$(CONFIG_PARAVIRT) += paravirt.o
67-
obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
67+
obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o pi/
6868
obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o
6969
obj-$(CONFIG_ELF_CORE) += elfcore.o
7070
obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \

‎arch/arm64/kernel/cpufeature.c

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -633,7 +633,10 @@ static const struct arm64_ftr_bits ftr_raz[] = {
633633
__ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override)
634634

635635
struct arm64_ftr_override __ro_after_init id_aa64mmfr1_override;
636+
struct arm64_ftr_override __ro_after_init id_aa64pfr0_override;
636637
struct arm64_ftr_override __ro_after_init id_aa64pfr1_override;
638+
struct arm64_ftr_override __ro_after_init id_aa64zfr0_override;
639+
struct arm64_ftr_override __ro_after_init id_aa64smfr0_override;
637640
struct arm64_ftr_override __ro_after_init id_aa64isar1_override;
638641
struct arm64_ftr_override __ro_after_init id_aa64isar2_override;
639642

@@ -670,11 +673,14 @@ static const struct __ftr_reg_entry {
670673
ARM64_FTR_REG(SYS_ID_MMFR5_EL1, ftr_id_mmfr5),
671674

672675
/* Op1 = 0, CRn = 0, CRm = 4 */
673-
ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
676+
ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0,
677+
&id_aa64pfr0_override),
674678
ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1,
675679
&id_aa64pfr1_override),
676-
ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0),
677-
ARM64_FTR_REG(SYS_ID_AA64SMFR0_EL1, ftr_id_aa64smfr0),
680+
ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0,
681+
&id_aa64zfr0_override),
682+
ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64SMFR0_EL1, ftr_id_aa64smfr0,
683+
&id_aa64smfr0_override),
678684

679685
/* Op1 = 0, CRn = 0, CRm = 5 */
680686
ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
@@ -3295,7 +3301,7 @@ subsys_initcall_sync(init_32bit_el0_mask);
32953301

32963302
static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
32973303
{
3298-
cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
3304+
cpu_replace_ttbr1(lm_alias(swapper_pg_dir), idmap_pg_dir);
32993305
}
33003306

33013307
/*

0 commit comments

Comments
 (0)