26e5d89c73
`jmp mpenter`.
x86-64 requires rsp to be 16-byte aligned immediately before a `call`, so a
function starts with rsp ending in 8 -- the return address -- and its
`push %rbp; mov %rsp,%rbp` prologue leaves %rbp 16-byte aligned. Compilers
build on that: an aligned vector store to a frame slot is `movaps`, and
`movaps` to a misaligned address is #GP.
entry64mp loaded the AP's stack pointer, which is page-aligned, and *jumped*.
Nothing pushed a return address, so %rbp came out eight bytes off -- on every
frame, on every application processor, for the life of the machine.
It cost nothing for as long as no function an AP ran needed an aligned vector
store. Then the console's glyph path started returning a small struct by value
(the cell-as-a-surface change), clang spilled it with movaps, and CPU 1 took a
#GP the instant it announced itself on a verbose boot:
unexpected trap 13 from cpu 1 rip ffff800000120a3b <draw_glyph_px+0x1cb>
120a37: 0f 10 45 90 movups -0x70(%rbp), %xmm0
120a3b: 0f 29 45 b0 movaps %xmm0, -0x50(%rbp) <- #GP
movups on the line above loads fine; movaps stores and faults. The
displacement is a multiple of 16, so what is misaligned is %rbp itself, which
is a fact about how the function was entered.
`call mpenter`, not `jmp`. Eight bytes for an ABI the whole toolchain assumes.
**It was hiding behind a second bug.** The first report was not a #GP at all:
Stop code ACQUIRE
#2 cprintf+0xae
#3 trap+0x2c4
trap() reported the fault with cprintf(), cprintf() takes the console lock,
and the faulting code was already holding it -- on a verbose boot every
character is drawn under that lock. acquire() panicked about a lock its own
call chain owned, and the trap number, the faulting address and the registers
were all lost. A kernel-mode trap ends in panic() a dozen lines later anyway,
so the console goes into panic mode *before* the report rather than during it.
**And a third, in the same boot.** The console reported "JetBrains Mono, 8x16
cells" -- a name from one font and metrics from another. The built-in bitmap
became a provider like any other in the previous commit, so draw_glyph_px()
installs it lazily on the first glyph; on a verbose boot that happens before
gfx_font_init() runs, which then returned early because a font was already
installed. The console spent the whole boot in the fallback while claiming not
to be. "A font is installed" and "a typeface was chosen" had been the same test
and quietly stopped being; they are two flags now.
Verified: verbose boot reaches a shell twice running with no traps and no
panics, and reports 10x21 cells -- the real typeface -- where it reported 8x16
before. Both supported configurations quiet: one boot each, no traps, no
panics, socktest and memfdtest green.
Written up as section 22 of docs/history/kernel.md, because the interesting
part is not the missing eight bytes but that three separate faults had to be
peeled off in order before the first one could be seen.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
269 lines
No EOL
9 KiB
ArmAsm
269 lines
No EOL
9 KiB
ArmAsm
/* entry.S
|
|
*
|
|
* Copyright (c) 2013 Brian Swetland
|
|
* Modified by Nexoniarz
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining
|
|
* a copy of this software and associated documentation files (the
|
|
* "Software"), to deal in the Software without restriction, including
|
|
* without limitation the rights to use, copy, modify, merge, publish,
|
|
* distribute, sublicense, and/or sell copies of the Software, and to
|
|
* permit persons to whom the Software is furnished to do so, subject to
|
|
* the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be
|
|
* included in all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
*
|
|
*/
|
|
|
|
#define mboot_magic 0x1badb002
|
|
#define mboot_flags 0x00010000
|
|
#include "mmu.h"
|
|
#include "schboot.h"
|
|
|
|
/* A second entry point, for firmware that has already put us in long mode.
|
|
*
|
|
* The multiboot header below is entered in 32-bit protected mode, and
|
|
* mboot_entry builds the page tables and makes the long-mode transition
|
|
* itself. UEFI firmware hands a loader long mode with paging already on, so
|
|
* that path would mean dropping back to protected mode only to climb out
|
|
* again -- a mode switch whose failure mode is a triple fault with no console.
|
|
*
|
|
* Instead the 64-bit entry is published where a loader can find it, by the
|
|
* same trick the BIOS loader already uses to find the multiboot header: scan
|
|
* the first 16 KB for a magic. boot/uefi/ looks for SCH64_MAGIC and jumps to
|
|
* the physical address recorded here.
|
|
*
|
|
* The contract at that address, which boot/uefi/main.c must satisfy:
|
|
* - long mode, interrupts off
|
|
* - CR3 pointing at page tables that identity-map the low 4 GB *and* map
|
|
* the same 4 GB at 0xffff800000000000 -- entry64low's first act is a
|
|
* movabs to a high address, so a loader that only identity-maps triple
|
|
* faults on the very first instruction
|
|
* - RBX = 0, marking the boot processor rather than an AP
|
|
* Those are exactly the conditions mboot_entry has established by the time it
|
|
* reaches entry64low, which is why this points there and not somewhere new.
|
|
*/
|
|
.code64
|
|
.global mboot_header
|
|
.global mboot_entry
|
|
|
|
mboot_header:
|
|
.long mboot_magic
|
|
.long mboot_flags
|
|
.long (-mboot_magic -mboot_flags)
|
|
.long mboot_load_addr
|
|
.long mboot_load_addr
|
|
.long mboot_load_end
|
|
.long mboot_bss_end
|
|
.long mboot_entry_addr
|
|
|
|
/* After the multiboot header, never before it: the header's own header_addr
|
|
field says it lives at mboot_load_addr, and the BIOS loader computes the
|
|
kernel's file offset from (header_addr - load_addr). Sixteen bytes inserted
|
|
ahead of it makes that arithmetic wrong and loads the kernel shifted, which
|
|
fails as a triple fault rather than as anything that names the cause. */
|
|
.align 8
|
|
.global sch64_header
|
|
sch64_header:
|
|
.long SCH64_MAGIC0
|
|
.long SCH64_MAGIC1
|
|
.quad (entry64uefi - mboot_header + mboot_load_addr)
|
|
|
|
.code32
|
|
mboot_entry:
|
|
mov %eax, %ebp
|
|
|
|
# bootstrap page tables (0x1000 - 0x3000)
|
|
xor %eax, %eax
|
|
mov $0x1000, %edi
|
|
mov $0x2000, %ecx
|
|
rep stosb
|
|
|
|
# Bootstrap page tables (Identity mapping 4GB)
|
|
# PML4T[0] i PML4T[256] -> PDPT pod 0x2000
|
|
mov $(0x2000 | PTE_P | PTE_W), %edx
|
|
mov %edx, 0x1000
|
|
mov %edx, 0x1800
|
|
|
|
# PDPT mapping 4GB using 1GB huge pages
|
|
mov $(0x00000000 | PTE_P | PTE_PS | PTE_W), %edx
|
|
mov %edx, 0x2000
|
|
mov $(0x40000000 | PTE_P | PTE_PS | PTE_W), %edx
|
|
mov %edx, 0x2008
|
|
mov $(0x80000000 | PTE_P | PTE_PS | PTE_W), %edx
|
|
mov %edx, 0x2010
|
|
mov $(0xC0000000 | PTE_P | PTE_PS | PTE_W), %edx
|
|
mov %edx, 0x2018
|
|
|
|
mov %ebp, %ebx
|
|
|
|
.global entry32mp
|
|
entry32mp:
|
|
mov $0x1000, %eax
|
|
mov %eax, %cr3
|
|
|
|
lgdt (gdtr64 - mboot_header + mboot_load_addr)
|
|
|
|
mov %cr4, %eax
|
|
bts $5, %eax
|
|
mov %eax, %cr4
|
|
|
|
mov $MSR_EFER, %ecx
|
|
rdmsr
|
|
bts $0, %eax
|
|
bts $8, %eax
|
|
wrmsr
|
|
|
|
mov %cr0, %eax
|
|
orl $(CR0_PG | CR0_WP | CR0_MP), %eax
|
|
mov %eax, %cr0
|
|
|
|
ljmp $8, $(entry64low - mboot_header + mboot_load_addr)
|
|
|
|
.align 16
|
|
gdtr64:
|
|
.word gdt64_end - gdt64_begin - 1;
|
|
.quad gdt64_begin - mboot_header + mboot_load_addr
|
|
|
|
.align 16
|
|
gdt64_begin:
|
|
.long 0x00000000
|
|
.long 0x00000000
|
|
.long 0x00000000 # 64-bit code
|
|
.long 0x00209800
|
|
.long 0x00000000 # 64-bit data
|
|
.long 0x00009000
|
|
gdt64_end:
|
|
|
|
.align 16
|
|
.code64
|
|
|
|
/* Entered by boot/uefi/ with the firmware's GDT still loaded.
|
|
*
|
|
* mboot_entry reaches entry64low by way of entry32mp, which does the lgdt on
|
|
* the way past. A loader that arrives already in long mode skips all of that,
|
|
* so it arrives running on the firmware's descriptors -- OVMF's CS is 0x38 --
|
|
* and the first thing the kernel does that depends on its own selectors takes
|
|
* a #GP against a GDT the firmware no longer owns. It surfaced as
|
|
* "unexpected trap 13 ... cs=38", a long way from the cause.
|
|
*
|
|
* So load the kernel's GDT here and reload CS through it before going on.
|
|
* Doing it in the kernel rather than in the loader is deliberate: the GDT is
|
|
* the kernel's, and a loader that had to know its layout would have to be
|
|
* changed every time it moved.
|
|
*/
|
|
.global entry64uefi
|
|
entry64uefi:
|
|
/* Enable SYSCALL/SYSRET.
|
|
*
|
|
* The 32-bit entry above sets EFER.SCE on its way into long mode. This path
|
|
* skips all of that, because firmware has already done the mode switch -- and
|
|
* with it skips the one bit that is not about paging. UEFI has no reason to
|
|
* set SCE and does not, so the kernel came up with SYSCALL disabled and the
|
|
* first sysretq back to user space raised #UD: "unexpected trap 6 ... rip
|
|
* <syscall_trapret>" in pid 1, at zero ticks. It stayed hidden until this
|
|
* path first got far enough to run a user program.
|
|
*
|
|
* Read-modify-write: LME and LMA are already set by firmware and must stay.
|
|
*/
|
|
mov $MSR_EFER, %ecx
|
|
rdmsr
|
|
bts $0, %eax
|
|
wrmsr
|
|
|
|
lgdt (gdtr64 - mboot_header + mboot_load_addr)
|
|
movq $(entry64low - mboot_header + mboot_load_addr), %rax
|
|
pushq $8 /* the 64-bit code selector in gdt64 */
|
|
pushq %rax
|
|
lretq /* far return: reloads CS */
|
|
|
|
entry64low:
|
|
movabs $entry64high, %rax
|
|
jmp *%rax
|
|
|
|
.global _start
|
|
_start:
|
|
entry64high:
|
|
xor %rax, %rax
|
|
mov %ax, %ss
|
|
mov %ax, %ds
|
|
mov %ax, %es
|
|
mov %ax, %fs
|
|
mov %ax, %gs
|
|
|
|
# Is this an application processor, or the boot processor?
|
|
#
|
|
# This check used to be a comment saying it had not been written yet, and
|
|
# every AP fell straight through into the BSP path below: it set RSP to the
|
|
# *boot processor's* stack, passed its own flag to kmain() as a framebuffer
|
|
# address, and re-ran the entire bootstrap -- e820_init, kinit1, kvmalloc --
|
|
# on a second core sharing one stack with the first. That is the SMP triple
|
|
# fault. QEMU hid it for years by never reporting more than one CPU (see
|
|
# kernel/arch/x86_64/acpi.c); VMware did not.
|
|
#
|
|
# EBX is the discriminator, and entryother.S sets it to 1 immediately before
|
|
# entering long mode for exactly this purpose. It is safe: on the boot
|
|
# processor EBX carries the VESA framebuffer address from the loader, which is
|
|
# either zero or a physical MMIO address, and can never be 1.
|
|
cmp $1, %ebx
|
|
je entry64mp
|
|
|
|
# Setup initial stack
|
|
movabs $0xFFFF800000010000, %rax
|
|
mov %rax, %rsp
|
|
|
|
# Pass VESA address to main (1st argument in RDI)
|
|
mov %rbx, %rdi
|
|
|
|
call kmain
|
|
|
|
.global __deadloop
|
|
__deadloop:
|
|
jmp .
|
|
|
|
entry64mp:
|
|
mov $0x7000, %rax
|
|
mov -16(%rax), %rsp
|
|
|
|
# `call`, not `jmp`, and the difference is an ABI requirement rather than a
|
|
# style preference.
|
|
#
|
|
# x86-64 says rsp is 16-byte aligned immediately before a `call`, so a
|
|
# function starts with rsp ending in 8 -- the return address the call just
|
|
# pushed -- and its `push %rbp; mov %rsp,%rbp` prologue therefore leaves
|
|
# %rbp 16-byte aligned. Compilers rely on that: an aligned SSE store to a
|
|
# frame slot is `movaps`, and `movaps` to a misaligned address is #GP.
|
|
#
|
|
# This was a `jmp` into a 16-byte-aligned stack, so every frame on every
|
|
# application processor was off by eight, for the life of the machine. It
|
|
# cost nothing for as long as no function the APs ran happened to need an
|
|
# aligned vector store -- and then one did, deep in the console, and CPU 1
|
|
# took a #GP the moment it announced itself on a verbose boot.
|
|
#
|
|
# mpenter never returns, so the pushed address is only ever a backtrace
|
|
# entry. Eight bytes for an ABI the whole toolchain assumes is a bargain.
|
|
call mpenter
|
|
jmp __deadloop
|
|
|
|
.global wrmsr
|
|
wrmsr:
|
|
mov %rdi, %rcx
|
|
mov %rsi, %rax
|
|
shr $32, %rsi
|
|
mov %rsi, %rdx
|
|
wrmsr
|
|
retq
|
|
|
|
.global ignore_sysret
|
|
ignore_sysret:
|
|
mov $-38, %rax
|
|
sysretq |