amd64: FRED support

FRED support as defined starting from the SDM rev. 90, requires a new
'events' entry point to receive user and kernel mode exceptions and
interrupts notifications from the hardware.  A minimal asm trampoline is
enough, rest can be implemented in C due to the clean FRED organization
of the event reporting.

The syscall entry is handled by a microptimized assembly path, directly
calling into the amd64_syscall() handler, instead of the generic events
entry point.

Tested by:	emaste
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D55829

sys/amd64/amd64/cpu_switch.S

@@ -187,14 +187,22 @@ do_kthread:
 	cmpq	%rax,%rdx
 	jne	do_tss
 done_tss:
+	movq	%r8,PCPU(CURPCB)
+	cmpb	$0,fred
+	jne	1f
 	movq	TD_MD_STACK_BASE(%r12),%r9
 	movq	%r9,PCPU(RSP0)
-	movq	%r8,PCPU(CURPCB)
 	movq	PCPU(PTI_RSP0),%rax
 	cmpq	$~0,PCPU(UCR3)
 	cmove	%r9,%rax
 	movq	%rax,TSS_RSP0(%rdx)
-	movq	%r12,PCPU(CURTHREAD)		/* into next thread */
+	jmp	2f
+1:	movq	TD_MD_STACK_BASE(%r12),%rdx
+	movl	%edx,%eax
+	shrq	$32,%rdx
+	movl	$MSR_FRED_RSP0,%ecx
+	wrmsr
+2:	movq	%r12,PCPU(CURTHREAD)		/* into next thread */
 
 	/* Test if debug registers should be restored. */
 	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
@@ -365,8 +373,36 @@ ENTRY(savectx)
 	movl	%eax,PCB_SFMASK(%rdi)
 	movl	%edx,PCB_SFMASK+4(%rdi)
 
-	sgdt	PCB_GDT(%rdi)
-	sidt	PCB_IDT(%rdi)
+	cmpb	$0, fred
+	je	1f
+	movl	$MSR_FRED_RSP0,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP0(%rdi)
+	movl	%edx,PCB_FRED_RSP0+4(%rdi)
+	movl	$MSR_FRED_RSP1,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP1(%rdi)
+	movl	%edx,PCB_FRED_RSP1+4(%rdi)
+	movl	$MSR_FRED_RSP2,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP2(%rdi)
+	movl	%edx,PCB_FRED_RSP2+4(%rdi)
+	movl	$MSR_FRED_RSP3,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP3(%rdi)
+	movl	%edx,PCB_FRED_RSP3+4(%rdi)
+	movl	$MSR_FRED_STKLVLS,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_STKLVLS(%rdi)
+	movl	%edx,PCB_FRED_STKLVLS+4(%rdi)
+	movl	$MSR_FRED_CONFIG,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_CONFIG(%rdi)
+	movl	%edx,PCB_FRED_CONFIG+4(%rdi)
+	jmp	2f
+
+1:	sidt	PCB_IDT(%rdi)
+2:	sgdt	PCB_GDT(%rdi)
 	sldt	PCB_LDT(%rdi)
 	str	PCB_TR(%rdi)
 
@@ -442,8 +478,35 @@ ENTRY(resumectx)
 	movq	%rax,%cr3
 
 	/* Restore descriptor tables. */
+	cmpb	$0,fred
+	jne	1f
 	lidt	PCB_IDT(%rdi)
-	lldt	PCB_LDT(%rdi)
+	jmp	2f
+1:	movl	$MSR_FRED_RSP0,%ecx
+	movl	PCB_FRED_RSP0(%rdi),%eax
+	movl	4 + PCB_FRED_RSP0(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_RSP1,%ecx
+	movl	PCB_FRED_RSP1(%rdi),%eax
+	movl	4 + PCB_FRED_RSP1(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_RSP2,%ecx
+	movl	PCB_FRED_RSP2(%rdi),%eax
+	movl	4 + PCB_FRED_RSP2(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_RSP3,%ecx
+	movl	PCB_FRED_RSP3(%rdi),%eax
+	movl	4 + PCB_FRED_RSP3(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_STKLVLS,%ecx
+	movl	PCB_FRED_STKLVLS(%rdi),%eax
+	movl	4 + PCB_FRED_STKLVLS(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_CONFIG,%ecx
+	movl	PCB_FRED_CONFIG(%rdi),%eax
+	movl	4 + PCB_FRED_CONFIG(%rdi),%edx
+	wrmsr
+2:	lldt	PCB_LDT(%rdi)
 
 #define	SDT_SYSTSS	9
 #define	SDT_SYSBSY	11

sys/amd64/amd64/db_trace.c

@@ -167,7 +167,10 @@ db_nextframe(db_addr_t *fp, db_addr_t *ip, struct thread *td)
 		    strcmp(name, "fork_trampoline") == 0 ||
 		    strcmp(name, "mchk_calltrap") == 0 ||
 		    strcmp(name, "nmi_calltrap") == 0 ||
-		    strcmp(name, "Xdblfault") == 0)
+		    strcmp(name, "Xdblfault") == 0 ||
+		    strcmp(name, "fred_events_handlers") == 0 ||
+		    strcmp(name, "user_fred_syscall") == 0 ||
+		    strcmp(name, "user_fred_non_syscall") == 0)
 			frame_type = TRAP;
 		else if (strncmp(name, "Xatpic_intr", 11) == 0 ||
 		    strncmp(name, "Xapic_isr", 9) == 0 ||

sys/amd64/amd64/exception.S

@@ -1063,6 +1063,8 @@ ENTRY(fork_trampoline)
 	movq	%rbx,%rsi		/* arg1 */
 	movq	%rsp,%rdx		/* trapframe pointer */
 	call	fork_exit
+	cmpb	$0, fred
+	jne	fred_checkast
 	jmp	doreti			/* Handle any ASTs */
 
 /*
@@ -1401,6 +1403,286 @@ gsbase_load_fault:
 	movq	$0,PCB_GSBASE(%r8)
 	jmp	doreti
 
+/*
+ * FRED guarantees that on entry, %rflags is set to '2', which in
+ * particular means that interrupts are disabled, and DF/AC are clear.
+ * Interrupts are re-enabled by the C handler, if appropriate.
+ */
+	ALIGN_TEXT
+user_fred_syscall:
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	SAVE_SEGS
+	movq	PCPU(CURPCB),%r11
+	andl	$~PCB_FULL_IRET,PCB_FLAGS(%r11)
+	sti
+	movw	$KUDSEL,TF_SS(%rsp)
+	movw	$KUCSEL,TF_CS(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	andl	$TF_FRED_EVINFO2_INSTLENMASK,%eax
+	shrl	$TF_FRED_EVINFO2_INSTLENSHIFT,%eax
+	movq	%rax,TF_ERR(%rsp)
+	movq	PCPU(CURTHREAD),%rdi
+	movl	TF_RFLAGS(%rsp),%esi
+	andl	$PSL_T,%esi
+	call	amd64_syscall
+	movq	PCPU(CURPCB),%rax
+	cli
+	testl	$PCB_FULL_IRET,PCB_FLAGS(%rax)
+	jnz	fred_checkast
+	movq	PCPU(CURTHREAD),%rax
+	cmpl	$0,TD_AST(%rax)
+	jnz	fred_checkast
+	movq	TF_RDI(%rsp),%rdi
+	movq	TF_RSI(%rsp),%rsi
+	movq	TF_RDX(%rsp),%rdx
+	movq	TF_RAX(%rsp),%rax
+	xorl	%r8d,%r8d
+	xorl	%r9d,%r9d
+	xorl	%r10d,%r10d
+	xorl	%r11d,%r11d
+	addq	$TF_ERR,%rsp
+	.globl	fred_syscall_eretu
+fred_syscall_eretu:
+	eretu
+
+	ALIGN_TEXT
+user_fred_non_syscall:
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r10,TF_R10(%rsp)
+	movq	%r11,TF_R11(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	SAVE_SEGS
+	movq	PCPU(CURTHREAD),%r9
+	movq	TD_PCB(%r9),%r8
+	rdfsbase %rax
+	movq	%rax,PCB_FSBASE(%r8)
+	rdgsbase %rax
+	movq	%rax,PCB_GSBASE(%r8)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	KMSAN_ENTER
+	movq	%rsp, %rdi
+	call	trap_fred_u
+	KMSAN_LEAVE
+fred_checkast:
+	cli
+	movq	PCPU(CURTHREAD),%rax
+	movq	TD_PCB(%rax),%r8
+	cmpl	$0,TD_AST(%rax)
+	je	fred_gouser
+	sti
+	movq	%rsp,%rdi	/* pass a pointer to the trapframe */
+	call	ast
+	jmp	fred_checkast
+fred_gouser:
+	testl	$PCB_FULL_IRET,PCB_FLAGS(%r8)
+	jz	fred_u_regs
+	andl	$~PCB_FULL_IRET,PCB_FLAGS(%r8)
+	testl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	jne	fred_do_segs
+	movw	$KUDSEL,%ax
+	movw	%ax,TF_DS(%rsp)
+	movw	%ax,TF_ES(%rsp)
+	movw	$KUF32SEL,TF_FS(%rsp)
+	movw	$KUG32SEL,TF_GS(%rsp)
+fred_do_segs:
+	/* Restore %fs and fsbase */
+	movw	TF_FS(%rsp),%ax
+	.globl	fred_ld_fs
+fred_ld_fs:
+	movw	%ax,%fs
+	testl	$PCB_32BIT,PCB_FLAGS(%r8)
+	jnz	after_fred_fs
+	movq	PCB_FSBASE(%r8),%rax
+	.globl	fred_ld_fsbase
+fred_ld_fsbase:
+	wrfsbase %rax
+after_fred_fs:
+	/* Restore %gs and kgsbase */
+	movw	TF_GS(%rsp),%ax
+	.globl	fred_lkgs
+fred_lkgs:
+	lkgs	%ax
+	testl	$PCB_32BIT,PCB_FLAGS(%r8)
+	jnz	after_fred_gs
+	movl	PCB_GSBASE(%r8),%eax
+	movl	PCB_GSBASE+4(%r8),%edx
+	movl	$MSR_KGSBASE,%ecx
+	.globl	fred_ld_kgsbase
+fred_ld_kgsbase:
+	wrmsr
+after_fred_gs:
+	.globl	fred_ld_es
+fred_ld_es:
+	movw	TF_ES(%rsp),%es
+	.globl	fred_ld_ds
+fred_ld_ds:
+	movw	TF_DS(%rsp),%ds
+fred_u_regs:
+	RESTORE_REGS
+	addq	$TF_ERR,%rsp
+	.globl	fred_eretu
+fred_eretu:
+	eretu
+
+	.globl	fred_ld_fs_fault
+fred_ld_fs_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUF32SEL,TF_FS(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_ld_fsbase_fault
+fred_ld_fsbase_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movq	PCPU(CURTHREAD),%r9
+	movq	TD_PCB(%r9),%r8
+	movq	$0,PCB_FSBASE(%r8)
+	jmp	fred_checkast
+
+	.globl	fred_lkgs_fault
+fred_lkgs_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUG32SEL,TF_GS(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_ld_kgsbase_fault
+fred_ld_kgsbase_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movq	PCPU(CURTHREAD),%r9
+	movq	TD_PCB(%r9),%r8
+	movq	$0,PCB_GSBASE(%r8)
+	jmp	fred_checkast
+
+	.globl	fred_ld_es_fault
+fred_ld_es_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUDSEL,TF_ES(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_ld_ds_fault
+fred_ld_ds_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUDSEL,TF_DS(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_eretu_fault
+fred_eretu_fault:
+	subq	$TF_ERR,%rsp
+	movq	%rdi,TF_RDI(%rsp)
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	movq	%rsi,TF_RSI(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r10,TF_R10(%rsp)
+	movq	%r11,TF_R11(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	SAVE_SEGS
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_GP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp, %rdi
+	KMSAN_ENTER
+	call	trap_fred_u
+	KMSAN_LEAVE
+	jmp	fred_checkast
+
+/*
+ * Content for the FRED entries page.
+*/
+
+	.p2align 12,0x90
+	.globl	fred_events_handlers, fred_events_handlers_end
+fred_events_handlers:
+	subq	$TF_ERR,%rsp
+	movq	%rax,TF_RAX(%rsp)
+	movq	%rdi,TF_RDI(%rsp)
+	movq	%rsi,TF_RSI(%rsp)
+	movl	TF_FRED_EVINFO2(%rsp),%edi
+	movl	%edi,%eax
+	andl	$TF_FRED_EVINFO2_TYPEMASK,%edi
+	cmpl	$TF_FRED_EVINFO2_TYPE_SYSCALL,%edi
+	je	user_fred_syscall
+	jmp	user_fred_non_syscall
+
+	. = fred_events_handlers + 256
+	subq	$TF_ERR,%rsp
+	movq	%rax,TF_RAX(%rsp)
+	movq	%rdi,TF_RDI(%rsp)
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	movq	%rsi,TF_RSI(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r10,TF_R10(%rsp)
+	movq	%r11,TF_R11(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	movq	%rsp, %rdi
+	KMSAN_ENTER
+	call	trap_fred_k
+	KMSAN_LEAVE
+	RESTORE_REGS
+	addq	$TF_ERR,%rsp
+	erets
+fred_events_handlers_end:
+
+	.globl	fred_out_of_nmi
+fred_out_of_nmi:
+	pushq	%rbp
+	movq	%rsp,%rbp
+	xorl	%eax,%eax
+	pushq	$0
+	pushq	$0
+	movw	%ss,%ax
+	orl	$(1<<18),%eax	/* NMI block release for ERETS */
+	pushq	%rax
+	pushq	%rbp
+	pushfq
+	xorl	%eax,%eax
+	movw	%cs,%ax
+	pushq	%rax
+	pushq	$1f
+	pushq	$0
+	erets
+1:	popq	%rbp
+	retq
+
 #ifdef HWPMC_HOOKS
 	ENTRY(end_exceptions)
 #endif

sys/amd64/amd64/genassym.c

@@ -146,6 +146,12 @@ ASSYM(PCB_STAR, offsetof(struct pcb, pcb_star));
 ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar));
 ASSYM(PCB_CSTAR, offsetof(struct pcb, pcb_cstar));
 ASSYM(PCB_SFMASK, offsetof(struct pcb, pcb_sfmask));
+ASSYM(PCB_FRED_RSP0, offsetof(struct pcb, pcb_fred_rsp0));
+ASSYM(PCB_FRED_RSP1, offsetof(struct pcb, pcb_fred_rsp1));
+ASSYM(PCB_FRED_RSP2, offsetof(struct pcb, pcb_fred_rsp2));
+ASSYM(PCB_FRED_RSP3, offsetof(struct pcb, pcb_fred_rsp3));
+ASSYM(PCB_FRED_STKLVLS, offsetof(struct pcb, pcb_fred_stklvls));
+ASSYM(PCB_FRED_CONFIG, offsetof(struct pcb, pcb_fred_config));
 ASSYM(PCB_SIZE, sizeof(struct pcb));
 ASSYM(PCB_FULL_IRET, PCB_FULL_IRET);
 ASSYM(PCB_DBREGS, PCB_DBREGS);
@@ -180,9 +186,15 @@ ASSYM(TF_DS, offsetof(struct trapframe, tf_ds));
 ASSYM(TF_ES, offsetof(struct trapframe, tf_es));
 ASSYM(TF_FS, offsetof(struct trapframe, tf_fs));
 ASSYM(TF_GS, offsetof(struct trapframe, tf_gs));
+ASSYM(TF_FRED_EVINFO2, offsetof(struct trapframe, tf_fred_evinfo2));
 ASSYM(TF_FLAGS, offsetof(struct trapframe, tf_flags));
 ASSYM(TF_SIZE, sizeof(struct trapframe));
 ASSYM(TF_HASSEGS, TF_HASSEGS);
+ASSYM(TF_FRED_EVINFO2_TYPE_EXC, TF_FRED_EVINFO2_TYPE_EXC);
+ASSYM(TF_FRED_EVINFO2_TYPE_SYSCALL, TF_FRED_EVINFO2_TYPE_SYSCALL);
+ASSYM(TF_FRED_EVINFO2_TYPEMASK, TF_FRED_EVINFO2_TYPEMASK);
+ASSYM(TF_FRED_EVINFO2_INSTLENMASK, TF_FRED_EVINFO2_INSTLENMASK);
+ASSYM(TF_FRED_EVINFO2_INSTLENSHIFT, TF_FRED_EVINFO2_INSTLENSHIFT);
 
 ASSYM(PTI_RDX, offsetof(struct pti_frame, pti_rdx));
 ASSYM(PTI_RAX, offsetof(struct pti_frame, pti_rax));
@@ -272,6 +284,9 @@ ASSYM(TSSSEL, GSEL(GPROC0_SEL, SEL_KPL));
 ASSYM(LDTSEL, GSEL(GUSERLDT_SEL, SEL_KPL));
 ASSYM(SEL_RPL_MASK, SEL_RPL_MASK);
 
+ASSYM(IDT_NP, IDT_NP);
+ASSYM(IDT_GP, IDT_GP);
+
 ASSYM(__FreeBSD_version, __FreeBSD_version);
 
 #ifdef	HWPMC_HOOKS

sys/amd64/amd64/initcpu.c

@@ -397,3 +397,46 @@ initializecpucache(void)
 		cpu_stdext_feature &= ~CPUID_STDEXT_CLFLUSHOPT;
 	}
 }
+
+extern const char fred_events_handlers[], fred_events_handlers_end[];
+
+void
+amd64_cpu_init_fred(void)
+{
+	MPASS((((uintptr_t)fred_events_handlers) & PAGE_MASK) == 0);
+	/*
+	 * 128b = 0x2 * 64b redzone
+	 * SLC 0 for EXTINT
+	 * SLC 0 current
+	 */
+	wrmsr(MSR_FRED_CONFIG, (uintptr_t)fred_events_handlers |
+	    (0x2 << IA32_FRED_CONFIG_REDZONESZ_SHIFT) |
+	    (0 << IA32_FRED_CONFIG_EXTINT_SLC_SHIFT) | 0);
+
+	/*
+	 * With FRED, there is no need for workarounds requiring
+	 * dedicated stack for #DB.  Also, there is no need to have
+	 * separate stacks for #NM and #MC, because CSL rules ensure
+	 * that interrupted frame is safe even if NMI was interrupted
+	 * by machine check or vice versa.
+	 *
+	 * Still, we need to provide a stack for events that can occur
+	 * with the interrupts disabled, to avoid caring about the
+	 * ordering of the context update in cpu_switch.S.  Use CSL 1
+	 * for #NM and #MC.
+	 *
+	 * Since double fault handler is used to handle stack
+	 * overflow, it gets the dedicated stack, of the highest used
+	 * CSL 2.
+	 *
+	 * CSL 3 is unused, and we have some space to re-arrange this
+	 * if a need in the future appears.
+	 */
+	wrmsr(MSR_FRED_STKLVLS, (1ul << (IDT_NM * 2)) |
+	    (1ul << (IDT_MC * 2)) | (2ul << (IDT_DF * 2)));
+	wrmsr(MSR_FRED_RSP3, 0);
+	load_cr4(rcr4() | CR4_FRED);
+	if (IS_BSP() && bootverbose)
+		printf("FRED enabled\n");
+
+}

sys/amd64/amd64/machdep.c

@@ -206,10 +206,14 @@ long realmem = 0;
 int late_console = 1;
 int lass_enabled = 0;
 
+int __read_frequently fred = 0;
+SYSCTL_INT(_hw, OID_AUTO, fred, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+    &fred, 0,
+    "FRED is used");
+
 struct kva_md_info kmi;
 
 struct region_descriptor r_idt;
-
 struct pcpu *__pcpu;
 struct pcpu temp_bsp_pcpu;
 
@@ -347,9 +351,9 @@ cpu_setregs(void)
 static struct gate_descriptor idt0[NIDT];
 struct gate_descriptor *idt = &idt0[0];	/* interrupt descriptor table */
 
-static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(16);
+static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(64);
 static char mce0_stack[MCE_STACK_SIZE] __aligned(16);
-static char nmi0_stack[NMI_STACK_SIZE] __aligned(16);
+static char nmi0_stack[NMI_STACK_SIZE] __aligned(64);
 static char dbg0_stack[DBG_STACK_SIZE] __aligned(16);
 CTASSERT(sizeof(struct nmi_pcpu) == 16);
 
@@ -485,6 +489,9 @@ setidt(int idx, inthand_t *func, int typ, int dpl, int ist)
 {
 	struct gate_descriptor *ip;
 
+	if (fred)
+		return;
+
 	ip = idt + idx;
 	ip->gd_looffset = (uintptr_t)func;
 	ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
@@ -1186,9 +1193,11 @@ amd64_conf_fast_syscall(void)
 
 	msr = rdmsr(MSR_EFER) | EFER_SCE;
 	wrmsr(MSR_EFER, msr);
+	if (!fred) {
 		wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
 		    (u_int64_t)IDTVEC(fast_syscall));
 		wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+	}
 	msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
 	    ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
 	wrmsr(MSR_STAR, msr);
@@ -1230,33 +1239,49 @@ amd64_bsp_ist_init(struct pcpu *pc)
 
 	tssp = &pc->pc_common_tss;
 
-	/* doublefault stack space, runs on ist1 */
-	np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(dblfault_stack)]) - 1;
+	/* Doublefault stack space, runs on ist1 for IDT. */
+	if (fred) {
+		wrmsr(MSR_FRED_RSP2, (uint64_t)&dblfault_stack[
+		    sizeof(dblfault_stack)]);
+	} else {
+		np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(
+		    dblfault_stack)]) - 1;
 		np->np_pcpu = (register_t)pc;
 		tssp->tss_ist1 = (long)np;
+	}
 
 	/*
-	 * NMI stack, runs on ist2.  The pcpu pointer is stored just
-	 * above the start of the ist2 stack.
+	 * NMI stack.
+	 */
+	if (fred) {
+		wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi0_stack[
+		    sizeof(nmi0_stack)]);
+	} else {
+		/*
+		 * Runs on ist2 for IDT.  The pcpu pointer is stored
+		 * just above the start of the ist2 stack.
 		 */
 		np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
 		np->np_pcpu = (register_t)pc;
 		tssp->tss_ist2 = (long)np;
+	}
 
+	if (!fred) {
 		/*
-	 * MC# stack, runs on ist3.  The pcpu pointer is stored just
-	 * above the start of the ist3 stack.
+		 * MC# stack for IDT, runs on ist3.  The pcpu pointer
+		 * is stored just above the start of the ist3 stack.
 		 */
 		np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
 		np->np_pcpu = (register_t)pc;
 		tssp->tss_ist3 = (long)np;
 
 		/*
-	 * DB# stack, runs on ist4.
+		 * DB# stack for IDT, runs on ist4.
 		 */
 		np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
 		np->np_pcpu = (register_t)pc;
 		tssp->tss_ist4 = (long)np;
+	}
 }
 
 /*
@@ -1359,6 +1384,13 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 		TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
 	}
 
+	if ((cpu_stdext_feature4 & (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS)) ==
+	    (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS) &&
+	    (cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 && !pti) {
+		fred = 1;
+		TUNABLE_INT_FETCH("hw.fred", &fred);
+	}
+
 	sched_instance_select();
 
 	link_elf_ireloc();
@@ -1473,9 +1505,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
 	    &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
 #endif
+	if (!fred) {
 		r_idt.rd_limit = sizeof(idt0) - 1;
 		r_idt.rd_base = (long) idt;
 		lidt(&r_idt);
+	}
 
 	TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
 	TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable);
@@ -1564,6 +1598,9 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	if (getenv_is_true("debug.dump_modinfo_at_boot"))
 		preload_dump();
 
+	if (fred)
+		amd64_cpu_init_fred();
+
 #ifdef DEV_ISA
 #ifdef DEV_ATPIC
 	elcr_probe();
@@ -1828,6 +1865,11 @@ clear_pcb_flags(struct pcb *pcb, const u_int flags)
 
 extern const char wrmsr_early_safe_gp_handler[];
 
+/*
+ * What about FRED?  wrmsr_early_safe_start() is used before we
+ * switched CPU to the FRED mode.  We use IDT to catch #GP from MSR
+ * write even if BSP is switched to the FRED mode later.
+ */
 void
 wrmsr_early_safe_start(void)
 {

sys/amd64/amd64/mp_machdep.c

@@ -240,25 +240,51 @@ init_secondary(void)
 	    IOPERM_BITMAP_SIZE;
 	pc->pc_common_tss.tss_rsp0 = 0;
 
-	/* The doublefault stack runs on IST1. */
-	np = ((struct nmi_pcpu *)&doublefault_stack[DBLFAULT_STACK_SIZE]) - 1;
+	/*
+	 * The doublefault stack.
+	 * Runs on IST1 for IDT.
+	 * Uses CSL 2 for FRED.
+	 */
+	if (fred) {
+		wrmsr(MSR_FRED_RSP2, (uint64_t)&doublefault_stack[
+		    DBLFAULT_STACK_SIZE]);
+	} else {
+		np = ((struct nmi_pcpu *)&doublefault_stack[
+		    DBLFAULT_STACK_SIZE]) - 1;
 		np->np_pcpu = (register_t)pc;
 		pc->pc_common_tss.tss_ist1 = (long)np;
+	}
 
-	/* The NMI stack runs on IST2. */
+	/*
+	 * The NMI stack.
+	 * Runs on IST2 for IDT.
+	 * Uses CSL 1 for FRED.
+	 */
+	if (fred) {
+		wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi_stack[NMI_STACK_SIZE]);
+	} else {
 		np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
 		np->np_pcpu = (register_t)pc;
 		pc->pc_common_tss.tss_ist2 = (long)np;
+	}
 
-	/* The MC# stack runs on IST3. */
+	/*
+	 * The MC# stack.
+	 * Runs on IST3 for IDT.
+	 * Shares CSL 1 with NMI for FRED.
+	 */
+	if (!fred) {
 		np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
 		np->np_pcpu = (register_t)pc;
 		pc->pc_common_tss.tss_ist3 = (long)np;
+	}
 
-	/* The DB# stack runs on IST4. */
+	if (!fred) {
+		/* The DB# stack, used for for IDT, runs on IST4. */
 		np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
 		np->np_pcpu = (register_t)pc;
 		pc->pc_common_tss.tss_ist4 = (long)np;
+	}
 
 	/* Prepare private GDT */
 	gdt_segs[GPROC0_SEL].ssd_base = (long)&pc->pc_common_tss;
@@ -278,6 +304,9 @@ init_secondary(void)
 	wrmsr(MSR_KGSBASE, 0);		/* User value */
 	fix_cpuid();
 
+	if (fred)
+		amd64_cpu_init_fred();
+	else
 		lidt(&r_idt);
 
 	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
@@ -430,12 +459,17 @@ start_all_aps(void)
 		    M_WAITOK | M_ZERO);
 		doublefault_stack = kmem_malloc(DBLFAULT_STACK_SIZE,
 		    M_WAITOK | M_ZERO);
+		if (!fred) {
 			mce_stack = kmem_malloc(MCE_STACK_SIZE,
 			    M_WAITOK | M_ZERO);
+		}
 		nmi_stack = kmem_malloc_domainset(
 		    DOMAINSET_PREF(domain), NMI_STACK_SIZE, M_WAITOK | M_ZERO);
+		if (!fred) {
 			dbg_stack = kmem_malloc_domainset(
-		    DOMAINSET_PREF(domain), DBG_STACK_SIZE, M_WAITOK | M_ZERO);
+			    DOMAINSET_PREF(domain), DBG_STACK_SIZE,
+			    M_WAITOK | M_ZERO);
+		}
 		dpcpu = kmem_malloc_domainset(DOMAINSET_PREF(domain),
 		    DPCPU_SIZE, M_WAITOK | M_ZERO);
 

sys/amd64/amd64/trap.c

@@ -41,6 +41,7 @@
  * AMD64 Trap and System call handling
  */
 
+#include "opt_atpic.h"
 #include "opt_clock.h"
 #include "opt_cpu.h"
 #include "opt_hwpmc_hooks.h"
@@ -61,6 +62,7 @@
 #include <sys/mutex.h>
 #include <sys/resourcevar.h>
 #include <sys/signalvar.h>
+#include <sys/smp.h>
 #include <sys/syscall.h>
 #include <sys/sysctl.h>
 #include <sys/sysent.h>
@@ -83,6 +85,8 @@ PMC_SOFT_DEFINE( , , page_fault, write);
 
 #include <machine/cpu.h>
 #include <machine/intr_machdep.h>
+#include <x86/apicreg.h>
+#include <x86/apicvar.h>
 #include <x86/mca.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
@@ -90,6 +94,7 @@ PMC_SOFT_DEFINE( , , page_fault, write);
 #include <machine/stack.h>
 #include <machine/trap.h>
 #include <machine/tss.h>
+#include <x86/isa/icu.h>
 
 #ifdef KDTRACE_HOOKS
 #include <sys/dtrace_bsd.h>
@@ -995,6 +1000,14 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str)
 	    frame->tf_r11, frame->tf_r12);
 	printf("r13: %016lx r14: %016lx r15: %016lx\n", frame->tf_r13,
 	    frame->tf_r14, frame->tf_r15);
+	if (fred) {
+		struct trapframe_fred *f;
+
+		f = __containerof(frame, struct trapframe_fred, tf_idt);
+		printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+		    f->tf_fred_evdata, frame->tf_fred_evinfo1,
+		    frame->tf_fred_evinfo2);
+	}
 }
 
 static void
@@ -1069,6 +1082,14 @@ dblfault_handler(struct trapframe *frame)
 	    frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
 	    frame->tf_fs, frame->tf_gs,
 	    rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
+	if (fred) {
+		struct trapframe_fred *f;
+
+		f = __containerof(frame, struct trapframe_fred, tf_idt);
+		printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+		    f->tf_fred_evdata, frame->tf_fred_evinfo1,
+		    frame->tf_fred_evinfo2);
+	}
 	/* Print these separately in case pcpu accesses trap. */
 	printf("cpuid = %d; apic id = %02x\n", PCPU_GET(cpuid),
 	    PCPU_GET(apic_id));
@@ -1295,3 +1316,540 @@ amd64_syscall(struct thread *td, int traced)
 
 	amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
 }
+
+static void
+trap_fred_extint(struct trapframe_fred *frame, u_int vec)
+{
+	switch (vec) {
+	case IPI_RENDEZVOUS:
+#ifdef COUNT_IPIS
+		ipi_rendezvous_counts[PCPU_GET(cpuid)]++;
+#endif
+		smp_rendezvous_action();
+		lapic_eoi();
+		break;
+	case IPI_INVLOP:
+		invlop_handler();
+		lapic_eoi();
+		break;
+	case IPI_BITMAP_VECTOR:
+		lapic_eoi();
+		ipi_bitmap_handler(&frame->tf_idt);
+		break;
+	case IPI_STOP:
+		lapic_eoi();
+		cpustop_handler();
+		break;
+	case IPI_SUSPEND:
+		cpususpend_handler();
+		lapic_eoi();
+		break;
+	case IPI_SWI:
+		lapic_eoi();
+		ipi_swi_handler(&frame->tf_idt);
+		break;
+	case IPI_OFF:
+		cpuoff_handler();
+		lapic_eoi();
+		break;
+	case APIC_SPURIOUS_INT:
+		break;
+	case APIC_TIMER_INT:
+		lapic_handle_timer(&frame->tf_idt);
+		break;
+	case APIC_CMC_INT:
+		lapic_handle_cmc();
+		break;
+	case APIC_ERROR_INT:
+		lapic_handle_error();
+		break;
+	case APIC_THERMAL_INT:
+		lapic_handle_thermal();
+		break;
+#ifdef XENHVM
+	case IDT_EVTCHN:
+		xen_arch_intr_handle_upcall(&frame->tf_idt);
+		break;
+#endif
+	default:
+		if (vec >= IPI_DYN_FIRST && vec <= IPI_DYN_LAST) {
+			void (*fi)(struct trapframe *);
+
+			fi = fred_ipi_handlers[vec - IPI_DYN_FIRST];
+			if (fi != NULL) {
+				fi(&frame->tf_idt);
+				lapic_eoi();
+			} else {
+				panic("DYN IPI %d without FRED handler", vec);
+			}
+			break;
+		}
+#ifdef DEV_ISA
+#ifdef DEV_ATPIC
+		if (vec >= IDT_IO_INTS && vec < IDT_IO_INTS + NUM_ISA_IRQS) {
+			atpic_handle_intr(vec - IDT_IO_INTS, &frame->tf_idt);
+			break;
+		}
+#endif
+#endif
+		/* apic ioint */
+		lapic_handle_intr(vec, &frame->tf_idt);
+		break;
+	}
+}
+
+void fred_out_of_nmi(void);
+
+static const int fred_ev_to_trapno_table[] = {
+	[IDT_DE] =	T_DIVIDE,
+	[IDT_DB] =	T_TRCTRAP,
+	[IDT_NMI] =	T_RESERVED,
+	[IDT_BP] =	T_BPTFLT,
+	[IDT_OF] =	T_OFLOW,
+	[IDT_BR] =	T_BOUND,
+	[IDT_UD] =	T_PRIVINFLT,
+	[IDT_NM] =	T_DNA,
+	[IDT_DF] =	T_DOUBLEFLT,
+	[IDT_FPUGP] =	T_FPOPFLT,
+	[IDT_TS] =	T_TSSFLT,
+	[IDT_NP] =	T_SEGNPFLT,
+	[IDT_SS] =	T_STKFLT,
+	[IDT_GP] =	T_PROTFLT,
+	[IDT_PF] =	T_PAGEFLT,
+	[15] =		T_RESERVED,
+	[IDT_MF] =	T_ARITHTRAP,
+	[IDT_AC] =	T_ALIGNFLT,
+	[IDT_MC] =	T_MCHK,
+	[IDT_XF] =	T_XMMFLT,
+	[20] =		T_RESERVED,
+	[21] =		T_RESERVED,
+	[22] =		T_RESERVED,
+	[23] =		T_RESERVED,
+	[24] =		T_RESERVED,
+	[25] =		T_RESERVED,
+	[26] =		T_RESERVED,
+	[27] =		T_RESERVED,
+	[28] =		T_RESERVED,
+	[29] =		T_RESERVED,
+	[30] =		T_RESERVED,
+	[31] =		T_RESERVED,
+};
+
+static void
+trap_fred_ev_to_trapno(struct trapframe_fred *frame, u_int exc)
+{
+	frame->tf_idt.tf_trapno = fred_ev_to_trapno_table[exc];
+}
+
+static void
+trap_fred_handle_u_exc(struct thread *td, struct trapframe_fred *frame,
+    u_int exc, ksiginfo_t *ksi, bool *do_trapsig)
+{
+	struct proc *p;
+	register_t addr;
+	int pf, signo, ucode;
+
+	addr = frame->tf_idt.tf_rip;
+	signo = 0;
+	ucode = 0;
+	frame->tf_idt.tf_addr = 0;
+	trap_check_intr_user(td, &frame->tf_idt);
+
+	switch (exc) {
+	case IDT_DE:
+		ucode = FPE_INTDIV;
+		signo = SIGFPE;
+		break;
+	case IDT_DB:
+		signo = SIGTRAP;
+		ucode = TRAP_TRACE;
+		if ((frame->tf_fred_evdata & TF_FRED_EVDATA_BS) != 0)
+			trap_clear_step(td, &frame->tf_idt);
+		break;
+	case IDT_BP:
+#ifdef KDTRACE_HOOKS
+		if (trap_user_dtrace(&frame->tf_idt, &dtrace_pid_probe_ptr))
+			return;
+#endif
+		signo = SIGTRAP;
+		ucode = TRAP_BRKPT;
+		break;
+	case IDT_OF:
+		ucode = FPE_INTOVF;
+		signo = SIGFPE;
+		break;
+	case IDT_BR:
+		ucode = FPE_FLTSUB;
+		signo = SIGFPE;
+		break;
+	case IDT_UD:
+		signo = SIGILL;
+		ucode = ILL_PRVOPC;
+		break;
+	case IDT_NM:
+		KASSERT(PCB_USER_FPU(td->td_pcb),
+		    ("kernel FPU ctx has leaked"));
+		fpudna();
+		return;
+	case IDT_DF:
+		signo = SIGBUS;
+		ucode = BUS_OBJERR;
+		break;
+	case IDT_FPUGP:
+		ucode = ILL_COPROC;
+		signo = SIGILL;
+		break;
+	case IDT_TS:
+		signo = SIGBUS;
+		ucode = BUS_OBJERR;
+		break;
+	case IDT_NP:
+		signo = SIGBUS;
+		ucode = BUS_ADRERR;
+		break;
+	case IDT_SS:
+		signo = SIGBUS;
+		ucode = BUS_ADRERR;
+		break;
+	case IDT_GP:
+		signo = SIGBUS;
+		ucode = BUS_OBJERR;
+		break;
+	case IDT_PF:
+		p = td->td_proc;
+		frame->tf_idt.tf_addr = addr = frame->tf_fred_evdata;
+		if (*p->p_sysent->sv_trap != NULL &&
+		    (*p->p_sysent->sv_trap)(td) == 0)
+			return;
+		pf = trap_pfault(&frame->tf_idt, true, &signo, &ucode);
+		if (pf == -1 || pf == 0)
+			return;
+		break;
+	case IDT_MF:
+		ucode = fputrap_x87();
+		if (ucode == -1)
+			return;
+		signo = SIGFPE;
+		break;
+	case IDT_AC:
+		signo = SIGBUS;
+		ucode = BUS_ADRALN;
+		break;
+	case IDT_MC:
+		mca_intr();
+		return;
+	case IDT_XF:
+		ucode = fputrap_sse();
+		if (ucode == -1)
+			return;
+		signo = SIGFPE;
+		break;
+	default:
+		panic("missed FRED handler for exception %d", exc);
+	}
+
+	ksiginfo_init_trap(ksi);
+	ksi->ksi_signo = signo;
+	ksi->ksi_code = ucode;
+	ksi->ksi_trapno = frame->tf_idt.tf_trapno;
+	ksi->ksi_addr = (void *)addr;
+	trap_uprintf_signal(td, &frame->tf_idt, addr, signo, ucode);
+	KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
+	*do_trapsig = true;
+}
+
+static u_int
+fred_inst_len(const struct trapframe_fred *frame)
+{
+	return ((frame->tf_idt.tf_fred_evinfo2 & TF_FRED_EVINFO2_INSTLENMASK) >>
+	    TF_FRED_EVINFO2_INSTLENSHIFT);
+}
+
+int fred_verbose_events;
+
+void trap_fred_u(struct trapframe_fred *frame);
+void
+trap_fred_u(struct trapframe_fred *frame)
+{
+	struct thread *td;
+	ksiginfo_t ksi;
+	u_int type, vec;
+	bool do_trapsig;
+
+	td = curthread;
+	type = frame->tf_idt.tf_fred_evinfo2 & TF_FRED_EVINFO2_TYPEMASK;
+	vec = frame->tf_idt.tf_fred_evinfo2 & TF_FRED_EVINFO2_VECMASK;
+	do_trapsig = false;
+	if (fred_verbose_events) {
+		printf("pid %d tid %d type %d vec %d rax %#lx\n",
+		    td->td_proc->p_pid, td->td_tid,
+		    (frame->tf_idt.tf_fred_evinfo2 &
+		    TF_FRED_EVINFO2_TYPEMASK) >> 16,
+		    vec, frame->tf_idt.tf_rax);
+	}
+	if (__predict_true((td->td_proc->p_sysent->sv_flags &
+	    (SV_ABI_MASK | SV_LP64)) == (SV_ABI_FREEBSD | SV_LP64) &&
+	    (type == TF_FRED_EVINFO2_TYPE_SYSCALL ||
+	    type == TF_FRED_EVINFO2_TYPE_EXTINT ||
+	    (type == TF_FRED_EVINFO2_TYPE_EXC && vec == IDT_PF))))
+		clear_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+	else
+		set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+	switch (type) {
+	case TF_FRED_EVINFO2_TYPE_EXTINT:
+		trap_fred_extint(frame, vec);
+		break;
+	case TF_FRED_EVINFO2_TYPE_NMI:
+		frame->tf_idt.tf_trapno = T_NMI;
+		nmi_handle_intr(&frame->tf_idt);
+#ifdef HWPMC_HOOKS
+		if ((td->td_pflags & TDP_CALLCHAIN) != 0) {
+			int (*ph)(struct thread *, int, void *);
+
+			fred_out_of_nmi();
+			ph = atomic_load_ptr(&pmc_hook);
+			if (ph != NULL) {
+				enable_intr();
+				ph(td, PMC_FN_USER_CALLCHAIN, &frame->tf_idt);
+			}
+		}
+#endif
+		break;
+	case TF_FRED_EVINFO2_TYPE_INT1:
+	case TF_FRED_EVINFO2_TYPE_INT3:
+	case TF_FRED_EVINFO2_TYPE_EXC:
+		enable_intr();
+		trap_fred_ev_to_trapno(frame, vec);
+#ifdef KDTRACE_HOOKS
+		if (dtrace_trap_func != NULL && (*dtrace_trap_func)(
+		    &frame->tf_idt, frame->tf_idt.tf_trapno) != 0)
+			return;
+#endif
+		td->td_pticks = 0;
+		td->td_frame = &frame->tf_idt;
+		if (td->td_cowgen != atomic_load_int(&td->td_proc->p_cowgen))
+			thread_cow_update(td);
+		trap_fred_handle_u_exc(td, frame, vec, &ksi, &do_trapsig);
+		break;
+	case TF_FRED_EVINFO2_TYPE_INTn:
+		enable_intr();
+#ifdef COMPAT_FREEBSD32
+		if (vec == IDT_SYSCALL) {
+			frame->tf_idt.tf_err = fred_inst_len(frame);
+			ia32_syscall(&frame->tf_idt);
+			break;
+		}
+#endif
+		frame->tf_idt.tf_trapno = vec;
+		ksiginfo_init_trap(&ksi);
+		ksi.ksi_signo = SIGBUS;
+		ksi.ksi_code = BUS_OBJERR;
+		ksi.ksi_trapno = frame->tf_idt.tf_trapno;
+		ksi.ksi_addr = (void *)frame->tf_idt.tf_rip;
+		do_trapsig = true;
+		break;
+	case TF_FRED_EVINFO2_TYPE_SYSCALL:
+		/* Excessive */
+		enable_intr();
+		if (vec == TF_FRED_EVINFO2_VEC_SYSCALL) {
+			frame->tf_idt.tf_err = fred_inst_len(frame);
+			td->td_frame = &frame->tf_idt;
+			amd64_syscall(td, (frame->tf_idt.tf_rflags & PSL_T) !=
+			    0);
+		} else {
+			/* SYSENTER returns without action */
+		}
+		break;
+	default:
+		__unreachable();
+	}
+	if (do_trapsig)
+		trapsignal(td, &ksi);
+	userret(td, &frame->tf_idt);
+	KASSERT(PCB_USER_FPU(td->td_pcb),
+	    ("Return from trap with kernel FPU ctx leaked"));
+}
+
+extern const char fred_ld_fs[];
+extern const char fred_ld_fs_fault[];
+extern const char fred_ld_fsbase[];
+extern const char fred_ld_fsbase_fault[];
+extern const char fred_lkgs[];
+extern const char fred_lkgs_fault[];
+extern const char fred_ld_kgsbase[];
+extern const char fred_ld_kgsbase_fault[];
+extern const char fred_ld_es[];
+extern const char fred_ld_es_fault[];
+extern const char fred_ld_ds[];
+extern const char fred_ld_ds_fault[];
+extern const char fred_eretu[];
+extern const char fred_syscall_eretu[];
+extern const char fred_eretu_fault[];
+
+static const struct sfhandler fred_sfhandlers[] = {
+	{
+		.faddr = (uintptr_t)fred_ld_fs,
+		.fhandler = (uintptr_t)fred_ld_fs_fault,
+	},
+	{
+		.faddr = (uintptr_t)fred_ld_fsbase,
+		.fhandler = (uintptr_t)fred_ld_fsbase_fault,
+	},
+	{
+		.faddr = (uintptr_t)fred_lkgs,
+		.fhandler = (uintptr_t)fred_lkgs_fault,
+	},
+	{
+		.faddr = (uintptr_t)fred_ld_kgsbase,
+		.fhandler = (uintptr_t)fred_ld_kgsbase_fault,
+	},
+	{
+		.faddr = (uintptr_t)fred_ld_es,
+		.fhandler = (uintptr_t)fred_ld_es_fault,
+	},
+	{
+		.faddr = (uintptr_t)fred_ld_ds,
+		.fhandler = (uintptr_t)fred_ld_ds_fault,
+	},
+	{
+		.faddr = (uintptr_t)fred_eretu,
+		.fhandler = (uintptr_t)fred_eretu_fault,
+	},
+	{
+		.faddr = (uintptr_t)fred_syscall_eretu,
+		.fhandler = (uintptr_t)fred_eretu_fault,
+	},
+};
+
+static void
+trap_fred_handle_k_exc(struct thread *td, struct trapframe_fred *frame,
+    u_int exc)
+{
+	register_t addr;
+	int i;
+
+	addr = frame->tf_idt.tf_rip;
+	frame->tf_idt.tf_addr = 0;
+	KASSERT(cold || td->td_ucred != NULL,
+	    ("kernel trap doesn't have ucred"));
+
+	/*
+	 * FRED exception handler runs with the interrupts disabled,
+	 * unlike the IDT case.  Only for page faults the interrupts
+	 * are enabled, since this is the only legitimate exception
+	 * from the kernel mode.  Anything else results in panic, so
+	 * there is no sense to enable interrupts.
+	 */
+
+	if (exc != IDT_PF && trap_check_pcb_onfault(td, &frame->tf_idt))
+		return;
+
+	switch (exc) {
+	case IDT_DE:
+		break;
+	case IDT_DB:
+		if (user_dbreg_trap(frame->tf_fred_evdata))
+			return;
+		/* FALLTHROUGH */
+	case IDT_BP:
+#ifdef KDB
+		if (kdb_trap(frame->tf_idt.tf_trapno, frame->tf_fred_evdata,
+		    &frame->tf_idt))
+			return;
+#endif
+		break;
+	case IDT_OF:
+		break;
+	case IDT_BR:
+		break;
+	case IDT_UD:
+		break;
+	case IDT_NM:
+		if (PCB_USER_FPU(td->td_pcb))
+			panic("Unregistered use of FPU in kernel");
+		fpudna();
+		return;
+	case IDT_DF:
+		dblfault_handler(&frame->tf_idt);
+		break;
+	case IDT_FPUGP:
+		break;
+	case IDT_TS:
+		/*
+		 * PSL_NT is cleared automatically by FRED entry.
+		 */
+		break;
+	case IDT_NP:
+	case IDT_SS:
+	case IDT_GP:
+		if (td->td_intr_nesting_level != 0)
+			break;
+		for (i = 0; i < nitems(fred_sfhandlers); i++) {
+			if (frame->tf_idt.tf_rip == fred_sfhandlers[i].faddr) {
+				KASSERT((read_rflags() & PSL_I) == 0,
+				    ("interrupts enabled"));
+				frame->tf_idt.tf_rip = fred_sfhandlers[i].
+				    fhandler;
+				return;
+			}
+		}
+		if (curpcb->pcb_onfault != NULL) {
+			frame->tf_idt.tf_rip = (long)curpcb->pcb_onfault;
+			return;
+		}
+		break;
+	case IDT_PF:
+		frame->tf_idt.tf_addr = addr = frame->tf_fred_evdata;
+		if ((frame->tf_idt.tf_rflags & PSL_I) != 0)
+			enable_intr();
+		trap_pfault(&frame->tf_idt, false, NULL, NULL);
+		return;
+	case IDT_MF:
+		break;
+	case IDT_AC:
+		break;
+	case IDT_MC:
+		mca_intr();
+		return;
+	case IDT_XF:
+		break;
+	default:
+		printf("Missing FRED handler for exception %d", exc);
+		trap_fatal(&frame->tf_idt, 0);
+	}
+	trap_fatal(&frame->tf_idt, 0);
+}
+
+void trap_fred_k(struct trapframe_fred *frame);
+void
+trap_fred_k(struct trapframe_fred *frame)
+{
+	struct thread *td;
+	u_int vec;
+
+	td = curthread;
+	vec = frame->tf_idt.tf_fred_evinfo2 & TF_FRED_EVINFO2_VECMASK;
+	switch (frame->tf_idt.tf_fred_evinfo2 & TF_FRED_EVINFO2_TYPEMASK) {
+	case TF_FRED_EVINFO2_TYPE_EXTINT:
+		trap_fred_extint(frame, vec);
+		break;
+	case TF_FRED_EVINFO2_TYPE_NMI:
+		frame->tf_idt.tf_trapno = T_NMI;
+		nmi_handle_intr(&frame->tf_idt);
+		break;
+	case TF_FRED_EVINFO2_TYPE_INT1:
+	case TF_FRED_EVINFO2_TYPE_INT3:
+	case TF_FRED_EVINFO2_TYPE_EXC:
+		trap_fred_ev_to_trapno(frame, vec);
+		trap_fred_handle_k_exc(td, frame, vec);
+		break;
+	case TF_FRED_EVINFO2_TYPE_INTn:
+		panic("INT%d from kernel", vec);
+		break;
+	case TF_FRED_EVINFO2_TYPE_SYSCALL:
+		panic("syscall from kernel");
+		break;
+	default:
+		__unreachable();
+	}
+}

sys/amd64/amd64/vm_machdep.c

@@ -378,7 +378,15 @@ void
 cpu_thread_new_kstack(struct thread *td)
 {
 	td->td_md.md_stack_base = td_kstack_top(td);
-	td->td_frame = (struct trapframe *)td->td_md.md_stack_base - 1;
+	if (fred) {
+		struct trapframe_fred *f;
+
+		f = (struct trapframe_fred *)td->td_md.md_stack_base - 1;
+		td->td_frame = &f->tf_idt;
+	} else {
+		td->td_frame = (struct trapframe *)td->td_md.
+		    md_stack_base - 1;
+	}
 }
 
 void

sys/amd64/include/md_var.h

@@ -65,12 +65,14 @@ struct	sysentvec;
 struct trapframe;
 
 void	amd64_conf_fast_syscall(void);
+void	amd64_cpu_init_fred(void);
 void	amd64_db_resume_dbreg(void);
 vm_paddr_t amd64_loadaddr(void);
 void	amd64_lower_shared_page(struct sysentvec *);
 void	amd64_bsp_pcpu_init1(struct pcpu *pc);
 void	amd64_bsp_pcpu_init2(uint64_t rsp0);
 void	amd64_bsp_ist_init(struct pcpu *pc);
+void	amd64_bsp_fred_csl_init(struct pcpu *pc);
 void	amd64_syscall(struct thread *td, int traced);
 void	amd64_syscall_ret_flush_l1d(int error);
 void	amd64_syscall_ret_flush_l1d_recalc(void);

sys/amd64/include/pcb.h

@@ -102,11 +102,17 @@ struct pcb {
 	register_t	pcb_lstar;
 	register_t	pcb_cstar;
 	register_t	pcb_sfmask;
+	/* FRED MSRs */
+	register_t	pcb_fred_rsp0;
+	register_t	pcb_fred_rsp1;
+	register_t	pcb_fred_rsp2;
+	register_t	pcb_fred_rsp3;
+	register_t	pcb_fred_stklvls;
+	register_t	pcb_fred_config;
 
 	struct savefpu	*pcb_save;
 
 	register_t	pcb_tlsbase;	/* not same as pcb_fsbase */
-	uint64_t	pcb_pad[4];
 };
 
 /* Per-CPU state saved during suspend and resume. */

sys/amd64/vmm/intel/vmx.c

@@ -894,7 +894,7 @@ vmx_modinit(int ipinum)
 		    &tmp);
 		if (error == 0) {
 			pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
-			    &IDTVEC(justreturn));
+			    &IDTVEC(justreturn), vmm_justreturn);
 			if (pirvec < 0) {
 				if (bootverbose) {
 					printf("vmx_modinit: unable to "

sys/amd64/vmm/vmm.c

@@ -287,7 +287,7 @@ vmm_modinit(void)
 	vmm_host_state_init();
 
 	vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
-	    &IDTVEC(justreturn));
+	    &IDTVEC(justreturn), vmm_justreturn);
 	if (vmm_ipinum < 0)
 		vmm_ipinum = IPI_AST;
 

sys/amd64/vmm/vmm_lapic.c

@@ -237,3 +237,8 @@ lapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, int size,
 	error = vlapic_read(vlapic, 1, off, rval, arg);
 	return (error);
 }
+
+void
+vmm_justreturn(struct trapframe *frame)
+{
+}

sys/amd64/vmm/vmm_lapic.h

@@ -71,4 +71,6 @@ int	lapic_set_local_intr(struct vm *vm, struct vcpu *vcpu, int vector);
 
 int	lapic_intr_msi(struct vm *vm, uint64_t addr, uint64_t msg);
 
+void	vmm_justreturn(struct trapframe *);
+
 #endif

sys/dev/hyperv/vmbus/x86/vmbus_x86.c

@@ -151,7 +151,8 @@ vmbus_setup_intr1(struct vmbus_softc *sc)
 	 * free IDT vector for Hyper-V ISR and set it up.
 	 */
 	sc->vmbus_idtvec = lapic_ipi_alloc(
-	    pti ? IDTVEC(vmbus_isr_pti) : IDTVEC(vmbus_isr));
+	    pti ? IDTVEC(vmbus_isr_pti) : IDTVEC(vmbus_isr),
+	    vmbus_handle_intr);
 	if (sc->vmbus_idtvec < 0) {
 #if defined(__amd64__) && defined(KLD_MODULE)
 		pmap_pti_remove_kva(VMBUS_ISR_ADDR, VMBUS_ISR_ADDR + PAGE_SIZE);

sys/i386/i386/machdep.c

@@ -187,6 +187,8 @@ struct mtx icu_lock;
 
 struct mem_range_softc mem_range_softc;
 
+int fred;
+
 extern char start_exceptions[], end_exceptions[];
 
 extern struct sysentvec elf32_freebsd_sysvec;

sys/kern/kern_thread.c

@@ -92,7 +92,7 @@ _Static_assert(offsetof(struct thread, td_pflags) == 0x114,
     "struct thread KBI td_pflags");
 _Static_assert(offsetof(struct thread, td_frame) == 0x4e8,
     "struct thread KBI td_frame");
-_Static_assert(offsetof(struct thread, td_emuldata) == 0x6f0,
+_Static_assert(offsetof(struct thread, td_emuldata) == 0x700,
     "struct thread KBI td_emuldata");
 _Static_assert(offsetof(struct proc, p_flag) == 0xb8,
     "struct proc KBI p_flag");

sys/x86/include/apicvar.h

@@ -253,7 +253,8 @@ lapic_ipi_vectored(u_int vector, int dest)
 }
 
 int	lapic_ipi_wait(int delay);
-int	lapic_ipi_alloc(inthand_t *ipifunc);
+int	lapic_ipi_alloc(inthand_t *ipifunc,
+	    void (*fred_ipifunc)(struct trapframe *));
 void	lapic_ipi_free(int vector);
 int	lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked);
 int	lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode);

sys/x86/include/frame.h

@@ -200,7 +200,6 @@ struct trapframe_fred {
 #define	TF_HASBASES	0x00000002
 #define	TF_HASFPXSTATE	0x00000004
 #define	TF_RESERV0	0x00000008 /* no tlsbase in the trapframe */
-#define	TF_FRED		0x00000010
 #endif /* __amd64__ */
 
 #endif /* _MACHINE_FRAME_H_ */

sys/x86/include/x86_var.h

@@ -97,6 +97,7 @@ extern	int	x86_rngds_mitg_enable;
 extern	int	zenbleed_enable;
 extern	int	cpu_amdc1e_bug;
 extern	char	bootmethod[16];
+extern	int	fred;
 
 struct	pcb;
 struct	thread;
@@ -114,6 +115,8 @@ struct	minidumpstate;
  */
 typedef void alias_for_inthand_t(void);
 
+extern void (*fred_ipi_handlers[])(struct trapframe *);
+
 bool	acpi_get_fadt_bootflags(uint16_t *flagsp);
 void	*alloc_fpusave(int flags);
 u_int	cpu_auxmsr(void);

sys/x86/x86/local_apic.c

@@ -2394,6 +2394,8 @@ native_lapic_ipi_vectored(u_int vector, int dest)
 void (*ipi_vectored)(u_int, int) = &native_lapic_ipi_vectored;
 #endif /* SMP */
 
+void (*fred_ipi_handlers[IPI_DYN_LAST - IPI_DYN_FIRST])(struct trapframe *);
+
 /*
  * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
  * visible.
@@ -2408,10 +2410,9 @@ void (*ipi_vectored)(u_int, int) = &native_lapic_ipi_vectored;
  * the IDT slot update is globally visible before the IPI is delivered.
  */
 int
-lapic_ipi_alloc(inthand_t *ipifunc)
+lapic_ipi_alloc(inthand_t *ipifunc, void (*fred_ipifunc)(struct trapframe *))
 {
-	struct gate_descriptor *ip;
-	long func;
+	void (*fip)(struct trapframe *);
 	int idx, vector;
 
 	KASSERT(ipifunc != &IDTVEC(rsvd) && ipifunc != &IDTVEC(rsvd_pti),
@@ -2420,14 +2421,10 @@ lapic_ipi_alloc(inthand_t *ipifunc)
 	vector = -1;
 	mtx_lock_spin(&icu_lock);
 	for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
-		ip = &idt[idx];
-		func = (ip->gd_hioffset << 16) | ip->gd_looffset;
-#ifdef __i386__
-		func -= setidt_disp;
-#endif
-		if ((!pti && func == (uintptr_t)&IDTVEC(rsvd)) ||
-		    (pti && func == (uintptr_t)&IDTVEC(rsvd_pti))) {
+		fip = fred_ipi_handlers[idx - IPI_DYN_FIRST];
+		if (fip == NULL) {
 			vector = idx;
+			fred_ipi_handlers[idx - IPI_DYN_FIRST] = fred_ipifunc;
 			setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
 			break;
 		}
@@ -2446,6 +2443,10 @@ lapic_ipi_free(int vector)
 	    ("%s: invalid vector %d", __func__, vector));
 
 	mtx_lock_spin(&icu_lock);
+	KASSERT(fred_ipi_handlers[vector - IPI_DYN_FIRST] != NULL,
+	    ("NULL fred func %d", vector));
+	fred_ipi_handlers[vector - IPI_DYN_FIRST] = NULL;
+	if (!fred) {
 		ip = &idt[vector];
 		func = (ip->gd_hioffset << 16) | ip->gd_looffset;
 #ifdef __i386__
@@ -2453,8 +2454,9 @@ lapic_ipi_free(int vector)
 #endif
 		KASSERT(func != (uintptr_t)&IDTVEC(rsvd) &&
 		    func != (uintptr_t)&IDTVEC(rsvd_pti),
-	    ("invalid idtfunc %#lx", func));
-	setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC,
-	    SEL_KPL, GSEL_APIC);
+		    ("invalid idtfunc %d %#lx", vector, func));
+		setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd),
+		    SDT_APIC, SEL_KPL, GSEL_APIC);
+	}
 	mtx_unlock_spin(&icu_lock);
 }