linux: Implement PTRACE_GETREGSET NT_PRFPREG and NT_X86_XSTATE

Implement NT_PRFPREG and NT_X86_XSTATE for PTRACE_GETREGSET on amd64.
Chrome's crashpad handler uses these to collect floating-point and
extended CPU register state for crash dumps.

Other architectures retain the previous EINVAL stub behavior.

Signed-off-by:	Ricardo Branco <[email protected]>
PR:		289285
Reviewed by:	kib
Pull-Request:	https://github.com/freebsd/freebsd-src/pull/2165

(cherry picked from commit c515e60406)

sys/amd64/amd64/pmap.c

@@ -1379,6 +1379,7 @@ static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
 static vm_page_t pmap_large_map_getptp_unlocked(void);
 static vm_paddr_t pmap_large_map_kextract(vm_offset_t va);
+static bool pmap_page_is_mapped_locked(vm_page_t m);
 #if VM_NRESERVLEVEL > 0
 static bool pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
     vm_page_t mpte, struct rwlock **lockp);
@@ -5211,7 +5212,6 @@ reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain)
 	struct pv_chunks_list *pvc;
 	struct pv_chunk *pc, *pc_marker, *pc_marker_end;
 	struct pv_chunk_header pc_marker_b, pc_marker_end_b;
-	struct md_page *pvh;
 	pd_entry_t *pde;
 	pmap_t next_pmap, pmap;
 	pt_entry_t *pte, tpte;
@@ -5331,14 +5331,8 @@ reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain)
 				CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 				TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 				m->md.pv_gen++;
-				if (TAILQ_EMPTY(&m->md.pv_list) &&
-				    (m->flags & PG_FICTITIOUS) == 0) {
-					pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
-					if (TAILQ_EMPTY(&pvh->pv_list)) {
-						vm_page_aflag_clear(m,
-						    PGA_WRITEABLE);
-					}
-				}
+				if (!pmap_page_is_mapped_locked(m))
+					vm_page_aflag_clear(m, PGA_WRITEABLE);
 				pmap_delayed_invl_page(m);
 				pc->pc_map[field] |= 1UL << bit;
 				pmap_unuse_pt(pmap, va, *pde, &free);
@@ -6224,7 +6218,6 @@ static int
 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
     pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp)
 {
-	struct md_page *pvh;
 	pt_entry_t oldpte, PG_A, PG_M, PG_RW;
 	vm_page_t m;
 
@@ -6245,12 +6238,8 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
 			vm_page_aflag_set(m, PGA_REFERENCED);
 		CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 		pmap_pvh_free(&m->md, pmap, va);
-		if (TAILQ_EMPTY(&m->md.pv_list) &&
-		    (m->flags & PG_FICTITIOUS) == 0) {
-			pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
-			if (TAILQ_EMPTY(&pvh->pv_list))
-				vm_page_aflag_clear(m, PGA_WRITEABLE);
-		}
+		if (!pmap_page_is_mapped_locked(m))
+			vm_page_aflag_clear(m, PGA_WRITEABLE);
 		pmap_delayed_invl_page(m);
 	}
 	return (pmap_unuse_pt(pmap, va, ptepde, free));
@@ -7299,10 +7288,13 @@ retry:
 			    ("pmap_enter: no PV entry for %#lx", va));
 			if ((newpte & PG_MANAGED) == 0)
 				free_pv_entry(pmap, pv);
+
+			/*
+			 * The old page is likely COW, so check "writeable"
+			 * first.
+			 */
 			if ((om->a.flags & PGA_WRITEABLE) != 0 &&
-			    TAILQ_EMPTY(&om->md.pv_list) &&
-			    ((om->flags & PG_FICTITIOUS) != 0 ||
-			    TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
+			    !pmap_page_is_mapped_locked(om))
 				vm_page_aflag_clear(om, PGA_WRITEABLE);
 		} else {
 			/*
@@ -8463,13 +8455,22 @@ pmap_page_is_mapped(vm_page_t m)
 		return (false);
 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 	rw_rlock(lock);
-	rv = !TAILQ_EMPTY(&m->md.pv_list) ||
-	    ((m->flags & PG_FICTITIOUS) == 0 &&
-	    !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
+	rv = pmap_page_is_mapped_locked(m);
 	rw_runlock(lock);
 	return (rv);
 }
 
+/*
+ * The page's PV list lock must be held.
+ */
+static __always_inline bool
+pmap_page_is_mapped_locked(vm_page_t m)
+{
+	return (!TAILQ_EMPTY(&m->md.pv_list) ||
+	    ((m->flags & PG_FICTITIOUS) == 0 &&
+	    !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)));
+}
+
 /*
  * Destroy all managed, non-wired mappings in the given user-space
  * pmap.  This pmap cannot be active on any processor besides the
@@ -8662,12 +8663,8 @@ pmap_remove_pages(pmap_t pmap)
 					TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 					m->md.pv_gen++;
 					if ((m->a.flags & PGA_WRITEABLE) != 0 &&
-					    TAILQ_EMPTY(&m->md.pv_list) &&
-					    (m->flags & PG_FICTITIOUS) == 0) {
-						pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
-						if (TAILQ_EMPTY(&pvh->pv_list))
-							vm_page_aflag_clear(m, PGA_WRITEABLE);
-					}
+					    !pmap_page_is_mapped_locked(m))
+						vm_page_aflag_clear(m, PGA_WRITEABLE);
 				}
 				pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
 #ifdef PV_STATS

sys/amd64/linux/linux.h

@@ -154,6 +154,8 @@ struct l_newstat {
 #define	LINUX_SA_NOCLDSTOP	0x00000001
 #define	LINUX_SA_NOCLDWAIT	0x00000002
 #define	LINUX_SA_SIGINFO	0x00000004
+#define	LINUX_SA_UNSUPPORTED	0x00000400
+#define	LINUX_SA_EXPOSE_TAGBITS	0x00000800
 #define	LINUX_SA_RESTORER	0x04000000
 #define	LINUX_SA_ONSTACK	0x08000000
 #define	LINUX_SA_RESTART	0x10000000
@@ -235,14 +237,32 @@ struct linux_pt_regset {
 	l_ulong gs;
 };
 
+/* This corresponds to 'user_i387_struct' in Linux. */
+struct linux_pt_fpregset {
+	l_ushort cwd;
+	l_ushort swd;
+	l_ushort twd;
+	l_ushort fop;
+	uint64_t rip;
+	uint64_t rdp;
+	uint32_t mxcsr;
+	uint32_t mxcsr_mask;
+	uint32_t st_space[32];
+	uint32_t xmm_space[64];
+	uint32_t padding[24];
+};
+
 #ifdef _KERNEL
 struct reg;
+struct fpreg;
 struct syscall_info;
 
 void	bsd_to_linux_regset(const struct reg *b_reg,
 	    struct linux_pt_regset *l_regset);
 void	linux_to_bsd_regset(struct reg *b_reg,
 	    const struct linux_pt_regset *l_regset);
+void	bsd_to_linux_fpregset(const struct fpreg *b_fpreg,
+	    struct linux_pt_fpregset *l_fpregset);
 void	linux_ptrace_get_syscall_info_machdep(const struct reg *reg,
 	    struct syscall_info *si);
 int	linux_ptrace_getregs_machdep(struct thread *td, pid_t pid,

sys/amd64/linux/linux_machdep.c

@@ -281,6 +281,26 @@ linux_to_bsd_regset(struct reg *b_reg, const struct linux_pt_regset *l_regset)
 	b_reg->r_gs = l_regset->gs;
 }
 
+void
+bsd_to_linux_fpregset(const struct fpreg *b_fpreg,
+    struct linux_pt_fpregset *l_fpregset)
+{
+	l_fpregset->cwd = b_fpreg->fpr_env[0];
+	l_fpregset->swd = b_fpreg->fpr_env[0] >> 16;
+	l_fpregset->twd = b_fpreg->fpr_env[0] >> 32;
+	l_fpregset->fop = b_fpreg->fpr_env[0] >> 48;
+	l_fpregset->rip = b_fpreg->fpr_env[1];
+	l_fpregset->rdp = b_fpreg->fpr_env[2];
+	l_fpregset->mxcsr = b_fpreg->fpr_env[3];
+	l_fpregset->mxcsr_mask = b_fpreg->fpr_env[3] >> 32;
+
+	memcpy(l_fpregset->st_space, b_fpreg->fpr_acc,
+	    sizeof(l_fpregset->st_space));
+	memcpy(l_fpregset->xmm_space, b_fpreg->fpr_xacc,
+	    sizeof(l_fpregset->xmm_space));
+	memset(l_fpregset->padding, 0, sizeof(l_fpregset->padding));
+}
+
 void
 linux_ptrace_get_syscall_info_machdep(const struct reg *reg,
     struct syscall_info *si)

sys/amd64/linux32/linux.h

@@ -255,6 +255,8 @@ struct l_statfs64 {
 #define	LINUX_SA_NOCLDSTOP	0x00000001
 #define	LINUX_SA_NOCLDWAIT	0x00000002
 #define	LINUX_SA_SIGINFO	0x00000004
+#define	LINUX_SA_UNSUPPORTED	0x00000400
+#define	LINUX_SA_EXPOSE_TAGBITS	0x00000800
 #define	LINUX_SA_RESTORER	0x04000000
 #define	LINUX_SA_ONSTACK	0x08000000
 #define	LINUX_SA_RESTART	0x10000000

sys/arm64/arm64/pmap.c

@@ -497,6 +497,7 @@ static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,
 static pt_entry_t pmap_load_l3c(pt_entry_t *l3p);
 static void pmap_mask_set_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,
     vm_offset_t *vap, vm_offset_t va_next, pt_entry_t mask, pt_entry_t nbits);
+static bool pmap_page_is_mapped_locked(vm_page_t m);
 static bool pmap_pv_insert_l3c(pmap_t pmap, vm_offset_t va, vm_page_t m,
     struct rwlock **lockp);
 static void pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va);
@@ -3351,7 +3352,6 @@ reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain)
 	struct pv_chunks_list *pvc;
 	struct pv_chunk *pc, *pc_marker, *pc_marker_end;
 	struct pv_chunk_header pc_marker_b, pc_marker_end_b;
-	struct md_page *pvh;
 	pd_entry_t *pde;
 	pmap_t next_pmap, pmap;
 	pt_entry_t *pte, tpte;
@@ -3453,14 +3453,8 @@ reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain)
 				CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 				TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 				m->md.pv_gen++;
-				if (TAILQ_EMPTY(&m->md.pv_list) &&
-				    (m->flags & PG_FICTITIOUS) == 0) {
-					pvh = page_to_pvh(m);
-					if (TAILQ_EMPTY(&pvh->pv_list)) {
-						vm_page_aflag_clear(m,
-						    PGA_WRITEABLE);
-					}
-				}
+				if (!pmap_page_is_mapped_locked(m))
+					vm_page_aflag_clear(m, PGA_WRITEABLE);
 				pc->pc_map[field] |= 1UL << bit;
 				pmap_unuse_pt(pmap, va, pmap_load(pde), &free);
 				freed++;
@@ -4066,7 +4060,6 @@ static int
 pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,
     pd_entry_t l2e, struct spglist *free, struct rwlock **lockp)
 {
-	struct md_page *pvh;
 	pt_entry_t old_l3;
 	vm_page_t m;
 
@@ -4087,12 +4080,8 @@ pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,
 			vm_page_aflag_set(m, PGA_REFERENCED);
 		CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 		pmap_pvh_free(&m->md, pmap, va);
-		if (TAILQ_EMPTY(&m->md.pv_list) &&
-		    (m->flags & PG_FICTITIOUS) == 0) {
-			pvh = page_to_pvh(m);
-			if (TAILQ_EMPTY(&pvh->pv_list))
-				vm_page_aflag_clear(m, PGA_WRITEABLE);
-		}
+		if (!pmap_page_is_mapped_locked(m))
+			vm_page_aflag_clear(m, PGA_WRITEABLE);
 	}
 	return (pmap_unuse_pt(pmap, va, l2e, free));
 }
@@ -4194,7 +4183,6 @@ static void
 pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva,
     vm_offset_t eva, struct spglist *free, struct rwlock **lockp)
 {
-	struct md_page *pvh;
 	struct rwlock *new_lock;
 	pt_entry_t *l3, old_l3;
 	vm_offset_t va;
@@ -4272,12 +4260,8 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva,
 				rw_wlock(*lockp);
 			}
 			pmap_pvh_free(&m->md, pmap, sva);
-			if (TAILQ_EMPTY(&m->md.pv_list) &&
-			    (m->flags & PG_FICTITIOUS) == 0) {
-				pvh = page_to_pvh(m);
-				if (TAILQ_EMPTY(&pvh->pv_list))
-					vm_page_aflag_clear(m, PGA_WRITEABLE);
-			}
+			if (!pmap_page_is_mapped_locked(m))
+				vm_page_aflag_clear(m, PGA_WRITEABLE);
 		}
 		if (l3pg != NULL && pmap_unwire_l3(pmap, sva, l3pg, free)) {
 			/*
@@ -5641,10 +5625,13 @@ havel3:
 			pv = pmap_pvh_remove(&om->md, pmap, va);
 			if ((m->oflags & VPO_UNMANAGED) != 0)
 				free_pv_entry(pmap, pv);
+
+			/*
+			 * The old page is likely COW, so check "writeable"
+			 * first.
+			 */
 			if ((om->a.flags & PGA_WRITEABLE) != 0 &&
-			    TAILQ_EMPTY(&om->md.pv_list) &&
-			    ((om->flags & PG_FICTITIOUS) != 0 ||
-			    TAILQ_EMPTY(&page_to_pvh(om)->pv_list)))
+			    !pmap_page_is_mapped_locked(om))
 				vm_page_aflag_clear(om, PGA_WRITEABLE);
 		} else {
 			KASSERT((orig_l3 & ATTR_AF) != 0,
@@ -7116,13 +7103,22 @@ pmap_page_is_mapped(vm_page_t m)
 		return (false);
 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 	rw_rlock(lock);
-	rv = !TAILQ_EMPTY(&m->md.pv_list) ||
-	    ((m->flags & PG_FICTITIOUS) == 0 &&
-	    !TAILQ_EMPTY(&page_to_pvh(m)->pv_list));
+	rv = pmap_page_is_mapped_locked(m);
 	rw_runlock(lock);
 	return (rv);
 }
 
+/*
+ * The page's PV list lock must be held.
+ */
+static __always_inline bool
+pmap_page_is_mapped_locked(vm_page_t m)
+{
+	return (!TAILQ_EMPTY(&m->md.pv_list) ||
+	    ((m->flags & PG_FICTITIOUS) == 0 &&
+	    !TAILQ_EMPTY(&page_to_pvh(m)->pv_list)));
+}
+
 /*
  * Destroy all managed, non-wired mappings in the given user-space
  * pmap.  This pmap cannot be active on any processor besides the
@@ -7291,13 +7287,9 @@ pmap_remove_pages(pmap_t pmap)
 					    pv_next);
 					m->md.pv_gen++;
 					if ((m->a.flags & PGA_WRITEABLE) != 0 &&
-					    TAILQ_EMPTY(&m->md.pv_list) &&
-					    (m->flags & PG_FICTITIOUS) == 0) {
-						pvh = page_to_pvh(m);
-						if (TAILQ_EMPTY(&pvh->pv_list))
-							vm_page_aflag_clear(m,
-							    PGA_WRITEABLE);
-					}
+					    !pmap_page_is_mapped_locked(m))
+						vm_page_aflag_clear(m,
+						    PGA_WRITEABLE);
 					break;
 				}
 				pmap_unuse_pt(pmap, pv->pv_va, pmap_load(pde),

sys/arm64/linux/linux.h

@@ -150,6 +150,8 @@ struct l_newstat {
 #define	LINUX_SA_NOCLDSTOP	0x00000001
 #define	LINUX_SA_NOCLDWAIT	0x00000002
 #define	LINUX_SA_SIGINFO	0x00000004
+#define	LINUX_SA_UNSUPPORTED	0x00000400
+#define	LINUX_SA_EXPOSE_TAGBITS	0x00000800
 #define	LINUX_SA_RESTORER	0x04000000
 #define	LINUX_SA_ONSTACK	0x08000000
 #define	LINUX_SA_RESTART	0x10000000

sys/compat/linux/linux_ptrace.c

@@ -335,6 +335,87 @@ linux_ptrace_getregset_prstatus(struct thread *td, pid_t pid, l_ulong data)
 	return (error);
 }
 
+#ifdef __amd64__
+static int
+linux_ptrace_getregset_prfpreg(struct thread *td, pid_t pid, l_ulong data)
+{
+	struct fpreg b_fpreg;
+	struct linux_pt_fpregset l_fpregset;
+	struct iovec iov;
+	size_t len;
+	int error;
+
+	error = copyin((const void *)data, &iov, sizeof(iov));
+	if (error != 0) {
+		linux_msg(td, "copyin error %d", error);
+		return (error);
+	}
+
+	error = kern_ptrace(td, PT_GETFPREGS, pid, &b_fpreg, 0);
+	if (error != 0)
+		return (error);
+
+	bsd_to_linux_fpregset(&b_fpreg, &l_fpregset);
+
+	len = MIN(iov.iov_len, sizeof(l_fpregset));
+	error = copyout(&l_fpregset, iov.iov_base, len);
+	if (error != 0) {
+		linux_msg(td, "copyout error %d", error);
+		return (error);
+	}
+
+	iov.iov_len = len;
+	error = copyout(&iov, (void *)data, sizeof(iov));
+	if (error != 0)
+		linux_msg(td, "iov copyout error %d", error);
+
+	return (error);
+}
+
+static int
+linux_ptrace_getregset_xstate(struct thread *td, pid_t pid, l_ulong data)
+{
+	struct ptrace_xstate_info info;
+	struct iovec iov;
+	void *xstate;
+	size_t len;
+	int error;
+
+	error = copyin((const void *)data, &iov, sizeof(iov));
+	if (error != 0) {
+		linux_msg(td, "copyin error %d", error);
+		return (error);
+	}
+
+	error = kern_ptrace(td, PT_GETXSTATE_INFO, pid, &info, sizeof(info));
+	if (error != 0)
+		return (error);
+
+	xstate = malloc(info.xsave_len, M_LINUX, M_WAITOK | M_ZERO);
+
+	error = kern_ptrace(td, PT_GETXSTATE, pid, xstate, info.xsave_len);
+	if (error != 0) {
+		free(xstate, M_LINUX);
+		return (error);
+	}
+
+	len = MIN(iov.iov_len, info.xsave_len);
+	error = copyout(xstate, iov.iov_base, len);
+	free(xstate, M_LINUX);
+	if (error != 0) {
+		linux_msg(td, "copyout error %d", error);
+		return (error);
+	}
+
+	iov.iov_len = len;
+	error = copyout(&iov, (void *)data, sizeof(iov));
+	if (error != 0)
+		linux_msg(td, "iov copyout error %d", error);
+
+	return (error);
+}
+#endif /* __amd64__ */
+
 static int
 linux_ptrace_getregset(struct thread *td, pid_t pid, l_ulong addr, l_ulong data)
 {
@@ -342,14 +423,12 @@ linux_ptrace_getregset(struct thread *td, pid_t pid, l_ulong addr, l_ulong data)
 	switch (addr) {
 	case LINUX_NT_PRSTATUS:
 		return (linux_ptrace_getregset_prstatus(td, pid, data));
+#ifdef __amd64__
 	case LINUX_NT_PRFPREG:
-		linux_msg(td, "PTRAGE_GETREGSET NT_PRFPREG not implemented; "
-		    "returning EINVAL");
-		return (EINVAL);
+		return (linux_ptrace_getregset_prfpreg(td, pid, data));
 	case LINUX_NT_X86_XSTATE:
-		linux_msg(td, "PTRAGE_GETREGSET NT_X86_XSTATE not implemented; "
-		    "returning EINVAL");
-		return (EINVAL);
+		return (linux_ptrace_getregset_xstate(td, pid, data));
+#endif
 	default:
 		linux_msg(td, "PTRACE_GETREGSET request %#lx not implemented; "
 		    "returning EINVAL", addr);

sys/compat/linux/linux_signal.c

@@ -129,6 +129,14 @@ linux_to_bsd_sigaction(l_sigaction_t *lsa, struct sigaction *bsa)
 		bsa->sa_flags |= SA_NODEFER;
 	}
 
+	/*
+	 * SA_UNSUPPORTED was introduced in Linux 5.11 to probe support for
+	 * other flags such as SA_EXPOSE_TAGBITS, introduced at the same time.
+	 * Ignore both.
+	 */
+	if (lsa->lsa_flags & (LINUX_SA_UNSUPPORTED | LINUX_SA_EXPOSE_TAGBITS))
+		flags &= ~(LINUX_SA_UNSUPPORTED | LINUX_SA_EXPOSE_TAGBITS);
+
 	if (flags != 0)
 		linux_msg(curthread, "unsupported sigaction flag %#lx", flags);
 }

sys/i386/linux/linux.h

@@ -227,6 +227,8 @@ struct l_statfs64 {
 #define	LINUX_SA_NOCLDSTOP	0x00000001
 #define	LINUX_SA_NOCLDWAIT	0x00000002
 #define	LINUX_SA_SIGINFO	0x00000004
+#define	LINUX_SA_UNSUPPORTED	0x00000400
+#define	LINUX_SA_EXPOSE_TAGBITS	0x00000800
 #define	LINUX_SA_RESTORER	0x04000000
 #define	LINUX_SA_ONSTACK	0x08000000
 #define	LINUX_SA_RESTART	0x10000000

sys/kern/vfs_init.c

@@ -427,6 +427,8 @@ vfs_register(struct vfsconf *vfc)
 		 */
 		hashval = fnv_32_str(vfc->vfc_name, FNV1_32_INIT);
 		hashval &= 0xff;
+		if (hashval == 0)
+			hashval = 1;
 		secondpass = 0;
 		do {
 			/* Look for and fix any collision. */