amd_cpuid.patch

commit ec046ffc2b8b9ed6916e402ee580e18da6673709
Author: Keno Fischer <[email protected]>
Date:   Sun Jul 3 01:53:53 2022 +0000

    WIP: AMD CPUID override
    
    Since e9ea1e7, we've had the ability to turn a userspace `cpuid`
    instruction into a SIGSEGV using the ARCH_(GET|SET)_CPUID arch_prctl.
    However, this capability is limited to supported hardware, which
    currently means Intel CPUs. AMD CPUs do not have a documented facility
    for achieving equivalent results, but they do have MSRs to mask certain
    bits from the CPUID feature response. Masking off such bits is a
    primary use case of the CPUID faulting feature, thus it is desirable
    to expose this capability of AMD CPUs to userspace.
    
    This patch does just that by adding two new arch_prctls that closely
    match the capabilities provided by the AMD hardware:
    
    - ARCH_SET_CPUID_FEATURE_DISABLE
    
    Takes a pointer to a struct with the following layout:
    ```
    struct cpuid_feature_mask {
        uint32_t leaf1_ecx_mask;
        uint32_t leaf1_edx_mask;
        uint32_t leaf8000_0001_ecx_mask;
        uint32_t leaf8000_0001_edx_mask;
    };
    ```
    
    any bits set in the field will be masked off from the CPUID response
    for the corresponding cpuid leaf/register. The setting is preserved
    across both fork and exec (the latter unlike ARCH_SET_CPUID).
    
    - ARCH_SET_CPUID_FEATURE_DISABLE
    
    Takes a pointer to the same struct and simply reads it back into
    userspace.

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 03acc823838a..bbf157ff1602 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -230,6 +230,7 @@
 #define X86_FEATURE_FLEXPRIORITY	( 8*32+ 2) /* Intel FlexPriority */
 #define X86_FEATURE_EPT			( 8*32+ 3) /* Intel Extended Page Table */
 #define X86_FEATURE_VPID		( 8*32+ 4) /* Intel Virtual Processor ID */
+#define X86_FEATURE_AMD_CPUID_OVERRIDE	( 8*32+ 5) /* AMD CPUID Masking */
 
 #define X86_FEATURE_VMMCALL		( 8*32+15) /* Prefer VMMCALL to VMCALL */
 #define X86_FEATURE_XENPV		( 8*32+16) /* "" Xen paravirtual guest */
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index d27e0581b777..2154992e04a0 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -507,6 +507,7 @@
 #define MSR_AMD_PPIN_CTL		0xc00102f0
 #define MSR_AMD_PPIN			0xc00102f1
 #define MSR_AMD64_CPUID_FN_1		0xc0011004
+#define MSR_AMD64_CPUID_FN_80000001 0xc0011005
 #define MSR_AMD64_LS_CFG		0xc0011020
 #define MSR_AMD64_DC_CFG		0xc0011022
 #define MSR_AMD64_BU_CFG2		0xc001102a
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 356308c73951..f9585268ab87 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -500,6 +500,8 @@ struct thread_struct {
 	unsigned long           virtual_dr6;
 	/* Keep track of the exact dr7 value set by the user */
 	unsigned long           ptrace_dr7;
+	/* CPUID feature masks. Set bits are masked out from CPUID */
+	unsigned int		cpuid_masks_inverted[4];
 	/* Fault info: */
 	unsigned long		cr2;
 	unsigned long		trap_nr;
@@ -797,6 +799,8 @@ extern int get_tsc_mode(unsigned long adr);
 extern int set_tsc_mode(unsigned int val);
 
 DECLARE_PER_CPU(u64, msr_misc_features_shadow);
+DECLARE_PER_CPU(u64, msr_cpuid_mask_base);
+DECLARE_PER_CPU(u64, msr_cpuid_ext_mask_base);
 
 extern u16 get_llc_id(unsigned int cpu);
 
diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h
index 12ef86b19910..59796b5be90d 100644
--- a/arch/x86/include/asm/proto.h
+++ b/arch/x86/include/asm/proto.h
@@ -42,6 +42,6 @@ void x86_configure_nx(void);
 
 extern int reboot_force;
 
-long do_arch_prctl_common(int option, unsigned long arg2);
+long do_arch_prctl_common(int option, unsigned long arg2, unsigned long arg3);
 
 #endif /* _ASM_X86_PROTO_H */
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index f0cb881c1d69..0811aedbc85e 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -92,6 +92,7 @@ struct thread_info {
 #define TIF_NOCPUID		15	/* CPUID is not accessible in userland */
 #define TIF_NOTSC		16	/* TSC is not accessible in userland */
 #define TIF_NOTIFY_SIGNAL	17	/* signal notifications exist */
+#define TIF_CPUID_MASK		18	/* Spome CPUID features are masked off */
 #define TIF_MEMDIE		20	/* is terminating due to OOM killer */
 #define TIF_POLLING_NRFLAG	21	/* idle is polling for TIF_NEED_RESCHED */
 #define TIF_IO_BITMAP		22	/* uses I/O bitmap */
@@ -115,6 +116,7 @@ struct thread_info {
 #define _TIF_NOCPUID		(1 << TIF_NOCPUID)
 #define _TIF_NOTSC		(1 << TIF_NOTSC)
 #define _TIF_NOTIFY_SIGNAL	(1 << TIF_NOTIFY_SIGNAL)
+#define _TIF_CPUID_MASK		(1 << TIF_CPUID_MASK)
 #define _TIF_POLLING_NRFLAG	(1 << TIF_POLLING_NRFLAG)
 #define _TIF_IO_BITMAP		(1 << TIF_IO_BITMAP)
 #define _TIF_SPEC_FORCE_UPDATE	(1 << TIF_SPEC_FORCE_UPDATE)
@@ -126,7 +128,7 @@ struct thread_info {
 /* flags to check in __switch_to() */
 #define _TIF_WORK_CTXSW_BASE					\
 	(_TIF_NOCPUID | _TIF_NOTSC | _TIF_BLOCKSTEP |		\
-	 _TIF_SSBD | _TIF_SPEC_FORCE_UPDATE)
+	 _TIF_SSBD | _TIF_SPEC_FORCE_UPDATE | _TIF_CPUID_MASK)
 
 /*
  * Avoid calls to __switch_to_xtra() on UP as STIBP is not evaluated.
diff --git a/arch/x86/include/uapi/asm/prctl.h b/arch/x86/include/uapi/asm/prctl.h
index 500b96e71f18..31a5742431a3 100644
--- a/arch/x86/include/uapi/asm/prctl.h
+++ b/arch/x86/include/uapi/asm/prctl.h
@@ -9,6 +9,16 @@
 
 #define ARCH_GET_CPUID			0x1011
 #define ARCH_SET_CPUID			0x1012
+#define ARCH_GET_CPUID_FEATURE_DISABLE 0x1013
+#define ARCH_SET_CPUID_FEATURE_DISABLE 0x1014
+
+/* Set bits in the specified leaf/register are masked off from the cpuid response. */
+struct cpuid_feature_mask {
+    uint32_t leaf1_ecx_mask;
+    uint32_t leaf1_edx_mask;
+    uint32_t leaf8000_0001_ecx_mask;
+    uint32_t leaf8000_0001_edx_mask;
+};
 
 #define ARCH_GET_XCOMP_SUPP		0x1021
 #define ARCH_GET_XCOMP_PERM		0x1022
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 0c0b09796ced..c301bda1e84e 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -966,6 +966,17 @@ static void init_amd(struct cpuinfo_x86 *c)
 		msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT);
 
 	check_null_seg_clears_base(c);
+
+	/* Family 0x14 and above have CPUID reset MSRs */
+	if (c->x86 > 0x14) {
+		u64 msr, msr_ext;
+		if (!rdmsrl_safe(MSR_AMD64_CPUID_FN_1, &msr) &&
+			!rdmsrl_safe(MSR_AMD64_CPUID_FN_80000001, &msr_ext)) {
+			this_cpu_write(msr_cpuid_mask_base, msr);
+			this_cpu_write(msr_cpuid_ext_mask_base, msr_ext);
+			set_cpu_cap(c, X86_FEATURE_AMD_CPUID_OVERRIDE);
+		}
+	}
 }
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 9b2772b7e1f3..f9b0e645e973 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -173,6 +173,9 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 	frame->flags = X86_EFLAGS_FIXED;
 #endif
 
+	memcpy(p->thread.cpuid_masks_inverted, current->thread.cpuid_masks_inverted,
+	       sizeof(p->thread.cpuid_masks_inverted));
+
 	fpu_clone(p, clone_flags, args->fn);
 
 	/* Kernel thread ? */
@@ -347,6 +350,99 @@ static int set_cpuid_mode(unsigned long cpuid_enabled)
 	return 0;
 }
 
+DEFINE_PER_CPU(u64, msr_cpuid_mask_base);
+DEFINE_PER_CPU(u64, msr_cpuid_ext_mask_base);
+
+static void apply_cpuid_override(void)
+{
+	u64 msr, msr_ext;
+	u32 *masks;
+
+	msr = this_cpu_read(msr_cpuid_mask_base);
+	msr_ext = this_cpu_read(msr_cpuid_ext_mask_base);
+
+	masks = current->thread.cpuid_masks_inverted;
+
+	msr |= ((u64)masks[0] << 32) | masks[1];
+	msr_ext |= ((u64)masks[2] << 32) | masks[3];
+
+	wrmsrl(MSR_AMD64_CPUID_FN_1, msr);
+	wrmsrl(MSR_AMD64_CPUID_FN_80000001, msr_ext);
+}
+
+static void enable_cpuid_override(void)
+{
+	preempt_disable();
+	if (!test_and_set_thread_flag(TIF_CPUID_MASK)) {
+		/*
+		 * Must flip the CPU state synchronously with
+		 * TIF_NOCPUID in the current running context.
+		 */
+		apply_cpuid_override();
+	}
+	preempt_enable();
+}
+
+static void disable_cpuid_override(void)
+{
+	preempt_disable();
+	if (test_and_clear_thread_flag(TIF_CPUID_MASK)) {
+		/*
+		 * Must flip the CPU state synchronously with
+		 * TIF_NOCPUID in the current running context.
+		 */
+		apply_cpuid_override();
+	}
+	preempt_enable();
+}
+
+static int amd_set_cpuid_feature_mask(struct cpuid_feature_mask __user *feature_mask)
+{
+	struct cpuid_feature_mask mask;
+	bool any_mask;
+
+	if (copy_from_user(&mask, feature_mask, sizeof(mask)))
+		return -EFAULT;
+
+	memcpy(current->thread.cpuid_masks_inverted, &mask, sizeof(mask));
+
+	any_mask = mask.leaf1_ecx_mask || mask.leaf1_edx_mask ||
+		mask.leaf8000_0001_ecx_mask || mask.leaf8000_0001_edx_mask;
+
+	if (any_mask)
+		enable_cpuid_override();
+	else
+		disable_cpuid_override();
+
+	return 0;
+}
+
+static int set_cpuid_feature_mask(struct cpuid_feature_mask __user *feature_mask)
+{
+	if (!boot_cpu_has(X86_FEATURE_AMD_CPUID_OVERRIDE))
+		return -ENODEV;
+
+	return amd_set_cpuid_feature_mask(feature_mask);
+}
+
+static int amd_get_cpuid_feature_mask(struct cpuid_feature_mask __user *feature_mask)
+{
+	struct cpuid_feature_mask mask;
+	memcpy(&mask, current->thread.cpuid_masks_inverted, sizeof(mask));
+	if (copy_to_user(feature_mask, &mask, sizeof(mask)))
+		return -EFAULT;
+	return 0;
+}
+
+static int get_cpuid_feature_mask(struct cpuid_feature_mask __user *feature_mask)
+{
+	if (!boot_cpu_has(X86_FEATURE_AMD_CPUID_OVERRIDE))
+		return -ENODEV;
+
+	return amd_get_cpuid_feature_mask(feature_mask);
+}
+
+
 /*
  * Called immediately after a successful exec.
  */
@@ -677,6 +773,9 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p)
 	if ((tifp ^ tifn) & _TIF_NOCPUID)
 		set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
 
+	if ((tifp | tifn) & _TIF_CPUID_MASK)
+		apply_cpuid_override();
+
 	if (likely(!((tifp | tifn) & _TIF_SPEC_FORCE_UPDATE))) {
 		__speculation_ctrl_update(tifp, tifn);
 	} else {
@@ -985,7 +1084,7 @@ unsigned long __get_wchan(struct task_struct *p)
 	return addr;
 }
 
-long do_arch_prctl_common(int option, unsigned long arg2)
+long do_arch_prctl_common(int option, unsigned long arg2, unsigned long arg3)
 {
 	switch (option) {
 	case ARCH_GET_CPUID:
@@ -998,6 +1097,14 @@ long do_arch_prctl_common(int option, unsigned long arg2)
 	case ARCH_GET_XCOMP_GUEST_PERM:
 	case ARCH_REQ_XCOMP_GUEST_PERM:
 		return fpu_xstate_prctl(option, arg2);
+	case ARCH_GET_CPUID_FEATURE_DISABLE:
+		if (arg3 != sizeof(struct cpuid_feature_mask))
+			return -EINVAL;
+		return get_cpuid_feature_mask((struct cpuid_feature_mask __user *)arg2);
+	case ARCH_SET_CPUID_FEATURE_DISABLE:
+		if (arg3 != sizeof(struct cpuid_feature_mask))
+			return -EINVAL;
+		return set_cpuid_feature_mask((struct cpuid_feature_mask __user *)arg2);
 	}
 
 	return -EINVAL;
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 2f314b170c9f..7e1927835bf7 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -217,7 +217,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	return prev_p;
 }
 
-SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
+SYSCALL_DEFINE3(arch_prctl, int, option, unsigned long, arg2, unsigned long, arg3)
 {
-	return do_arch_prctl_common(option, arg2);
+	return do_arch_prctl_common(option, arg2, arg3);
 }
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 1962008fe743..fdf21ec020e7 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -838,21 +838,21 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
 	return ret;
 }
 
-SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
+SYSCALL_DEFINE3(arch_prctl, int, option, unsigned long, arg2, unsigned long, arg3)
 {
 	long ret;
 
 	ret = do_arch_prctl_64(current, option, arg2);
 	if (ret == -EINVAL)
-		ret = do_arch_prctl_common(option, arg2);
+		ret = do_arch_prctl_common(option, arg2, arg3);
 
 	return ret;
 }
 
 #ifdef CONFIG_IA32_EMULATION
-COMPAT_SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
+COMPAT_SYSCALL_DEFINE3(arch_prctl, int, option, unsigned long, arg2, unsigned long, arg3)
 {
-	return do_arch_prctl_common(option, arg2);
+	return do_arch_prctl_common(option, arg2, arg3);
 }
 #endif
 
diff --git a/arch/x86/um/syscalls_64.c b/arch/x86/um/syscalls_64.c
index 27b29ae6c471..1a466bd1b1df 100644
--- a/arch/x86/um/syscalls_64.c
+++ b/arch/x86/um/syscalls_64.c
@@ -76,9 +76,9 @@ long arch_prctl(struct task_struct *task, int option,
 	return ret;
 }
 
-SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
+SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2, unsigned long, arg3)
 {
-	return arch_prctl(current, option, (unsigned long __user *) arg2);
+	return arch_prctl(current, option, (unsigned long __user *) arg2, arg3);
 }
 
 void arch_switch_to(struct task_struct *to)
diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h
index 03acc823838a..b8f50bac4983 100644
--- a/tools/arch/x86/include/asm/cpufeatures.h
+++ b/tools/arch/x86/include/asm/cpufeatures.h
@@ -230,6 +230,7 @@
 #define X86_FEATURE_FLEXPRIORITY	( 8*32+ 2) /* Intel FlexPriority */
 #define X86_FEATURE_EPT			( 8*32+ 3) /* Intel Extended Page Table */
 #define X86_FEATURE_VPID		( 8*32+ 4) /* Intel Virtual Processor ID */
+#define X86_FEATURE_AMD_CPUID_OVERRIDE	( 8*32+ 5) /* AMD CPUID override */
 
 #define X86_FEATURE_VMMCALL		( 8*32+15) /* Prefer VMMCALL to VMCALL */
 #define X86_FEATURE_XENPV		( 8*32+16) /* "" Xen paravirtual guest */
diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h
index d27e0581b777..6848cfb5cc36 100644
--- a/tools/arch/x86/include/asm/msr-index.h
+++ b/tools/arch/x86/include/asm/msr-index.h
@@ -507,6 +507,7 @@
 #define MSR_AMD_PPIN_CTL		0xc00102f0
 #define MSR_AMD_PPIN			0xc00102f1
 #define MSR_AMD64_CPUID_FN_1		0xc0011004
+#define MSR_AMD64_CPUID_FN_2		0xc0011005
 #define MSR_AMD64_LS_CFG		0xc0011020
 #define MSR_AMD64_DC_CFG		0xc0011022
 #define MSR_AMD64_BU_CFG2		0xc001102a