summaryrefslogtreecommitdiff
path: root/arch/x86/kvm/x86.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kvm/x86.c')
-rw-r--r--arch/x86/kvm/x86.c158
1 files changed, 115 insertions, 43 deletions
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 72d37c8930ad..391f4a5ce6dd 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -121,8 +121,10 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
#define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE
-#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
- KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
+#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
+ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \
+ KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \
+ KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
static void process_nmi(struct kvm_vcpu *vcpu);
@@ -183,6 +185,10 @@ bool __read_mostly enable_pmu = true;
EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_pmu);
module_param(enable_pmu, bool, 0444);
+/* Enable/disabled mediated PMU virtualization. */
+bool __read_mostly enable_mediated_pmu;
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu);
+
bool __read_mostly eager_page_split = true;
module_param(eager_page_split, bool, 0644);
@@ -2211,6 +2217,9 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_invd);
fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu)
{
+ if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+ return EXIT_FASTPATH_NONE;
+
if (!kvm_emulate_invd(vcpu))
return EXIT_FASTPATH_EXIT_USERSPACE;
@@ -2267,6 +2276,9 @@ static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
+ if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+ return EXIT_FASTPATH_NONE;
+
switch (msr) {
case APIC_BASE_MSR + (APIC_ICR >> 4):
if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic) ||
@@ -2314,13 +2326,14 @@ static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
u64 val;
/*
- * Disallow writes to immutable feature MSRs after KVM_RUN. KVM does
- * not support modifying the guest vCPU model on the fly, e.g. changing
- * the nVMX capabilities while L2 is running is nonsensical. Allow
- * writes of the same value, e.g. to allow userspace to blindly stuff
- * all MSRs when emulating RESET.
+ * Reject writes to immutable feature MSRs if the vCPU model is frozen,
+ * as KVM doesn't support modifying the guest vCPU model on the fly,
+ * e.g. changing the VMX capabilities MSRs while L2 is active is
+ * nonsensical. Allow writes of the same value, e.g. so that userspace
+ * can blindly stuff all MSRs when emulating RESET.
*/
- if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index) &&
+ if (!kvm_can_set_cpuid_and_feature_msrs(vcpu) &&
+ kvm_is_immutable_feature_msr(index) &&
(do_get_msr(vcpu, index, &val) || *data != val))
return -EINVAL;
@@ -3941,6 +3954,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
vcpu->arch.perf_capabilities = data;
kvm_pmu_refresh(vcpu);
+ kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
break;
case MSR_IA32_PRED_CMD: {
u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB);
@@ -4096,47 +4110,47 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_KVM_WALL_CLOCK_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
vcpu->kvm->arch.wall_clock = data;
kvm_write_wall_clock(vcpu->kvm, data, 0);
break;
case MSR_KVM_WALL_CLOCK:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
vcpu->kvm->arch.wall_clock = data;
kvm_write_wall_clock(vcpu->kvm, data, 0);
break;
case MSR_KVM_SYSTEM_TIME_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
break;
case MSR_KVM_SYSTEM_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
kvm_write_system_time(vcpu, data, true, msr_info->host_initiated);
break;
case MSR_KVM_ASYNC_PF_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (kvm_pv_enable_async_pf(vcpu, data))
return 1;
break;
case MSR_KVM_ASYNC_PF_INT:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (kvm_pv_enable_async_pf_int(vcpu, data))
return 1;
break;
case MSR_KVM_ASYNC_PF_ACK:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (data & 0x1) {
/*
* Pairs with the smp_mb__after_atomic() in
@@ -4149,7 +4163,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_KVM_STEAL_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (unlikely(!sched_info_on()))
return 1;
@@ -4167,7 +4181,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_KVM_PV_EOI_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
return 1;
@@ -4175,7 +4189,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
case MSR_KVM_POLL_CONTROL:
if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
/* only enable bit supported */
if (data & (-1ULL << 1))
@@ -4476,61 +4490,61 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_KVM_WALL_CLOCK:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->kvm->arch.wall_clock;
break;
case MSR_KVM_WALL_CLOCK_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->kvm->arch.wall_clock;
break;
case MSR_KVM_SYSTEM_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.time;
break;
case MSR_KVM_SYSTEM_TIME_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.time;
break;
case MSR_KVM_ASYNC_PF_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.apf.msr_en_val;
break;
case MSR_KVM_ASYNC_PF_INT:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.apf.msr_int_val;
break;
case MSR_KVM_ASYNC_PF_ACK:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = 0;
break;
case MSR_KVM_STEAL_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.st.msr_val;
break;
case MSR_KVM_PV_EOI_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.pv_eoi.msr_val;
break;
case MSR_KVM_POLL_CONTROL:
if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.msr_kvm_poll_control;
break;
@@ -4931,6 +4945,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
break;
case KVM_CAP_X2APIC_API:
r = KVM_X2APIC_API_VALID_FLAGS;
+ if (kvm && !irqchip_split(kvm))
+ r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST;
break;
case KVM_CAP_NESTED_STATE:
r = kvm_x86_ops.nested_ops->get_state ?
@@ -6726,7 +6742,7 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
case KVM_CAP_SPLIT_IRQCHIP: {
mutex_lock(&kvm->lock);
r = -EINVAL;
- if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
+ if (cap->args[0] > KVM_MAX_IRQ_ROUTES)
goto split_irqchip_unlock;
r = -EEXIST;
if (irqchip_in_kernel(kvm))
@@ -6748,11 +6764,24 @@ split_irqchip_unlock:
if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
break;
+ if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) &&
+ (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST))
+ break;
+
+ if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) &&
+ !irqchip_split(kvm))
+ break;
+
if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
kvm->arch.x2apic_format = true;
if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
kvm->arch.x2apic_broadcast_quirk_disabled = true;
+ if (cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST)
+ kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED;
+ if (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)
+ kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED;
+
r = 0;
break;
case KVM_CAP_X86_DISABLE_EXITS:
@@ -6863,7 +6892,7 @@ disable_exits_unlock:
break;
mutex_lock(&kvm->lock);
- if (!kvm->created_vcpus) {
+ if (!kvm->created_vcpus && !kvm->arch.created_mediated_pmu) {
kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
r = 0;
}
@@ -10133,7 +10162,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops)
set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
#endif
- kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
+ __kvm_register_perf_callbacks(ops->handle_intel_pt_intr,
+ enable_mediated_pmu ? kvm_handle_guest_mediated_pmi : NULL);
if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_mmu_enabled)
kvm_caps.supported_vm_types |= BIT(KVM_X86_SW_PROTECTED_VM);
@@ -10258,7 +10288,7 @@ static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid)
.dest_id = apicid,
};
- kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
+ kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq);
}
bool kvm_apicv_activated(struct kvm *kvm)
@@ -10899,16 +10929,9 @@ void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
* pending. At the same time, KVM_REQ_EVENT may not be set as APICv was
* still active when the interrupt got accepted. Make sure
* kvm_check_and_inject_events() is called to check for that.
- *
- * Update SVI when APICv gets enabled, otherwise SVI won't reflect the
- * highest bit in vISR and the next accelerated EOI in the guest won't
- * be virtualized correctly (the CPU uses SVI to determine which vISR
- * vector to clear).
*/
if (!apic->apicv_active)
kvm_make_request(KVM_REQ_EVENT, vcpu);
- else
- kvm_apic_update_hwapic_isr(vcpu);
out:
preempt_enable();
@@ -11348,6 +11371,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
run_flags |= KVM_RUN_LOAD_DEBUGCTL;
vcpu->arch.host_debugctl = debug_ctl;
+ kvm_mediated_pmu_load(vcpu);
+
guest_timing_enter_irqoff();
/*
@@ -11386,6 +11411,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
kvm_load_host_pkru(vcpu);
+ kvm_mediated_pmu_put(vcpu);
+
/*
* Do this here before restoring debug registers on the host. And
* since we do this before handling the vmexit, a DR access vmexit
@@ -11609,8 +11636,7 @@ static inline int vcpu_block(struct kvm_vcpu *vcpu)
if (is_guest_mode(vcpu)) {
int r = kvm_check_nested_events(vcpu);
- WARN_ON_ONCE(r == -EBUSY);
- if (r < 0)
+ if (r < 0 && r != -EBUSY)
return 0;
}
@@ -11724,6 +11750,9 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_halt);
fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu)
{
+ if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+ return EXIT_FASTPATH_NONE;
+
if (!kvm_emulate_halt(vcpu))
return EXIT_FASTPATH_EXIT_USERSPACE;
@@ -12158,9 +12187,11 @@ static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
return;
if (is_pae_paging(vcpu)) {
+ kvm_vcpu_srcu_read_lock(vcpu);
for (i = 0 ; i < 4 ; i++)
sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
+ kvm_vcpu_srcu_read_unlock(vcpu);
}
}
@@ -12661,8 +12692,13 @@ static int sync_regs(struct kvm_vcpu *vcpu)
return 0;
}
+#define PERF_MEDIATED_PMU_MSG \
+ "Failed to enable mediated vPMU, try disabling system wide perf events and nmi_watchdog.\n"
+
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
{
+ int r;
+
if (kvm_check_tsc_unstable() && kvm->created_vcpus)
pr_warn_once("SMP vm created on host with unstable TSC; "
"guest TSC will not be reliable\n");
@@ -12673,7 +12709,29 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
if (id >= kvm->arch.max_vcpu_ids)
return -EINVAL;
- return kvm_x86_call(vcpu_precreate)(kvm);
+ /*
+ * Note, any actions done by .vcpu_create() must be idempotent with
+ * respect to creating multiple vCPUs, and therefore are not undone if
+ * creating a vCPU fails (including failure during pre-create).
+ */
+ r = kvm_x86_call(vcpu_precreate)(kvm);
+ if (r)
+ return r;
+
+ if (enable_mediated_pmu && kvm->arch.enable_pmu &&
+ !kvm->arch.created_mediated_pmu) {
+ if (irqchip_in_kernel(kvm)) {
+ r = perf_create_mediated_pmu();
+ if (r) {
+ pr_warn_ratelimited(PERF_MEDIATED_PMU_MSG);
+ return r;
+ }
+ kvm->arch.created_mediated_pmu = true;
+ } else {
+ kvm->arch.enable_pmu = false;
+ }
+ }
+ return 0;
}
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
@@ -13320,7 +13378,7 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm)
#endif
kvm_mmu_pre_destroy_vm(kvm);
- static_call_cond(kvm_x86_vm_pre_destroy)(kvm);
+ kvm_x86_call(vm_pre_destroy)(kvm);
}
void kvm_arch_destroy_vm(struct kvm *kvm)
@@ -13339,6 +13397,8 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
mutex_unlock(&kvm->slots_lock);
}
+ if (kvm->arch.created_mediated_pmu)
+ perf_release_mediated_pmu();
kvm_destroy_vcpus(kvm);
kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
#ifdef CONFIG_KVM_IOAPIC
@@ -14143,6 +14203,13 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
return 1;
}
+ /*
+ * When ERAPS is supported, invalidating a specific PCID clears
+ * the RAP (Return Address Predicator).
+ */
+ if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
+ kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS);
+
kvm_invalidate_pcid(vcpu, operand.pcid);
return kvm_skip_emulated_instruction(vcpu);
@@ -14156,6 +14223,11 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
fallthrough;
case INVPCID_TYPE_ALL_INCL_GLOBAL:
+ /*
+ * Don't bother marking VCPU_EXREG_ERAPS dirty, SVM will take
+ * care of doing so when emulating the full guest TLB flush
+ * (the RAP is cleared on all implicit TLB flushes).
+ */
kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
return kvm_skip_emulated_instruction(vcpu);
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-05 14:40:11 +0000