Skip to content

Commit

Permalink
KVM: X86: Improve latency for single target IPI fastpath
Browse files Browse the repository at this point in the history 
IPI and Timer cause the main MSRs write vmexits in cloud environment
observation, let's optimize virtual IPI latency more aggressively to
inject target IPI as soon as possible.

Running kvm-unit-tests/vmexit.flat IPI testing on SKX server, disable
adaptive advance lapic timer and adaptive halt-polling to avoid the
interference, this patch can give another 7% improvement.

w/o fastpath   -> x86.c fastpath      4238 -> 3543  16.4%
x86.c fastpath -> vmx.c fastpath      3543 -> 3293     7%
w/o fastpath   -> vmx.c fastpath      4238 -> 3293  22.3%

Cc: Haiwei Li <lihaiwei@tencent.com>
Signed-off-by: Wanpeng Li <wanpengli@tencent.com>
Signed-off-by: Sean Christopherson <sean.j.christopherson@intel.com>
Message-Id: <20200410174703.1138-3-sean.j.christopherson@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
  • Loading branch information
Wanpeng Li authored and Paolo Bonzini committed Apr 21, 2020
1 parent 873e1da commit a9ab13f
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 23 deletions.
5 changes: 2 additions & 3 deletions arch/x86/include/asm/kvm_host.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1126,7 +1126,7 @@ struct kvm_x86_ops {
*/
void (*tlb_flush_guest)(struct kvm_vcpu *vcpu);

void (*run)(struct kvm_vcpu *vcpu);
enum exit_fastpath_completion (*run)(struct kvm_vcpu *vcpu);
int (*handle_exit)(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion exit_fastpath);
int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
Expand Down Expand Up @@ -1176,8 +1176,7 @@ struct kvm_x86_ops {
struct x86_instruction_info *info,
enum x86_intercept_stage stage,
struct x86_exception *exception);
void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion *exit_fastpath);
void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu);

int (*check_nested_events)(struct kvm_vcpu *vcpu);
void (*request_immediate_exit)(struct kvm_vcpu *vcpu);
Expand Down
24 changes: 16 additions & 8 deletions arch/x86/kvm/svm/svm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3299,10 +3299,21 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu)
svm_complete_interrupts(svm);
}

static enum exit_fastpath_completion svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
{
if (!is_guest_mode(vcpu) &&
to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
to_svm(vcpu)->vmcb->control.exit_info_1)
return handle_fastpath_set_msr_irqoff(vcpu);

return EXIT_FASTPATH_NONE;
}

void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs);

static void svm_vcpu_run(struct kvm_vcpu *vcpu)
static enum exit_fastpath_completion svm_vcpu_run(struct kvm_vcpu *vcpu)
{
enum exit_fastpath_completion exit_fastpath;
struct vcpu_svm *svm = to_svm(vcpu);

svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
Expand All @@ -3314,7 +3325,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
* again.
*/
if (unlikely(svm->nested.exit_required))
return;
return EXIT_FASTPATH_NONE;

/*
* Disable singlestep if we're injecting an interrupt/exception.
Expand Down Expand Up @@ -3398,6 +3409,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
stgi();

/* Any pending NMI will happen here */
exit_fastpath = svm_exit_handlers_fastpath(vcpu);

if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
kvm_after_interrupt(&svm->vcpu);
Expand Down Expand Up @@ -3426,6 +3438,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
svm_handle_mce(svm);

mark_all_clean(svm->vmcb);
return exit_fastpath;
}

static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root)
Expand Down Expand Up @@ -3727,13 +3740,8 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu,
return ret;
}

static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion *exit_fastpath)
static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
{
if (!is_guest_mode(vcpu) &&
to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
to_svm(vcpu)->vmcb->control.exit_info_1)
*exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu);
}

static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
Expand Down
22 changes: 13 additions & 9 deletions arch/x86/kvm/vmx/vmx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6350,18 +6350,14 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu)
}
STACK_FRAME_NON_STANDARD(handle_external_interrupt_irqoff);

static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion *exit_fastpath)
static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);

if (vmx->exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
handle_external_interrupt_irqoff(vcpu);
else if (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI)
handle_exception_nmi_irqoff(vmx);
else if (!is_guest_mode(vcpu) &&
vmx->exit_reason == EXIT_REASON_MSR_WRITE)
*exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu);
}

static bool vmx_has_emulated_msr(int index)
Expand Down Expand Up @@ -6565,8 +6561,9 @@ void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)

bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched);

static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
static enum exit_fastpath_completion vmx_vcpu_run(struct kvm_vcpu *vcpu)
{
enum exit_fastpath_completion exit_fastpath;
struct vcpu_vmx *vmx = to_vmx(vcpu);
unsigned long cr3, cr4;

Expand All @@ -6578,7 +6575,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
/* Don't enter VMX if guest state is invalid, let the exit handler
start emulation until we arrive back to a valid state */
if (vmx->emulation_required)
return;
return EXIT_FASTPATH_NONE;

if (vmx->ple_window_dirty) {
vmx->ple_window_dirty = false;
Expand Down Expand Up @@ -6726,21 +6723,28 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu)

if (unlikely(vmx->fail)) {
vmx->exit_reason = 0xdead;
return;
return EXIT_FASTPATH_NONE;
}

vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
if (unlikely((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY))
kvm_machine_check();

if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
return;
return EXIT_FASTPATH_NONE;

if (!is_guest_mode(vcpu) && vmx->exit_reason == EXIT_REASON_MSR_WRITE)
exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu);
else
exit_fastpath = EXIT_FASTPATH_NONE;

vmx->loaded_vmcs->launched = 1;
vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);

vmx_recover_nmi_blocking(vmx);
vmx_complete_interrupts(vmx);

return exit_fastpath;
}

static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
Expand Down
6 changes: 3 additions & 3 deletions arch/x86/kvm/x86.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8179,7 +8179,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
bool req_int_win =
dm_request_for_irq_injection(vcpu) &&
kvm_cpu_accept_dm_intr(vcpu);
enum exit_fastpath_completion exit_fastpath = EXIT_FASTPATH_NONE;
enum exit_fastpath_completion exit_fastpath;

bool req_immediate_exit = false;

Expand Down Expand Up @@ -8406,7 +8406,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
}

kvm_x86_ops.run(vcpu);
exit_fastpath = kvm_x86_ops.run(vcpu);

/*
* Do this here before restoring debug registers on the host. And
Expand Down Expand Up @@ -8438,7 +8438,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
vcpu->mode = OUTSIDE_GUEST_MODE;
smp_wmb();

kvm_x86_ops.handle_exit_irqoff(vcpu, &exit_fastpath);
kvm_x86_ops.handle_exit_irqoff(vcpu);

/*
* Consume any pending interrupts, including the possible source of
Expand Down

0 comments on commit a9ab13f

Please sign in to comment.