NothingOSS/android_kernel_msm-6.1_nothing_sm7635
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

KVM: x86/mmu: Avoid indirect call for get_cr3

Browse source 
[ Upstream commit 2fdcc1b324189b5fb20655baebd40cd82e2bdf0c ]

Most of the time, calls to get_guest_pgd result in calling
kvm_read_cr3 (the exception is only nested TDP).  Hardcode
the default instead of using the get_cr3 function, avoiding
a retpoline if they are enabled.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Mathias Krause <minipli@grsecurity.net>
Link: https://lore.kernel.org/r/20230322013731.102955-2-minipli@grsecurity.net
Signed-off-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Mathias Krause <minipli@grsecurity.net>	# backport to v6.1.x
Signed-off-by: Sasha Levin <sashal@kernel.org>
This commit is contained in:
Paolo Bonzini 2023-05-08 17:45:58 +02:00 • committed by Greg Kroah-Hartman
parent 28d0f85aff
commit d20a0195b3
2 changed files with 21 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

31
arch/x86/kvm/mmu/mmu.c
View file

@ -232,6 +232,20 @@ static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
return regs; return regs;
} }
static unsigned long get_guest_cr3(struct kvm_vcpu *vcpu)
{
return kvm_read_cr3(vcpu);
}
static inline unsigned long kvm_mmu_get_guest_pgd(struct kvm_vcpu *vcpu,
struct kvm_mmu *mmu)
{
if (IS_ENABLED(CONFIG_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)
return kvm_read_cr3(vcpu);
return mmu->get_guest_pgd(vcpu);
}
static inline bool kvm_available_flush_tlb_with_range(void) static inline bool kvm_available_flush_tlb_with_range(void)
{ {
return kvm_x86_ops.tlb_remote_flush_with_range; return kvm_x86_ops.tlb_remote_flush_with_range;
@ -3661,7 +3675,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
int quadrant, i, r; int quadrant, i, r;
hpa_t root; hpa_t root;
root_pgd = mmu->get_guest_pgd(vcpu); root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu);
root_gfn = root_pgd >> PAGE_SHIFT; root_gfn = root_pgd >> PAGE_SHIFT;
if (mmu_check_root(vcpu, root_gfn)) if (mmu_check_root(vcpu, root_gfn))
@ -4112,7 +4126,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
arch.token = alloc_apf_token(vcpu); arch.token = alloc_apf_token(vcpu);
arch.gfn = gfn; arch.gfn = gfn;
arch.direct_map = vcpu->arch.mmu->root_role.direct; arch.direct_map = vcpu->arch.mmu->root_role.direct;
arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);
return kvm_setup_async_pf(vcpu, cr2_or_gpa, return kvm_setup_async_pf(vcpu, cr2_or_gpa,
kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
@ -4131,7 +4145,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
return; return;
if (!vcpu->arch.mmu->root_role.direct && if (!vcpu->arch.mmu->root_role.direct &&
work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu)) work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))
return; return;
kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true); kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
@ -4488,11 +4502,6 @@ void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
} }
EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd); EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
return kvm_read_cr3(vcpu);
}
static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
unsigned int access) unsigned int access)
{ {
@ -5043,7 +5052,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
context->page_fault = kvm_tdp_page_fault; context->page_fault = kvm_tdp_page_fault;
context->sync_page = nonpaging_sync_page; context->sync_page = nonpaging_sync_page;
context->invlpg = NULL; context->invlpg = NULL;
context->get_guest_pgd = get_cr3; context->get_guest_pgd = get_guest_cr3;
context->get_pdptr = kvm_pdptr_read; context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault; context->inject_page_fault = kvm_inject_page_fault;
@ -5193,7 +5202,7 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
kvm_init_shadow_mmu(vcpu, cpu_role); kvm_init_shadow_mmu(vcpu, cpu_role);
context->get_guest_pgd = get_cr3; context->get_guest_pgd = get_guest_cr3;
context->get_pdptr = kvm_pdptr_read; context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault; context->inject_page_fault = kvm_inject_page_fault;
} }
@ -5207,7 +5216,7 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
return; return;
g_context->cpu_role.as_u64 = new_mode.as_u64; g_context->cpu_role.as_u64 = new_mode.as_u64;
g_context->get_guest_pgd = get_cr3; g_context->get_guest_pgd = get_guest_cr3;
g_context->get_pdptr = kvm_pdptr_read; g_context->get_pdptr = kvm_pdptr_read;
g_context->inject_page_fault = kvm_inject_page_fault; g_context->inject_page_fault = kvm_inject_page_fault;

2
arch/x86/kvm/mmu/paging_tmpl.h
View file

@ -324,7 +324,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
trace_kvm_mmu_pagetable_walk(addr, access); trace_kvm_mmu_pagetable_walk(addr, access);
retry_walk: retry_walk:
walker->level = mmu->cpu_role.base.level; walker->level = mmu->cpu_role.base.level;
pte = mmu->get_guest_pgd(vcpu); pte = kvm_mmu_get_guest_pgd(vcpu, mmu);
have_ad = PT_HAVE_ACCESSED_DIRTY(mmu); have_ad = PT_HAVE_ACCESSED_DIRTY(mmu);
#if PTTYPE == 64 #if PTTYPE == 64