Release 4.13 arch/x86/kvm/svm.c

Directory: arch/x86/kvm
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * AMD SVM support
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */


#define pr_fmt(fmt) "SVM: " fmt

#include <linux/kvm_host.h>

#include "irq.h"
#include "mmu.h"
#include "kvm_cache_regs.h"
#include "x86.h"
#include "cpuid.h"
#include "pmu.h"

#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/kernel.h>
#include <linux/vmalloc.h>
#include <linux/highmem.h>
#include <linux/sched.h>
#include <linux/trace_events.h>
#include <linux/slab.h>
#include <linux/amd-iommu.h>
#include <linux/hashtable.h>
#include <linux/frame.h>

#include <asm/apic.h>
#include <asm/perf_event.h>
#include <asm/tlbflush.h>
#include <asm/desc.h>
#include <asm/debugreg.h>
#include <asm/kvm_para.h>
#include <asm/irq_remapping.h>

#include <asm/virtext.h>
#include "trace.h"


#define __ex(x) __kvm_handle_fault_on_reboot(x)

MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");


static const struct x86_cpu_id svm_cpu_id[] = {
	X86_FEATURE_MATCH(X86_FEATURE_SVM),
	{}
};
MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);


#define IOPM_ALLOC_ORDER 2

#define MSRPM_ALLOC_ORDER 1


#define SEG_TYPE_LDT 2

#define SEG_TYPE_BUSY_TSS16 3


#define SVM_FEATURE_NPT            (1 <<  0)

#define SVM_FEATURE_LBRV           (1 <<  1)

#define SVM_FEATURE_SVML           (1 <<  2)

#define SVM_FEATURE_NRIP           (1 <<  3)

#define SVM_FEATURE_TSC_RATE       (1 <<  4)

#define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)

#define SVM_FEATURE_FLUSH_ASID     (1 <<  6)

#define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)

#define SVM_FEATURE_PAUSE_FILTER   (1 << 10)


#define SVM_AVIC_DOORBELL	0xc001011b


#define NESTED_EXIT_HOST	0	
/* Exit handled on host level */

#define NESTED_EXIT_DONE	1	
/* Exit caused nested vmexit  */

#define NESTED_EXIT_CONTINUE	2	
/* Further checks needed      */


#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))


#define TSC_RATIO_RSVD          0xffffff0000000000ULL

#define TSC_RATIO_MIN		0x0000000000000001ULL

#define TSC_RATIO_MAX		0x000000ffffffffffULL


#define AVIC_HPA_MASK	~((0xFFFULL << 52) | 0xFFF)

/*
 * 0xff is broadcast, so the max index allowed for physical APIC ID
 * table is 0xfe.  APIC IDs above 0xff are reserved.
 */

#define AVIC_MAX_PHYSICAL_ID_COUNT	255


#define AVIC_UNACCEL_ACCESS_WRITE_MASK		1

#define AVIC_UNACCEL_ACCESS_OFFSET_MASK		0xFF0

#define AVIC_UNACCEL_ACCESS_VECTOR_MASK		0xFFFFFFFF

/* AVIC GATAG is encoded using VM and VCPU IDs */

#define AVIC_VCPU_ID_BITS		8

#define AVIC_VCPU_ID_MASK		((1 << AVIC_VCPU_ID_BITS) - 1)


#define AVIC_VM_ID_BITS			24

#define AVIC_VM_ID_NR			(1 << AVIC_VM_ID_BITS)

#define AVIC_VM_ID_MASK			((1 << AVIC_VM_ID_BITS) - 1)


#define AVIC_GATAG(x, y)		(((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
                                                (y & AVIC_VCPU_ID_MASK))

#define AVIC_GATAG_TO_VMID(x)		((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)

#define AVIC_GATAG_TO_VCPUID(x)		(x & AVIC_VCPU_ID_MASK)


static bool erratum_383_found __read_mostly;


static const u32 host_save_user_msrs[] = {
#ifdef CONFIG_X86_64
	MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
	MSR_FS_BASE,
#endif
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
	MSR_TSC_AUX,
};


#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)

struct kvm_vcpu;


struct nested_state {
	
struct vmcb *hsave;
	
u64 hsave_msr;
	
u64 vm_cr_msr;
	
u64 vmcb;

	/* These are the merged vectors */
	
u32 *msrpm;

	/* gpa pointers to the real vectors */
	
u64 vmcb_msrpm;
	
u64 vmcb_iopm;

	/* A VMEXIT is required but not yet emulated */
	
bool exit_required;

	/* cache for intercepts of the guest */
	
u32 intercept_cr;
	
u32 intercept_dr;
	
u32 intercept_exceptions;
	
u64 intercept;

	/* Nested Paging related state */
	
u64 nested_cr3;
};


#define MSRPM_OFFSETS	16

static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;

/*
 * Set osvw_len to higher value when updated Revision Guides
 * are published and we know what the new status bits are
 */


static uint64_t osvw_len = 4, osvw_status;


struct vcpu_svm {
	
struct kvm_vcpu vcpu;
	
struct vmcb *vmcb;
	
unsigned long vmcb_pa;
	
struct svm_cpu_data *svm_data;
	
uint64_t asid_generation;
	
uint64_t sysenter_esp;
	
uint64_t sysenter_eip;
	
uint64_t tsc_aux;

	
u64 next_rip;

	
u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
	struct {
		
u16 fs;
		
u16 gs;
		
u16 ldt;
		
u64 gs_base;
	
} host;

	
u32 *msrpm;

	
ulong nmi_iret_rip;

	
struct nested_state nested;

	
bool nmi_singlestep;
	
u64 nmi_singlestep_guest_rflags;

	
unsigned int3_injected;
	
unsigned long int3_rip;

	/* cached guest cpuid flags for faster access */
	
bool nrips_enabled	: 1;

	
u32 ldr_reg;
	
struct page *avic_backing_page;
	
u64 *avic_physical_id_cache;
	
bool avic_is_running;

	/*
         * Per-vcpu list of struct amd_svm_iommu_ir:
         * This is used mainly to store interrupt remapping information used
         * when update the vcpu affinity. This avoids the need to scan for
         * IRTE and try to match ga_tag in the IOMMU driver.
         */
	
struct list_head ir_list;
	
spinlock_t ir_list_lock;
};

/*
 * This is a wrapper of struct amd_iommu_ir_data.
 */

struct amd_svm_iommu_ir {
	
struct list_head node;	/* Used by SVM for per-vcpu ir_list */
	
void *data;		/* Storing pointer to struct amd_ir_data */
};


#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK	(0xFF)

#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK		(1 << 31)


#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK	(0xFFULL)

#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK	(0xFFFFFFFFFFULL << 12)

#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK		(1ULL << 62)

#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK		(1ULL << 63)

static DEFINE_PER_CPU(u64, current_tsc_ratio);

#define TSC_RATIO_DEFAULT	0x0100000000ULL


#define MSR_INVALID			0xffffffffU


static const struct svm_direct_access_msrs {
	
u32 index;   /* Index of the MSR */
	
bool always; /* True if intercept is always on */

} direct_access_msrs[] = {
	{ .index = MSR_STAR,				.always = true  },
	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
#ifdef CONFIG_X86_64
	{ .index = MSR_GS_BASE,				.always = true  },
	{ .index = MSR_FS_BASE,				.always = true  },
	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
	{ .index = MSR_LSTAR,				.always = true  },
	{ .index = MSR_CSTAR,				.always = true  },
	{ .index = MSR_SYSCALL_MASK,			.always = true  },
#endif
	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
	{ .index = MSR_INVALID,				.always = false },
};

/* enable NPT for AMD64 and X86 with PAE */
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)

static bool npt_enabled = true;
#else

static bool npt_enabled;
#endif

/* allow nested paging (virtualized MMU) for all guests */

static int npt = true;
module_param(npt, int, S_IRUGO);

/* allow nested virtualization in KVM/SVM */

static int nested = true;
module_param(nested, int, S_IRUGO);

/* enable / disable AVIC */

static int avic;
#ifdef CONFIG_X86_LOCAL_APIC
module_param(avic, int, S_IRUGO);
#endif

/* enable/disable Virtual VMLOAD VMSAVE */

static int vls = true;
module_param(vls, int, 0444);

/* AVIC VM ID bit masks and lock */
static DECLARE_BITMAP(avic_vm_id_bitmap, AVIC_VM_ID_NR);
static DEFINE_SPINLOCK(avic_vm_id_lock);

static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
static void svm_flush_tlb(struct kvm_vcpu *vcpu);
static void svm_complete_interrupts(struct vcpu_svm *svm);

static int nested_svm_exit_handled(struct vcpu_svm *svm);
static int nested_svm_intercept(struct vcpu_svm *svm);
static int nested_svm_vmexit(struct vcpu_svm *svm);
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code);

enum {
	
VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
                            pause filter count */
	
VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
	
VMCB_ASID,	 /* ASID */
	
VMCB_INTR,	 /* int_ctl, int_vector */
	
VMCB_NPT,        /* npt_en, nCR3, gPAT */
	
VMCB_CR,	 /* CR0, CR3, CR4, EFER */
	
VMCB_DR,         /* DR6, DR7 */
	
VMCB_DT,         /* GDT, IDT */
	
VMCB_SEG,        /* CS, DS, SS, ES, CPL */
	
VMCB_CR2,        /* CR2 only */
	
VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
	
VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
                          * AVIC PHYSICAL_TABLE pointer,
                          * AVIC LOGICAL_TABLE pointer
                          */
	
VMCB_DIRTY_MAX,
};

/* TPR and CR2 are always written before VMRUN */

#define VMCB_ALWAYS_DIRTY_MASK	((1U << VMCB_INTR) | (1U << VMCB_CR2))


#define VMCB_AVIC_APIC_BAR_MASK		0xFFFFFFFFFF000ULL



static inline void mark_all_dirty(struct vmcb *vmcb)
{
	vmcb->control.clean = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void mark_all_clean(struct vmcb *vmcb)
{
	vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
			       & ~VMCB_ALWAYS_DIRTY_MASK;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%



static inline void mark_dirty(struct vmcb *vmcb, int bit)
{
	vmcb->control.clean &= ~(1 << bit);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
{
	return container_of(vcpu, struct vcpu_svm, vcpu);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 19 76.00% 1 50.00%
Rusty Russell 6 24.00% 1 50.00%
Total 25 100.00% 2 100.00%



static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
{
	svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
	mark_dirty(svm->vmcb, VMCB_AVIC);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 36 100.00% 1 100.00%
Total 36 100.00% 1 100.00%



static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 *entry = svm->avic_physical_id_cache;

	if (!entry)
		return false;

	return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 49 100.00% 1 100.00%
Total 49 100.00% 1 100.00%



static void recalc_intercepts(struct vcpu_svm *svm)
{
	struct vmcb_control_area *c, *h;
	struct nested_state *g;

	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);

	if (!is_guest_mode(&svm->vcpu))
		return;

	c = &svm->vmcb->control;
	h = &svm->nested.hsave->control;
	g = &svm->nested;

	c->intercept_cr = h->intercept_cr | g->intercept_cr;
	c->intercept_dr = h->intercept_dr | g->intercept_dr;
	c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
	c->intercept = h->intercept | g->intercept;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 120 100.00% 4 100.00%
Total 120 100.00% 4 100.00%



static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
{
	if (is_guest_mode(&svm->vcpu))
		return svm->nested.hsave;
	else
		return svm->vmcb;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 37 100.00% 1 100.00%
Total 37 100.00% 1 100.00%



static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_cr |= (1U << bit);

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 42 100.00% 1 100.00%
Total 42 100.00% 1 100.00%



static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_cr &= ~(1U << bit);

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 43 100.00% 1 100.00%
Total 43 100.00% 1 100.00%



static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	return vmcb->control.intercept_cr & (1U << bit);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 38 100.00% 1 100.00%
Total 38 100.00% 1 100.00%



static inline void set_dr_intercepts(struct vcpu_svm *svm)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
		| (1 << INTERCEPT_DR1_READ)
		| (1 << INTERCEPT_DR2_READ)
		| (1 << INTERCEPT_DR3_READ)
		| (1 << INTERCEPT_DR4_READ)
		| (1 << INTERCEPT_DR5_READ)
		| (1 << INTERCEPT_DR6_READ)
		| (1 << INTERCEPT_DR7_READ)
		| (1 << INTERCEPT_DR0_WRITE)
		| (1 << INTERCEPT_DR1_WRITE)
		| (1 << INTERCEPT_DR2_WRITE)
		| (1 << INTERCEPT_DR3_WRITE)
		| (1 << INTERCEPT_DR4_WRITE)
		| (1 << INTERCEPT_DR5_WRITE)
		| (1 << INTERCEPT_DR6_WRITE)
		| (1 << INTERCEPT_DR7_WRITE);

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Bonzini 94 72.87% 1 50.00%
Joerg Roedel 35 27.13% 1 50.00%
Total 129 100.00% 2 100.00%



static inline void clr_dr_intercepts(struct vcpu_svm *svm)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_dr = 0;

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 32 91.43% 1 50.00%
Paolo Bonzini 3 8.57% 1 50.00%
Total 35 100.00% 2 100.00%



static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_exceptions |= (1U << bit);

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 42 100.00% 1 100.00%
Total 42 100.00% 1 100.00%



static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_exceptions &= ~(1U << bit);

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 43 100.00% 1 100.00%
Total 43 100.00% 1 100.00%



static inline void set_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept |= (1ULL << bit);

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 42 100.00% 1 100.00%
Total 42 100.00% 1 100.00%



static inline void clr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept &= ~(1ULL << bit);

	recalc_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 43 100.00% 1 100.00%
Total 43 100.00% 1 100.00%



static inline void enable_gif(struct vcpu_svm *svm)
{
	svm->vcpu.arch.hflags |= HF_GIF_MASK;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 22 100.00% 1 100.00%
Total 22 100.00% 1 100.00%



static inline void disable_gif(struct vcpu_svm *svm)
{
	svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 23 100.00% 1 100.00%
Total 23 100.00% 1 100.00%



static inline bool gif_set(struct vcpu_svm *svm)
{
	return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 27 100.00% 1 100.00%
Total 27 100.00% 1 100.00%


static unsigned long iopm_base;


struct kvm_ldttss_desc {
	
u16 limit0;
	
u16 base0;
	



unsigned base1:8, type:5, dpl:2, p:1;
	



unsigned limit1:4, zero0:3, g:1, base2:8;
	
u32 base3;
	
u32 zero1;
} __attribute__((packed));


struct svm_cpu_data {
	
int cpu;

	
u64 asid_generation;
	
u32 max_asid;
	
u32 next_asid;
	
struct kvm_ldttss_desc *tss_desc;

	
struct page *save_area;
};

static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);


struct svm_init_data {
	
int cpu;
	
int r;
};


static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};


#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)

#define MSRS_RANGE_SIZE 2048

#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)



static u32 svm_msrpm_offset(u32 msr)
{
	u32 offset;
	int i;

	for (i = 0; i < NUM_MSR_MAPS; i++) {
		if (msr < msrpm_ranges[i] ||
		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
			continue;

		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */

		/* Now we have the u8 offset - but need the u32 offset */
		return offset / 4;
	}

	/* MSR not in any range */
	return MSR_INVALID;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 81 98.78% 2 66.67%
Avi Kivity 1 1.22% 1 33.33%
Total 82 100.00% 3 100.00%


#define MAX_INST_SIZE 15



static inline void clgi(void)
{
	asm volatile (__ex(SVM_CLGI));
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 12 100.00% 2 100.00%
Total 12 100.00% 2 100.00%



static inline void stgi(void)
{
	asm volatile (__ex(SVM_STGI));
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 12 100.00% 2 100.00%
Total 12 100.00% 2 100.00%



static inline void invlpga(unsigned long addr, u32 asid)
{
	asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 16 94.12% 1 50.00%
Joerg Roedel 1 5.88% 1 50.00%
Total 17 100.00% 2 100.00%



static int get_npt_level(void)
{
#ifdef CONFIG_X86_64
	return PT64_ROOT_LEVEL;
#else
	return PT32E_ROOT_LEVEL;
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%



static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	vcpu->arch.efer = efer;
	if (!npt_enabled && !(efer & EFER_LMA))
		efer &= ~EFER_LME;

	to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
	mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 37 56.06% 1 14.29%
Joerg Roedel 15 22.73% 2 28.57%
Zachary Amsden 8 12.12% 1 14.29%
Gregory Haskins 3 4.55% 1 14.29%
Carlo Marcelo Arenas Belon 2 3.03% 1 14.29%
Alexander Graf 1 1.52% 1 14.29%
Total 66 100.00% 7 100.00%



static int is_external_interrupt(u32 info)
{
	info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 24 100.00% 1 100.00%
Total 24 100.00% 1 100.00%



static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 ret = 0;

	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Glauber de Oliveira Costa 44 93.62% 1 33.33%
Jan Kiszka 2 4.26% 1 33.33%
Paolo Bonzini 1 2.13% 1 33.33%
Total 47 100.00% 3 100.00%



static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (mask == 0)
		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
	else
		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;

}

Contributors
Person Tokens Prop Commits CommitProp
Glauber de Oliveira Costa 52 100.00% 1 100.00%
Total 52 100.00% 1 100.00%



static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (svm->vmcb->control.next_rip != 0) {
		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
		svm->next_rip = svm->vmcb->control.next_rip;
	}

	if (!svm->next_rip) {
		if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
				EMULATE_DONE)
			printk(KERN_DEBUG "%s: NOP\n", __func__);
		return;
	}
	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
		printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
		       __func__, kvm_rip_read(vcpu), svm->next_rip);

	kvm_rip_write(vcpu, svm->next_rip);
	svm_set_interrupt_shadow(vcpu, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 53 40.46% 1 10.00%
Andre Przywara 24 18.32% 1 10.00%
Marcelo Tosatti 13 9.92% 1 10.00%
Gleb Natapov 11 8.40% 1 10.00%
Bandan Das 10 7.63% 1 10.00%
Gregory Haskins 10 7.63% 1 10.00%
Glauber de Oliveira Costa 6 4.58% 1 10.00%
Harvey Harrison 2 1.53% 1 10.00%
Dirk Müller 1 0.76% 1 10.00%
Rusty Russell 1 0.76% 1 10.00%
Total 131 100.00% 10 100.00%



static void svm_queue_exception(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	unsigned nr = vcpu->arch.exception.nr;
	bool has_error_code = vcpu->arch.exception.has_error_code;
	bool reinject = vcpu->arch.exception.reinject;
	u32 error_code = vcpu->arch.exception.error_code;

	/*
         * If we are within a nested VM we'd better #VMEXIT and let the guest
         * handle the exception
         */
	if (!reinject &&
	    nested_svm_check_exception(svm, nr, has_error_code, error_code))
		return;

	if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
		unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);

		/*
                 * For guest debugging where we have to reinject #BP if some
                 * INT3 is guest-owned:
                 * Emulate nRIP by moving RIP forward. Will fail if injection
                 * raises a fault that is not intercepted. Still better than
                 * failing in all cases.
                 */
		skip_emulated_instruction(&svm->vcpu);
		rip = kvm_rip_read(&svm->vcpu);
		svm->int3_rip = rip + svm->vmcb->save.cs.base;
		svm->int3_injected = rip - old_rip;
	}

	svm->vmcb->control.event_inj = nr
		| SVM_EVTINJ_VALID
		| (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
		| SVM_EVTINJ_TYPE_EXEPT;
	svm->vmcb->control.event_inj_err = error_code;
}

Contributors
Person Tokens Prop Commits CommitProp
Jan Kiszka 125 67.20% 2 25.00%
Wanpeng Li 44 23.66% 1 12.50%
Avi Kivity 12 6.45% 2 25.00%
Joerg Roedel 3 1.61% 2 25.00%
Eduardo Pereira Habkost 2 1.08% 1 12.50%
Total 186 100.00% 8 100.00%



static void svm_init_erratum_383(void)
{
	u32 low, high;
	int err;
	u64 val;

	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
		return;

	/* Use _safe variants to not break nested virtualization */
	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
	if (err)
		return;

	val |= (1ULL << 47);

	low  = lower_32_bits(val);
	high = upper_32_bits(val);

	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);

	erratum_383_found = true;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 74 93.67% 1 33.33%
Hans Rosenfeld 3 3.80% 1 33.33%
Borislav Petkov 2 2.53% 1 33.33%
Total 79 100.00% 3 100.00%



static void svm_init_osvw(struct kvm_vcpu *vcpu)
{
	/*
         * Guests should see errata 400 and 415 as fixed (assuming that
         * HLT and IO instructions are intercepted).
         */
	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
	vcpu->arch.osvw.status = osvw_status & ~(6ULL);

	/*
         * By increasing VCPU's osvw.length to 3 we are telling the guest that
         * all osvw.status bits inside that length, including bit 0 (which is
         * reserved for erratum 298), are valid. However, if host processor's
         * osvw_len is 0 then osvw_status[0] carries no information. We need to
         * be conservative here and therefore we tell the guest that erratum 298
         * is present (because we really don't know).
         */
	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
		vcpu->arch.osvw.status |= 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Boris Ostrovsky 70 100.00% 1 100.00%
Total 70 100.00% 1 100.00%



static int has_svm(void)
{
	const char *msg;

	if (!cpu_has_svm(&msg)) {
		printk(KERN_INFO "has_svm: %s\n", msg);
		return 0;
	}

	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 29 76.32% 1 33.33%
Eduardo Pereira Habkost 8 21.05% 1 33.33%
Joe Perches 1 2.63% 1 33.33%
Total 38 100.00% 3 100.00%



static void svm_hardware_disable(void)
{
	/* Make sure we clean up behind us */
	if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
		wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);

	cpu_svm_disable();

	amd_pmu_disable_virt();
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 18 62.07% 2 50.00%
Avi Kivity 9 31.03% 1 25.00%
Eduardo Pereira Habkost 2 6.90% 1 25.00%
Total 29 100.00% 4 100.00%



static int svm_hardware_enable(void)
{

	struct svm_cpu_data *sd;
	uint64_t efer;
	struct desc_struct *gdt;
	int me = raw_smp_processor_id();

	rdmsrl(MSR_EFER, efer);
	if (efer & EFER_SVME)
		return -EBUSY;

	if (!has_svm()) {
		pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
		return -EINVAL;
	}
	sd = per_cpu(svm_data, me);
	if (!sd) {
		pr_err("%s: svm_data is NULL on %d\n", __func__, me);
		return -EINVAL;
	}

	sd->asid_generation = 1;
	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
	sd->next_asid = sd->max_asid + 1;

	gdt = get_current_gdt_rw();
	sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);

	wrmsrl(MSR_EFER, efer | EFER_SVME);

	wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);

	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
		wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
		__this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
	}


	/*
         * Get OSVW bits.
         *
         * Note that it is possible to have a system with mixed processor
         * revisions and therefore different OSVW bits. If bits are not the same
         * on different processors then choose the worst case (i.e. if erratum
         * is present on one processor and not on another then assume that the
         * erratum is present everywhere).
         */
	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
		uint64_t len, status = 0;
		int err;

		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
		if (!err)
			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
						      &err);

		if (err)
			osvw_status = osvw_len = 0;
		else {
			if (len < osvw_len)
				osvw_len = len;
			osvw_status |= status;
			osvw_status &= (1ULL << osvw_len) - 1;
		}
	} else
		osvw_status = osvw_len = 0;

	svm_init_erratum_383();

	amd_pmu_enable_virt();

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 118 40.97% 1 9.09%
Boris Ostrovsky 92 31.94% 1 9.09%
Alexander Graf 30 10.42% 2 18.18%
Joerg Roedel 26 9.03% 3 27.27%
Tejun Heo 9 3.12% 1 9.09%
Borislav Petkov 8 2.78% 1 9.09%
Christoph Lameter 3 1.04% 1 9.09%
Thomas Garnier 2 0.69% 1 9.09%
Total 288 100.00% 11 100.00%



static void svm_cpu_uninit(int cpu)
{
	struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());

	if (!sd)
		return;

	per_cpu(svm_data, raw_smp_processor_id()) = NULL;
	__free_page(sd->save_area);
	kfree(sd);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 46 92.00% 1 50.00%
Tejun Heo 4 8.00% 1 50.00%
Total 50 100.00% 2 100.00%



static int svm_cpu_init(int cpu)
{
	struct svm_cpu_data *sd;
	int r;

	sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
	if (!sd)
		return -ENOMEM;
	sd->cpu = cpu;
	sd->save_area = alloc_page(GFP_KERNEL);
	r = -ENOMEM;
	if (!sd->save_area)
		goto err_1;

	per_cpu(svm_data, cpu) = sd;

	return 0;

err_1:
	kfree(sd);
	return r;

}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 83 91.21% 1 50.00%
Tejun Heo 8 8.79% 1 50.00%
Total 91 100.00% 2 100.00%



static bool valid_msr_intercept(u32 index)
{
	int i;

	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
		if (direct_access_msrs[i].index == index)
			return true;

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 25 53.19% 1 50.00%
Joerg Roedel 22 46.81% 1 50.00%
Total 47 100.00% 2 100.00%



static void set_msr_interception(u32 *msrpm, unsigned msr,
				 int read, int write)
{
	u8 bit_read, bit_write;
	unsigned long tmp;
	u32 offset;

	/*
         * If this warning triggers extend the direct_access_msrs list at the
         * beginning of the file
         */
	WARN_ON(!valid_msr_intercept(msr));

	offset    = svm_msrpm_offset(msr);
	bit_read  = 2 * (msr & 0x0f);
	bit_write = 2 * (msr & 0x0f) + 1;
	tmp       = msrpm[offset];

	BUG_ON(offset == MSR_INVALID);

	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);

	msrpm[offset] = tmp;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 85 66.93% 2 50.00%
Avi Kivity 41 32.28% 1 25.00%
Rusty Russell 1 0.79% 1 25.00%
Total 127 100.00% 4 100.00%



static void svm_vcpu_init_msrpm(u32 *msrpm)
{
	int i;

	memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));

	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
		if (!direct_access_msrs[i].always)
			continue;

		set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 48 64.00% 2 50.00%
Avi Kivity 25 33.33% 1 25.00%
Anthony Liguori 2 2.67% 1 25.00%
Total 75 100.00% 4 100.00%



static void add_msr_offset(u32 offset)
{
	int i;

	for (i = 0; i < MSRPM_OFFSETS; ++i) {

		/* Offset already in list? */
		if (msrpm_offsets[i] == offset)
			return;

		/* Slot used by another offset? */
		if (msrpm_offsets[i] != MSR_INVALID)
			continue;

		/* Add offset to list */
		msrpm_offsets[i] = offset;

		return;
	}

	/*
         * If this BUG triggers the msrpm_offsets table has an overflow. Just
         * increase MSRPM_OFFSETS in this case.
         */
	BUG();
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 60 96.77% 1 50.00%
Avi Kivity 2 3.23% 1 50.00%
Total 62 100.00% 2 100.00%



static void init_msrpm_offsets(void)
{
	int i;

	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));

	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
		u32 offset;

		offset = svm_msrpm_offset(direct_access_msrs[i].index);
		BUG_ON(offset == MSR_INVALID);

		add_msr_offset(offset);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 55 78.57% 2 66.67%
Avi Kivity 15 21.43% 1 33.33%
Total 70 100.00% 3 100.00%



static void svm_enable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 70 95.89% 1 33.33%
Janakarajan Natarajan 3 4.11% 2 66.67%
Total 73 100.00% 3 100.00%



static void svm_disable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 70 94.59% 1 33.33%
Janakarajan Natarajan 4 5.41% 2 66.67%
Total 74 100.00% 3 100.00%



static void disable_nmi_singlestep(struct vcpu_svm *svm)
{
	svm->nmi_singlestep = false;
	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
		/* Clear our flags if they were not set by the guest */
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Ladi Prosek 77 100.00% 2 100.00%
Total 77 100.00% 2 100.00%

/* Note:
 * This hash table is used to map VM_ID to a struct kvm_arch,
 * when handling AMD IOMMU GALOG notification to schedule in
 * a particular vCPU.
 */

#define SVM_VM_DATA_HASH_BITS	8
static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
static DEFINE_SPINLOCK(svm_vm_data_hash_lock);

/* Note:
 * This function is called from IOMMU driver to notify
 * SVM to schedule in a particular vCPU of a particular VM.
 */


static int avic_ga_log_notifier(u32 ga_tag)
{
	unsigned long flags;
	struct kvm_arch *ka = NULL;
	struct kvm_vcpu *vcpu = NULL;
	u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
	u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);

	pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);

	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
	hash_for_each_possible(svm_vm_data_hash, ka, hnode, vm_id) {
		struct kvm *kvm = container_of(ka, struct kvm, arch);
		struct kvm_arch *vm_data = &kvm->arch;

		if (vm_data->avic_vm_id != vm_id)
			continue;
		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
		break;
	}
	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);

	if (!vcpu)
		return 0;

	/* Note:
         * At this point, the IOMMU should have already set the pending
         * bit in the vAPIC backing page. So, we just need to schedule
         * in the vcpu.
         */
	if (vcpu->mode == OUTSIDE_GUEST_MODE)
		kvm_vcpu_wake_up(vcpu);

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 151 100.00% 1 100.00%
Total 151 100.00% 1 100.00%



static __init int svm_hardware_setup(void)
{
	int cpu;
	struct page *iopm_pages;
	void *iopm_va;
	int r;

	iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);

	if (!iopm_pages)
		return -ENOMEM;

	iopm_va = page_address(iopm_pages);
	memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;

	init_msrpm_offsets();

	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
		kvm_enable_efer_bits(EFER_FFXSR);

	if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
		kvm_has_tsc_control = true;
		kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
		kvm_tsc_scaling_ratio_frac_bits = 32;
	}

	if (nested) {
		printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
	}

	for_each_possible_cpu(cpu) {
		r = svm_cpu_init(cpu);
		if (r)
			goto err;
	}

	if (!boot_cpu_has(X86_FEATURE_NPT))
		npt_enabled = false;

	if (npt_enabled && !npt) {
		printk(KERN_INFO "kvm: Nested Paging disabled\n");
		npt_enabled = false;
	}

	if (npt_enabled) {
		printk(KERN_INFO "kvm: Nested Paging enabled\n");
		kvm_enable_tdp();
	} else
		kvm_disable_tdp();

	if (avic) {
		if (!npt_enabled ||
		    !boot_cpu_has(X86_FEATURE_AVIC) ||
		    !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
			avic = false;
		} else {
			pr_info("AVIC enabled\n");

			amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
		}
	}

	if (vls) {
		if (!npt_enabled ||
		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
		    !IS_ENABLED(CONFIG_X86_64)) {
			vls = false;
		} else {
			pr_info("Virtual VMLOAD VMSAVE supported\n");
		}
	}

	return 0;

err:
	__free_pages(iopm_pages, IOPM_ALLOC_ORDER);
	iopm_base = 0;
	return r;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 152 49.19% 9 45.00%
Suravee Suthikulpanit 43 13.92% 3 15.00%
Avi Kivity 39 12.62% 2 10.00%
Janakarajan Natarajan 36 11.65% 1 5.00%
Alexander Graf 29 9.39% 2 10.00%
Haozhong Zhang 8 2.59% 1 5.00%
Zachary Amsden 1 0.32% 1 5.00%
Borislav Petkov 1 0.32% 1 5.00%
Total 309 100.00% 20 100.00%



static __exit void svm_hardware_unsetup(void)
{
	int cpu;

	for_each_possible_cpu(cpu)
		svm_cpu_uninit(cpu);

	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
	iopm_base = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 25 67.57% 1 33.33%
Joerg Roedel 11 29.73% 1 33.33%
Zachary Amsden 1 2.70% 1 33.33%
Total 37 100.00% 3 100.00%



static void init_seg(struct vmcb_seg *seg)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
	seg->limit = 0xffff;
	seg->base = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 40 100.00% 1 100.00%
Total 40 100.00% 1 100.00%



static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | type;
	seg->limit = 0xffff;
	seg->base = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 40 100.00% 1 100.00%
Total 40 100.00% 1 100.00%



static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 g_tsc_offset = 0;

	if (is_guest_mode(vcpu)) {
		g_tsc_offset = svm->vmcb->control.tsc_offset -
			       svm->nested.hsave->control.tsc_offset;
		svm->nested.hsave->control.tsc_offset = offset;
	} else
		trace_kvm_write_tsc_offset(vcpu->vcpu_id,
					   svm->vmcb->control.tsc_offset,
					   offset);

	svm->vmcb->control.tsc_offset = offset + g_tsc_offset;

	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
}

Contributors
Person Tokens Prop Commits CommitProp
Zachary Amsden 80 73.39% 1 25.00%
Yoshihiro Yunomae 18 16.51% 1 25.00%
Joerg Roedel 11 10.09% 2 50.00%
Total 109 100.00% 4 100.00%



static void avic_init_vmcb(struct vcpu_svm *svm)
{
	struct vmcb *vmcb = svm->vmcb;
	struct kvm_arch *vm_data = &svm->vcpu.kvm->arch;
	phys_addr_t bpa = page_to_phys(svm->avic_backing_page);
	phys_addr_t lpa = page_to_phys(vm_data->avic_logical_id_table_page);
	phys_addr_t ppa = page_to_phys(vm_data->avic_physical_id_table_page);

	vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
	vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
	vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
	vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
	vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
	svm->vcpu.arch.apicv_active = true;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 91 75.83% 1 20.00%
Avi Kivity 19 15.83% 2 40.00%
Joerg Roedel 10 8.33% 2 40.00%
Total 120 100.00% 5 100.00%



static void init_vmcb(struct vcpu_svm *svm)
{
	struct vmcb_control_area *control = &svm->vmcb->control;
	struct vmcb_save_area *save = &svm->vmcb->save;

	svm->vcpu.arch.hflags = 0;

	set_cr_intercept(svm, INTERCEPT_CR0_READ);
	set_cr_intercept(svm, INTERCEPT_CR3_READ);
	set_cr_intercept(svm, INTERCEPT_CR4_READ);
	set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
	set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
	set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
	if (!kvm_vcpu_apicv_active(&svm->vcpu))
		set_cr_intercept(svm, INTERCEPT_CR8_WRITE);

	set_dr_intercepts(svm);

	set_exception_intercept(svm, PF_VECTOR);
	set_exception_intercept(svm, UD_VECTOR);
	set_exception_intercept(svm, MC_VECTOR);
	set_exception_intercept(svm, AC_VECTOR);
	set_exception_intercept(svm, DB_VECTOR);

	set_intercept(svm, INTERCEPT_INTR);
	set_intercept(svm, INTERCEPT_NMI);
	set_intercept(svm, INTERCEPT_SMI);
	set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
	set_intercept(svm, INTERCEPT_RDPMC);
	set_intercept(svm, INTERCEPT_CPUID);
	set_intercept(svm, INTERCEPT_INVD);
	set_intercept(svm, INTERCEPT_HLT);
	set_intercept(svm, INTERCEPT_INVLPG);
	set_intercept(svm, INTERCEPT_INVLPGA);
	set_intercept(svm, INTERCEPT_IOIO_PROT);
	set_intercept(svm, INTERCEPT_MSR_PROT);
	set_intercept(svm, INTERCEPT_TASK_SWITCH);
	set_intercept(svm, INTERCEPT_SHUTDOWN);
	set_intercept(svm, INTERCEPT_VMRUN);
	set_intercept(svm, INTERCEPT_VMMCALL);
	set_intercept(svm, INTERCEPT_VMLOAD);
	set_intercept(svm, INTERCEPT_VMSAVE);
	set_intercept(svm, INTERCEPT_STGI);
	set_intercept(svm, INTERCEPT_CLGI);
	set_intercept(svm, INTERCEPT_SKINIT);
	set_intercept(svm, INTERCEPT_WBINVD);
	set_intercept(svm, INTERCEPT_XSETBV);

	if (!kvm_mwait_in_guest()) {
		set_intercept(svm, INTERCEPT_MONITOR);
		set_intercept(svm, INTERCEPT_MWAIT);
	}

	control->iopm_base_pa = iopm_base;
	control->msrpm_base_pa = __pa(svm->msrpm);
	control->int_ctl = V_INTR_MASKING_MASK;

	init_seg(&save->es);
	init_seg(&save->ss);
	init_seg(&save->ds);
	init_seg(&save->fs);
	init_seg(&save->gs);

	save->cs.selector = 0xf000;
	save->cs.base = 0xffff0000;
	/* Executable/Readable Code Segment */
	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
	save->cs.limit = 0xffff;

	save->gdtr.limit = 0xffff;
	save->idtr.limit = 0xffff;

	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);

	svm_set_efer(&svm->vcpu, 0);
	save->dr6 = 0xffff0ff0;
	kvm_set_rflags(&svm->vcpu, 2);
	save->rip = 0x0000fff0;
	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;

	/*
         * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
         * It also updates the guest-visible cr0 value.
         */
	svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
	kvm_mmu_reset_context(&svm->vcpu);

	save->cr4 = X86_CR4_PAE;
	/* rdx = ?? */

	if (npt_enabled) {
		/* Setup VMCB for Nested Paging */
		control->nested_ctl = 1;
		clr_intercept(svm, INTERCEPT_INVLPG);
		clr_exception_intercept(svm, PF_VECTOR);
		clr_cr_intercept(svm, INTERCEPT_CR3_READ);
		clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
		save->g_pat = svm->vcpu.arch.pat;
		save->cr3 = 0;
		save->cr4 = 0;
	}
	svm->asid_generation = 0;

	svm->nested.vmcb = 0;
	svm->vcpu.arch.hflags = 0;

	if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
		control->pause_filter_count = 3000;
		set_intercept(svm, INTERCEPT_PAUSE);
	}

	if (avic)
		avic_init_vmcb(svm);

	/*
         * If hardware supports Virtual VMLOAD VMSAVE then enable it
         * in VMCB and clear intercepts to avoid #VMEXIT.
         */
	if (vls) {
		clr_intercept(svm, INTERCEPT_VMLOAD);
		clr_intercept(svm, INTERCEPT_VMSAVE);
		svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
	}

	mark_all_dirty(svm->vmcb);

	enable_gif(svm);

}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 311 44.05% 16 34.78%
Avi Kivity 225 31.87% 8 17.39%
Janakarajan Natarajan 31 4.39% 1 2.17%
Marcelo Tosatti 30 4.25% 4 8.70%
Suravee Suthikulpanit 20 2.83% 2 4.35%
Paolo Bonzini 17 2.41% 4 8.70%
Langsdorf, Mark 15 2.12% 1 2.17%
Alexander Graf 14 1.98% 2 4.35%
Michael S. Tsirkin 11 1.56% 1 2.17%
Igor Mammedov 8 1.13% 1 2.17%
Eduardo Pereira Habkost 7 0.99% 1 2.17%
Eric Northup 7 0.99% 1 2.17%
Radim Krčmář 7 0.99% 1 2.17%
Anthony Liguori 1 0.14% 1 2.17%
Nadav Amit 1 0.14% 1 2.17%
Rusty Russell 1 0.14% 1 2.17%
Total 706 100.00% 46 100.00%



static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
				       unsigned int index)
{
	u64 *avic_physical_id_table;
	struct kvm_arch *vm_data = &vcpu->kvm->arch;

	if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
		return NULL;

	avic_physical_id_table = page_address(vm_data->avic_physical_id_table_page);

	return &avic_physical_id_table[index];
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 32 56.14% 1 20.00%
Avi Kivity 15 26.32% 1 20.00%
Nadav Amit 7 12.28% 1 20.00%
Julian Stecklina 2 3.51% 1 20.00%
Dan Carpenter 1 1.75% 1 20.00%
Total 57 100.00% 5 100.00%

/**
 * Note:
 * AVIC hardware walks the nested page table to check permissions,
 * but does not use the SPA address specified in the leaf page
 * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
 * field of the VMCB. Therefore, we set up the
 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
 */


static int avic_init_access_page(struct kvm_vcpu *vcpu)
{
	struct kvm *kvm = vcpu->kvm;
	int ret;

	if (kvm->arch.apic_access_page_done)
		return 0;

	ret = x86_set_memory_region(kvm,
				    APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
				    APIC_DEFAULT_PHYS_BASE,
				    PAGE_SIZE);
	if (ret)
		return ret;

	kvm->arch.apic_access_page_done = true;
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 44 67.69% 1 33.33%
Avi Kivity 12 18.46% 1 33.33%
Rusty Russell 9 13.85% 1 33.33%
Total 65 100.00% 3 100.00%



static int avic_init_backing_page(struct kvm_vcpu *vcpu)
{
	int ret;
	u64 *entry, new_entry;
	int id = vcpu->vcpu_id;
	struct vcpu_svm *svm = to_svm(vcpu);

	ret = avic_init_access_page(vcpu);
	if (ret)
		return ret;

	if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
		return -EINVAL;

	if (!svm->vcpu.arch.apic->regs)
		return -EINVAL;

	svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);

	/* Setting AVIC backing page address in the phy APIC ID table */
	entry = avic_get_physical_id_entry(vcpu, id);
	if (!entry)
		return -EINVAL;

	new_entry = READ_ONCE(*entry);
	new_entry = (page_to_phys(svm->avic_backing_page) &
		     AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
		     AVIC_PHYSICAL_ID_ENTRY_VALID_MASK;
	WRITE_ONCE(*entry, new_entry);

	svm->avic_physical_id_cache = entry;

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 111 72.08% 1 12.50%
Alexander Graf 19 12.34% 2 25.00%
Rusty Russell 8 5.19% 1 12.50%
Joerg Roedel 7 4.55% 2 25.00%
Takuya Yoshikawa 6 3.90% 1 12.50%
Avi Kivity 3 1.95% 1 12.50%
Total 154 100.00% 8 100.00%



static inline int avic_get_next_vm_id(void)
{
	int id;

	spin_lock(&avic_vm_id_lock);

	/* AVIC VM ID is one-based. */
	id = find_next_zero_bit(avic_vm_id_bitmap, AVIC_VM_ID_NR, 1);
	if (id <= AVIC_VM_ID_MASK)
		__set_bit(id, avic_vm_id_bitmap);
	else
		id = -EAGAIN;

	spin_unlock(&avic_vm_id_lock);
	return id;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 58 100.00% 1 100.00%
Total 58 100.00% 1 100.00%



static inline int avic_free_vm_id(int id)
{
	if (id <= 0 || id > AVIC_VM_ID_MASK)
		return -EINVAL;

	spin_lock(&avic_vm_id_lock);
	__clear_bit(id, avic_vm_id_bitmap);
	spin_unlock(&avic_vm_id_lock);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 46 100.00% 1 100.00%
Total 46 100.00% 1 100.00%



static void avic_vm_destroy(struct kvm *kvm)
{
	unsigned long flags;
	struct kvm_arch *vm_data = &kvm->arch;

	if (!avic)
		return;

	avic_free_vm_id(vm_data->avic_vm_id);

	if (vm_data->avic_logical_id_table_page)
		__free_page(vm_data->avic_logical_id_table_page);
	if (vm_data->avic_physical_id_table_page)
		__free_page(vm_data->avic_physical_id_table_page);

	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
	hash_del(&vm_data->hnode);
	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 74 84.09% 3 50.00%
Avi Kivity 7 7.95% 1 16.67%
Dmitriy Vyukov 6 6.82% 1 16.67%
Alexander Graf 1 1.14% 1 16.67%
Total 88 100.00% 6 100.00%



static int avic_vm_init(struct kvm *kvm)
{
	unsigned long flags;
	int vm_id, err = -ENOMEM;
	struct kvm_arch *vm_data = &kvm->arch;
	struct page *p_page;
	struct page *l_page;

	if (!avic)
		return 0;

	vm_id = avic_get_next_vm_id();
	if (vm_id < 0)
		return vm_id;
	vm_data->avic_vm_id = (u32)vm_id;

	/* Allocating physical APIC ID table (4KB) */
	p_page = alloc_page(GFP_KERNEL);
	if (!p_page)
		goto free_avic;

	vm_data->avic_physical_id_table_page = p_page;
	clear_page(page_address(p_page));

	/* Allocating logical APIC ID table (4KB) */
	l_page = alloc_page(GFP_KERNEL);
	if (!l_page)
		goto free_avic;

	vm_data->avic_logical_id_table_page = l_page;
	clear_page(page_address(l_page));

	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
	hash_add(svm_vm_data_hash, &vm_data->hnode, vm_data->avic_vm_id);
	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);

	return 0;

free_avic:
	avic_vm_destroy(kvm);
	return err;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 131 74.01% 3 30.00%
Avi Kivity 18 10.17% 1 10.00%
Colin Ian King 12 6.78% 1 10.00%
Rusty Russell 4 2.26% 1 10.00%
Langsdorf, Mark 4 2.26% 1 10.00%
Takuya Yoshikawa 4 2.26% 1 10.00%
Boris Ostrovsky 3 1.69% 1 10.00%
Alexander Graf 1 0.56% 1 10.00%
Total 177 100.00% 10 100.00%


static inline int

avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
{
	int ret = 0;
	unsigned long flags;
	struct amd_svm_iommu_ir *ir;
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!kvm_arch_has_assigned_device(vcpu->kvm))
		return 0;

	/*
         * Here, we go through the per-vcpu ir_list to update all existing
         * interrupt remapping table entry targeting this vcpu.
         */
	spin_lock_irqsave(&svm->ir_list_lock, flags);

	if (list_empty(&svm->ir_list))
		goto out;

	list_for_each_entry(ir, &svm->ir_list, node) {
		ret = amd_iommu_update_ga(cpu, r, ir->data);
		if (ret)
			break;
	}
out:
	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 87 71.31% 2 25.00%
Avi Kivity 15 12.30% 1 12.50%
Gregory Haskins 10 8.20% 1 12.50%
Alexander Graf 6 4.92% 2 25.00%
Joerg Roedel 3 2.46% 1 12.50%
Rusty Russell 1 0.82% 1 12.50%
Total 122 100.00% 8 100.00%



static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
	u64 entry;
	/* ID = 0xff (broadcast), ID > 0xff (reserved) */
	int h_physical_id = kvm_cpu_get_apicid(cpu);
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!kvm_vcpu_apicv_active(vcpu))
		return;

	if (WARN_ON(h_physical_id >= AVIC_MAX_PHYSICAL_ID_COUNT))
		return;

	entry = READ_ONCE(*(svm->avic_physical_id_cache));
	WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);

	entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
	entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);

	entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
	if (svm->avic_is_running)
		entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;

	WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
	avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
					svm->avic_is_running);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 78 62.40% 3 23.08%
Avi Kivity 26 20.80% 6 46.15%
Gregory Haskins 10 8.00% 1 7.69%
Joerg Roedel 5 4.00% 1 7.69%
Anthony Liguori 5 4.00% 1 7.69%
Marcelo Tosatti 1 0.80% 1 7.69%
Total 125 100.00% 13 100.00%



static void avic_vcpu_put(struct kvm_vcpu *vcpu)
{
	u64 entry;
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!kvm_vcpu_apicv_active(vcpu))
		return;

	entry = READ_ONCE(*(svm->avic_physical_id_cache));
	if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
		avic_update_iommu_vcpu_affinity(vcpu, -1, 0);

	entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
	WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 58 74.36% 2 33.33%
Haozhong Zhang 8 10.26% 1 16.67%
Joerg Roedel 6 7.69% 1 16.67%
Paolo Bonzini 5 6.41% 1 16.67%
Christoph Lameter 1 1.28% 1 16.67%
Total 78 100.00% 6 100.00%

/**
 * This function is called during VCPU halt/unhalt.
 */


static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->avic_is_running = is_run;
	if (is_run)
		avic_vcpu_load(vcpu, vcpu->cpu);
	else
		avic_vcpu_put(vcpu);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 49 100.00% 1 100.00%
Total 49 100.00% 1 100.00%



static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 dummy;
	u32 eax = 1;

	if (!init_event) {
		svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
					   MSR_IA32_APICBASE_ENABLE;
		if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
			svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
	}
	init_vmcb(svm);

	kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy);
	kvm_register_write(vcpu, VCPU_REGS_RDX, eax);

	if (kvm_vcpu_apicv_active(vcpu) && !init_event)
		avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 33 27.73% 4 36.36%
Nadav Amit 30 25.21% 1 9.09%
Julian Stecklina 23 19.33% 1 9.09%
Gregory Haskins 14 11.76% 1 9.09%
Suravee Suthikulpanit 11 9.24% 1 9.09%
Anthony Liguori 5 4.20% 1 9.09%
Marcelo Tosatti 2 1.68% 1 9.09%
Joerg Roedel 1 0.84% 1 9.09%
Total 119 100.00% 11 100.00%



static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
{
	struct vcpu_svm *svm;
	struct page *page;
	struct page *msrpm_pages;
	struct page *hsave_page;
	struct page *nested_msrpm_pages;
	int err;

	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
	if (!svm) {
		err = -ENOMEM;
		goto out;
	}

	err = kvm_vcpu_init(&svm->vcpu, kvm, id);
	if (err)
		goto free_svm;

	err = -ENOMEM;
	page = alloc_page(GFP_KERNEL);
	if (!page)
		goto uninit;

	msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!msrpm_pages)
		goto free_page1;

	nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!nested_msrpm_pages)
		goto free_page2;

	hsave_page = alloc_page(GFP_KERNEL);
	if (!hsave_page)
		goto free_page3;

	if (avic) {
		err = avic_init_backing_page(&svm->vcpu);
		if (err)
			goto free_page4;

		INIT_LIST_HEAD(&svm->ir_list);
		spin_lock_init(&svm->ir_list_lock);
	}

	/* We initialize this flag to true to make sure that the is_running
         * bit would be set the first time the vcpu is loaded.
         */
	svm->avic_is_running = true;

	svm->nested.hsave = page_address(hsave_page);

	svm->msrpm = page_address(msrpm_pages);
	svm_vcpu_init_msrpm(svm->msrpm);

	svm->nested.msrpm = page_address(nested_msrpm_pages);
	svm_vcpu_init_msrpm(svm->nested.msrpm);

	svm->vmcb = page_address(page);
	clear_page(svm->vmcb);
	svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
	svm->asid_generation = 0;
	init_vmcb(svm);

	svm_init_osvw(&svm->vcpu);

	return &svm->vcpu;

free_page4:
	__free_page(hsave_page);
free_page3:
	__free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
free_page2:
	__free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
free_page1:
	__free_page(page);
uninit:
	kvm_vcpu_uninit(&svm->vcpu);
free_svm:
	kmem_cache_free(kvm_vcpu_cache, svm);
out:
	return ERR_PTR(err);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 89 24.52% 2 10.53%
Rusty Russell 61 16.80% 3 15.79%
Alexander Graf 61 16.80% 3 15.79%
Suravee Suthikulpanit 53 14.60% 3 15.79%
Takuya Yoshikawa 49 13.50% 1 5.26%
Joerg Roedel 35 9.64% 3 15.79%
Boris Ostrovsky 8 2.20% 1 5.26%
Huaitong Han 4 1.10% 1 5.26%
Gregory Haskins 2 0.55% 1 5.26%
Shani Moideen 1 0.28% 1 5.26%
Total 363 100.00% 19 100.00%



static void svm_free_vcpu(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	__free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
	__free_page(virt_to_page(svm->nested.hsave));
	__free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
	kvm_vcpu_uninit(vcpu);
	kmem_cache_free(kvm_vcpu_cache, svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 27 32.53% 1 12.50%
Alexander Graf 21 25.30% 2 25.00%
Joerg Roedel 17 20.48% 2 25.00%
Gregory Haskins 10 12.05% 1 12.50%
Rusty Russell 8 9.64% 2 25.00%
Total 83 100.00% 8 100.00%



static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	int i;

	if (unlikely(cpu != vcpu->cpu)) {
		svm->asid_generation = 0;
		mark_all_dirty(svm->vmcb);
	}

#ifdef CONFIG_X86_64
	rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
#endif
	savesegment(fs, svm->host.fs);
	savesegment(gs, svm->host.gs);
	svm->host.ldt = kvm_read_ldt();

	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
		rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);

	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
		u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
		if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
			__this_cpu_write(current_tsc_ratio, tsc_ratio);
			wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
		}
	}
	/* This assumes that the kernel never uses MSR_TSC_AUX */
	if (static_cpu_has(X86_FEATURE_RDTSCP))
		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);

	avic_vcpu_load(vcpu, cpu);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 79 39.90% 6 40.00%
Joerg Roedel 33 16.67% 2 13.33%
Anthony Liguori 29 14.65% 1 6.67%
Paolo Bonzini 17 8.59% 1 6.67%
Haozhong Zhang 13 6.57% 1 6.67%
Gregory Haskins 10 5.05% 1 6.67%
Suravee Suthikulpanit 7 3.54% 1 6.67%
Marcelo Tosatti 6 3.03% 1 6.67%
Christoph Lameter 4 2.02% 1 6.67%
Total 198 100.00% 15 100.00%



static void svm_vcpu_put(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	int i;

	avic_vcpu_put(vcpu);

	++vcpu->stat.host_state_reload;
	kvm_load_ldt(svm->host.ldt);
#ifdef CONFIG_X86_64
	loadsegment(fs, svm->host.fs);
	wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
	load_gs_index(svm->host.gs);
#else
#ifdef CONFIG_X86_32_LAZY_GS
	loadsegment(gs, svm->host.gs);
#endif
#endif
	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
		wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 72 56.69% 4 44.44%
Anthony Liguori 31 24.41% 1 11.11%
Gregory Haskins 10 7.87% 1 11.11%
Joerg Roedel 8 6.30% 1 11.11%
Suravee Suthikulpanit 5 3.94% 1 11.11%
Linus Torvalds 1 0.79% 1 11.11%
Total 127 100.00% 9 100.00%



static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
{
	avic_set_running(vcpu, false);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%



static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
{
	avic_set_running(vcpu, true);
}

Contributors
Person Tokens Prop Commits CommitProp
Suravee Suthikulpanit 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%



static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	unsigned long rflags = svm->vmcb->save.rflags;

	if (svm->nmi_singlestep) {
		/* Hide our flags if they were not set by the guest */
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
			rflags &= ~X86_EFLAGS_TF;
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
			rflags &= ~X86_EFLAGS_RF;
	}
	return rflags;
}

Contributors
Person Tokens Prop Commits CommitProp
Ladi Prosek 55 70.51% 1 33.33%
Avi Kivity 20 25.64% 1 33.33%
Gregory Haskins 3 3.85% 1 33.33%
Total 78 100.00% 3 100.00%



static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	if (to_svm(vcpu)->nmi_singlestep)
		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);

       /*
        * Any change of EFLAGS.VM is accompanied by a reload of SS
        * (caused by either a task switch or an inter-privilege IRET),
        * so we do not need to update the CPL here.
        */
	to_svm(vcpu)->vmcb->save.rflags = rflags;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 25 54.35% 1 25.00%
Ladi Prosek 17 36.96% 1 25.00%
Gregory Haskins 3 6.52% 1 25.00%
Andrea Gelmini 1 2.17% 1 25.00%
Total 46 100.00% 4 100.00%



static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
{
	switch (reg) {
	case VCPU_EXREG_PDPTR:
		BUG_ON(!npt_enabled);
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
		break;
	default:
		BUG();
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 48 94.12% 2 66.67%
Joerg Roedel 3 5.88% 1 33.33%
Total 51 100.00% 3 100.00%



static void svm_set_vintr(struct vcpu_svm *svm)
{
	set_intercept(svm, INTERCEPT_VINTR);
}

Contributors
Person Tokens Prop Commits CommitProp
Alexander Graf 14 77.78% 1 50.00%
Joerg Roedel 4 22.22% 1 50.00%
Total 18 100.00% 2 100.00%



static void svm_clear_vintr(struct vcpu_svm *svm)
{
	clr_intercept(svm, INTERCEPT_VINTR);
}

Contributors
Person Tokens Prop Commits CommitProp
Alexander Graf 14 77.78% 1 50.00%
Joerg Roedel 4 22.22% 1 50.00%
Total 18 100.00% 2 100.00%



static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
{
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;

	switch (seg) {
	case VCPU_SREG_CS: return &save->cs;
	case VCPU_SREG_DS: return &save->ds;
	case VCPU_SREG_ES: return &save->es;
	case VCPU_SREG_FS: return &save->fs;
	case VCPU_SREG_GS: return &save->gs;
	case VCPU_SREG_SS: return &save->ss;
	case VCPU_SREG_TR: return &save->tr;
	case VCPU_SREG_LDTR: return &save->ldtr;
	}
	BUG();
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 111 96.52% 1 33.33%
Gregory Haskins 3 2.61% 1 33.33%
Al Viro 1 0.87% 1 33.33%
Total 115 100.00% 3 100.00%



static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	return s->base;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%



static void svm_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	var->base = s->base;
	var->limit = s->limit;
	var->selector = s->selector;
	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;

	/*
         * AMD CPUs circa 2014 track the G bit for all segments except CS.
         * However, the SVM spec states that the G bit is not observed by the
         * CPU, and some VMware virtual CPUs drop the G bit for all segments.
         * So let's synthesize a legal G bit for all segments, this helps
         * running KVM nested. It also helps cross-vendor migration, because
         * Intel's vmentry has a check on the 'G' bit.
         */
	var->g = s->limit > 0xfffff;

	/*
         * AMD's VMCB does not have an explicit unusable field, so emulate it
         * for cross vendor migration purposes by "not present"
         */
	var->unusable = !var->present;

	switch (seg) {
	case VCPU_SREG_TR:
		/*
                 * Work around a bug where the busy flag in the tr selector
                 * isn't exposed
                 */
		var->type |= 0x2;
		break;
	case VCPU_SREG_DS:
	case VCPU_SREG_ES:
	case VCPU_SREG_FS:
	case VCPU_SREG_GS:
		/*
                 * The accessed bit must always be set in the segment
                 * descriptor cache, although it can be cleared in the
                 * descriptor, the cached bit always remains at 1. Since
                 * Intel has a check on this, set it here to support
                 * cross-vendor migration.
                 */
		if (!var->unusable)
			var->type |= 0x1;
		break;
	case VCPU_SREG_SS:
		/*
                 * On AMD CPUs sometimes the DB bit in the segment
                 * descriptor is left as 1, although the whole segment has
                 * been made unusable. Clear it here to pass an Intel VMX
                 * entry check when cross vendor migrating.
                 */
		if (var->unusable)
			var->db = 0;
		/* This is symmetric with svm_set_segment() */
		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
		break;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 156 63.16% 1 10.00%
Andre Przywara 58 23.48% 3 30.00%
Jan Kiszka 15 6.07% 1 10.00%
Amit Shah 11 4.45% 2 20.00%
Jim Mattson 4 1.62% 1 10.00%
Joerg Roedel 2 0.81% 1 10.00%
Roman Peniaev 1 0.40% 1 10.00%
Total 247 100.00% 10 100.00%



static int svm_get_cpl(struct kvm_vcpu *vcpu)
{
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;

	return save->cpl;
}

Contributors
Person Tokens Prop Commits CommitProp
Izik Eidus 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%



static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	dt->size = svm->vmcb->save.idtr.limit;
	dt->address = svm->vmcb->save.idtr.base;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 31 57.41% 1 20.00%
Gregory Haskins 10 18.52% 1 20.00%
Jan Kiszka 8 14.81% 1 20.00%
Gleb Natapov 3 5.56% 1 20.00%
Leonard Norrgård 2 3.70% 1 20.00%
Total 54 100.00% 5 100.00%



static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.idtr.limit = dt->size;
	svm->vmcb->save.idtr.base = dt->address ;
	mark_dirty(svm->vmcb, VMCB_DT);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 24 38.10% 1 16.67%
Izik Eidus 22 34.92% 1 16.67%
Joerg Roedel 9 14.29% 1 16.67%
Gleb Natapov 3 4.76% 1 16.67%
Gregory Haskins 3 4.76% 1 16.67%
Leonard Norrgård 2 3.17% 1 16.67%
Total 63 100.00% 6 100.00%



static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	dt->size = svm->vmcb->save.gdtr.limit;
	dt->address = svm->vmcb->save.gdtr.base;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 41 75.93% 1 33.33%
Gregory Haskins 10 18.52% 1 33.33%
Gleb Natapov 3 5.56% 1 33.33%
Total 54 100.00% 3 100.00%



static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.gdtr.limit = dt->size;
	svm->vmcb->save.gdtr.base = dt->address ;
	mark_dirty(svm->vmcb, VMCB_DT);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 42 66.67% 1 25.00%
Joerg Roedel 9 14.29% 1 25.00%
Gregory Haskins 9 14.29% 1 25.00%
Gleb Natapov 3 4.76% 1 25.00%
Total 63 100.00% 4 100.00%



static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 10 100.00% 1 100.00%
Total 10 100.00% 1 100.00%



static void svm_decache_cr3(struct kvm_vcpu *vcpu)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 10 100.00% 1 100.00%
Total 10 100.00% 1 100.00%



static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 9 90.00% 1 50.00%
Anthony Liguori 1 10.00% 1 50.00%
Total 10 100.00% 2 100.00%



static void update_cr0_intercept(struct vcpu_svm *svm)
{
	ulong gcr0 = svm->vcpu.arch.cr0;
	u64 *hcr0 = &svm->vmcb->save.cr0;

	*hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
		| (gcr0 & SVM_CR0_SELECTIVE_MASK);

	mark_dirty(svm->vmcb, VMCB_CR);

	if (gcr0 == *hcr0) {
		clr_cr_intercept(svm, INTERCEPT_CR0_READ);
		clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
	} else {
		set_cr_intercept(svm, INTERCEPT_CR0_READ);
		set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 67 66.34% 2 33.33%
Joerg Roedel 34 33.66% 4 66.67%
Total 101 100.00% 6 100.00%



static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
	struct vcpu_svm *svm = to_svm(vcpu);

#ifdef CONFIG_X86_64
	if (vcpu->arch.efer & EFER_LME) {
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
			vcpu->arch.efer |= EFER_LMA;
			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
		}

		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
			vcpu->arch.efer &= ~EFER_LMA;
			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
		}
	}
#endif
	vcpu->arch.cr0 = cr0;

	if (!npt_enabled)
		cr0 |= X86_CR0_PG | X86_CR0_WP;

	/*
         * re-enable caching here because the QEMU bios
         * does not do it - this results in some delay at
         * reboot
         */
	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
		cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
	svm->vmcb->save.cr0 = cr0;
	mark_dirty(svm->vmcb, VMCB_CR);
	update_cr0_intercept(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 124 68.13% 7 50.00%
Joerg Roedel 18 9.89% 2 14.29%
Paolo Bonzini 12 6.59% 1 7.14%
Gregory Haskins 10 5.49% 1 7.14%
Carlo Marcelo Arenas Belon 7 3.85% 1 7.14%
Xiantao Zhang 7 3.85% 1 7.14%
Rusty Russell 4 2.20% 1 7.14%
Total 182 100.00% 14 100.00%



static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
	unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;

	if (cr4 & X86_CR4_VMXE)
		return 1;

	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
		svm_flush_tlb(vcpu);

	vcpu->arch.cr4 = cr4;
	if (!npt_enabled)
		cr4 |= X86_CR4_PAE;
	cr4 |= host_cr4_mce;
	to_svm(vcpu)->vmcb->save.cr4 = cr4;
	mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 61 52.59% 6 54.55%
Avi Kivity 33 28.45% 1 9.09%
Nadav Har'El 13 11.21% 1 9.09%
Gregory Haskins 7 6.03% 1 9.09%
Andrew Lutomirski 1 0.86% 1 9.09%
Xiantao Zhang 1 0.86% 1 9.09%
Total 116 100.00% 11 100.00%



static void svm_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	s->base = var->base;
	s->limit = var->limit;
	s->selector = var->selector;
	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;

	/*
         * This is always accurate, except if SYSRET returned to a segment
         * with SS.DPL != 3.  Intel does not have this quirk, and always
         * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
         * would entail passing the CPL to userspace and back.
         */
	if (seg == VCPU_SREG_SS)
		/* This is symmetric with svm_get_segment() */
		svm->vmcb->save.cpl = (var->dpl & 3);

	mark_dirty(svm->vmcb, VMCB_SEG);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 176 81.86% 4 36.36%
Joerg Roedel 12 5.58% 4 36.36%
Paolo Bonzini 11 5.12% 1 9.09%
Roman Peniaev 11 5.12% 1 9.09%
Gregory Haskins 5 2.33% 1 9.09%
Total 215 100.00% 11 100.00%



static void update_bp_intercept(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	clr_exception_intercept(svm, BP_VECTOR);

	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
			set_exception_intercept(svm, BP_VECTOR);
	} else
		vcpu->guest_debug = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 29 48.33% 2 28.57%
Jan Kiszka 11 18.33% 1 14.29%
Gregory Haskins 10 16.67% 1 14.29%
Joerg Roedel 8 13.33% 1 14.29%
Paolo Bonzini 1 1.67% 1 14.29%
Gleb Natapov 1 1.67% 1 14.29%
Total 60 100.00% 7 100.00%



static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
{
	if (sd->next_asid > sd->max_asid) {
		++sd->asid_generation;
		sd->next_asid = 1;
		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
	}

	svm->asid_generation = sd->asid_generation;
	svm->vmcb->control.asid = sd->next_asid++;

	mark_dirty(svm->vmcb, VMCB_ASID);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 53 67.09% 3 27.27%
Joerg Roedel 12 15.19% 4 36.36%
Tejun Heo 7 8.86% 1 9.09%
Jan Kiszka 3 3.80% 1 9.09%
Rusty Russell 3 3.80% 1 9.09%
Paolo Bonzini 1 1.27% 1 9.09%
Total 79 100.00% 11 100.00%



static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
{
	return to_svm(vcpu)->vmcb->save.dr6;
}

Contributors
Person Tokens Prop Commits CommitProp
Jan Kiszka 20 86.96% 1 33.33%
Avi Kivity 3 13.04% 2 66.67%
Total 23 100.00% 3 100.00%



static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.dr6 = value;
	mark_dirty(svm->vmcb, VMCB_DR);
}

Contributors
Person Tokens Prop Commits CommitProp
Jan Kiszka 23 52.27% 1 33.33%
Avi Kivity 11 25.00% 1 33.33%
Joerg Roedel 10 22.73% 1 33.33%
Total 44 100.00% 3 100.00%



static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	get_debugreg(vcpu->arch.db[0], 0);
	get_debugreg(vcpu->arch.db[1], 1);
	get_debugreg(vcpu->arch.db[2], 2);
	get_debugreg(vcpu->arch.db[3], 3);
	vcpu->arch.dr6 = svm_get_dr6(vcpu);
	vcpu->arch.dr7 = svm->vmcb->save.dr7;

	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
	set_dr_intercepts(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Bonzini 90 77.59% 1 12.50%
Joerg Roedel 11 9.48% 4 50.00%
Avi Kivity 11 9.48% 1 12.50%
Xiantao Zhang 2 1.72% 1 12.50%
Gregory Haskins 2 1.72% 1 12.50%
Total 116 100.00% 8 100.00%



static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.dr7 = value;
	mark_dirty(svm->vmcb, VMCB_DR);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 27 61.36% 1 25.00%
Gregory Haskins 9 20.45% 1 25.00%
Joerg Roedel 7 15.91% 1 25.00%
Gleb Natapov 1 2.27% 1 25.00%
Total 44 100.00% 4 100.00%



static int pf_interception(struct vcpu_svm *svm)
{
	u64 fault_address = svm->vmcb->control.exit_info_2;
	u64 error_code = svm->vmcb->control.exit_info_1;

	return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
			svm->vmcb->control.insn_bytes,
			svm->vmcb->control.insn_len, !npt_enabled);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 36 55.38% 2 25.00%
Andre Przywara 10 15.38% 1 12.50%
Wanpeng Li 7 10.77% 1 12.50%
Paolo Bonzini 5 7.69% 1 12.50%
Gleb Natapov 4 6.15% 1 12.50%
Rusty Russell 2 3.08% 1 12.50%
Tom Lendacky 1 1.54% 1 12.50%
Total 65 100.00% 8 100.00%



static int db_interception(struct vcpu_svm *svm)
{
	struct kvm_run *kvm_run = svm->vcpu.run;

	if (!(svm->vcpu.guest_debug &
	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
		!svm->nmi_singlestep) {
		kvm_queue_exception(&svm->vcpu, DB_VECTOR);
		return 1;
	}

	if (svm->nmi_singlestep) {
		disable_nmi_singlestep(svm);
	}

	if (svm->vcpu.guest_debug &
	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
		kvm_run->debug.arch.pc =
			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		return 0;
	}

	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Jan Kiszka 92 67.65% 4 40.00%
Gleb Natapov 20 14.71% 1 10.00%
Avi Kivity 16 11.76% 2 20.00%
Joerg Roedel 5 3.68% 2 20.00%
Ladi Prosek 3 2.21% 1 10.00%
Total 136 100.00% 10 100.00%



static int bp_interception(struct vcpu_svm *svm)
{
	struct kvm_run *kvm_run = svm->vcpu.run;

	kvm_run->exit_reason = KVM_EXIT_DEBUG;
	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
	kvm_run->debug.arch.exception = BP_VECTOR;
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Jan Kiszka 63 94.03% 2 66.67%
Avi Kivity 4 5.97% 1 33.33%
Total 67 100.00% 3 100.00%



static int ud_interception(struct vcpu_svm *svm)
{
	int er;

	er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
	if (er != EMULATE_DONE)
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Anthony Liguori 23 51.11% 1 25.00%
Paolo Bonzini 19 42.22% 1 25.00%
Avi Kivity 2 4.44% 1 25.00%
Sheng Yang 1 2.22% 1 25.00%
Total 45 100.00% 4 100.00%



static int ac_interception(struct vcpu_svm *svm)
{
	kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Eric Northup 19 73.08% 1 50.00%
Paolo Bonzini 7 26.92% 1 50.00%
Total 26 100.00% 2 100.00%



static bool is_erratum_383(void)
{
	int err, i;
	u64 value;

	if (!erratum_383_found)
		return false;

	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
	if (err)
		return false;

	/* Bit 62 may or may not be set for this mce */
	value &= ~(1ULL << 62);

	if (value != 0xb600000000010015ULL)
		return false;

	/* Clear MCi_STATUS registers */
	for (i = 0; i < 6; ++i)
		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);

	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
	if (!err) {
		u32 low, high;

		value &= ~(1ULL << 2);
		low    = lower_32_bits(value);
		high   = upper_32_bits(value);

		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
	}

	/* Flush tlb to evict multi-match entries */
	__flush_tlb_all();

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 129 87.76% 4 44.44%
Gleb Natapov 9 6.12% 2 22.22%
Avi Kivity 7 4.76% 2 22.22%
Andre Przywara 2 1.36% 1 11.11%
Total 147 100.00% 9 100.00%



static void svm_handle_mce(struct vcpu_svm *svm)
{
	if (is_erratum_383()) {
		/*
                 * Erratum 383 triggered. Guest state is corrupt so kill the
                 * guest.
                 */
		pr_err("KVM: Guest triggered AMD Erratum 383\n");

		kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);

		return;
	}

	/*
         * On an #MC intercept the MCE handler is not called automatically in
         * the host. So do it by hand here.
         */
	asm volatile (
                "int $0x12\n");
	/* not sure if we ever come back to this point */

	return;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 31 75.61% 3 60.00%
Gleb Natapov 9 21.95% 1 20.00%
Avi Kivity 1 2.44% 1 20.00%
Total 41 100.00% 5 100.00%



static int mc_interception(struct vcpu_svm *svm)
{
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 11 78.57% 2 50.00%
Avi Kivity 2 14.29% 1 25.00%
Gleb Natapov 1 7.14% 1 25.00%
Total 14 100.00% 4 100.00%



static int shutdown_interception(struct vcpu_svm *svm)
{
	struct kvm_run *kvm_run = svm->vcpu.run;

	/*
         * VMCB is undefined after a SHUTDOWN intercept
         * so reinitialize it.
         */
	clear_page(svm->vmcb);
	init_vmcb(svm);

	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Jan Kiszka 18 40.91% 1 20.00%
Joerg Roedel 14 31.82% 1 20.00%
Avi Kivity 9 20.45% 1 20.00%
Gleb Natapov 2 4.55% 1 20.00%
Shani Moideen 1 2.27% 1 20.00%
Total 44 100.00% 5 100.00%



static int io_interception(struct vcpu_svm *svm)
{
	struct kvm_vcpu *vcpu = &svm->vcpu;
	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
	int size, in, string, ret;
	unsigned port;

	++svm->vcpu.stat.io_exits;
	string = (io_info & SVM_IOIO_STR_MASK) != 0;
	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
	if (string)
		return emulate_instruction(vcpu, 0) == EMULATE_DONE;

	port = io_info >> 16;
	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
	svm->next_rip = svm->vmcb->control.exit_info_2;
	ret = kvm_skip_emulated_instruction(&svm->vcpu);

	/*
         * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered
         * KVM_EXIT_DEBUG here.
         */
	if (in)
		return kvm_fast_pio_in(vcpu, size, port) && ret;
	else
		return kvm_fast_pio_out(vcpu, size, port) && ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Laurent Vivier 49 31.41% 2 15.38%
Gleb Natapov 42 26.92% 3 23.08%
Avi Kivity 19 12.18% 2 15.38%
Ladi Prosek 17 10.90% 1 7.69%
Jan Kiszka 13 8.33% 1 7.69%
Tom Lendacky 9 5.77% 1 7.69%
Guillaume Thouvenin 5 3.21% 1 7.69%
Mike Day 1 0.64% 1 7.69%
Rusty Russell 1 0.64% 1 7.69%
Total 156 100.00% 13 100.00%



static int nmi_interception(struct vcpu_svm *svm)
{
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 10 71.43% 1 50.00%
Gleb Natapov 4 28.57% 1 50.00%
Total 14 100.00% 2 100.00%



static int intr_interception(struct vcpu_svm *svm)
{
	++svm->vcpu.stat.irq_exits;
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 9 39.13% 1 33.33%
Jan Kiszka 8 34.78% 1 33.33%
Avi Kivity 6 26.09% 1 33.33%
Total 23 100.00% 3 100.00%



static int nop_on_interception(struct vcpu_svm *svm)
{
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 12 85.71% 1 50.00%
Rusty Russell 2 14.29% 1 50.00%
Total 14 100.00% 2 100.00%



static int halt_interception(struct vcpu_svm *svm)
{
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
	return kvm_emulate_halt(&svm->vcpu);
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 15 44.12% 3 50.00%
Jan Kiszka 9 26.47% 1 16.67%
Rusty Russell 5 14.71% 1 16.67%
Marcelo Tosatti 5 14.71% 1 16.67%
Total 34 100.00% 6 100.00%



static int vmmcall_interception(struct vcpu_svm *svm)
{
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	return kvm_emulate_hypercall(&svm->vcpu);
}

Contributors
Person Tokens Prop Commits CommitProp
Anthony Liguori 26 76.47% 1 16.67%
Avi Kivity 4 11.76% 2 33.33%
Dor Laor 2 5.88% 1 16.67%
Marcelo Tosatti 1 2.94% 1 16.67%
Andrey Smetanin 1 2.94% 1 16.67%
Total 34 100.00% 6 100.00%



static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	return svm->nested.nested_cr3;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 23 79.31% 1 50.00%
Eric Northup 6 20.69% 1 50.00%
Total 29 100.00% 2 100.00%



static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 cr3 = svm->nested.nested_cr3;
	u64 pdpte;
	int ret;

	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
				       offset_in_page(cr3) + index * 8, 8);
	if (ret)
		return 0;
	return pdpte;
}

Contributors
Person Tokens Prop Commits CommitProp
Avi Kivity 61 81.33% 1 33.33%
Eric Northup 13 17.33% 1 33.33%
Paolo Bonzini 1 1.33% 1 33.33%
Total 75 100.00% 3 100.00%



static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
				   unsigned long root)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.nested_cr3 = root;
	mark_dirty(svm->vmcb, VMCB_NPT);
	svm_flush_tlb(vcpu);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 20 40.82% 3 42.86%
Avi Kivity 18 36.73% 2 28.57%
Anthony Liguori 9 18.37% 1 14.29%
Rusty Russell 2 4.08% 1 14.29%
Total 49 100.00% 7 100.00%



static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
				       struct x86_exception *fault)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
		/*
                 * TODO: track the cause of the nested page fault, and
                 * correctly fill in the high bits of exit_info_1.
                 */
		svm->vmcb->control.exit_code = SVM_EXIT_NPF;
		svm->vmcb->control.exit_code_hi = 0;
		svm->vmcb->control.exit_info_1 = (1ULL << 32);
		svm->vmcb->control.exit_info_2 = fault->address;
	}

	svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
	svm->vmcb->control.exit_info_1 |= fault->error_code;

	/*
         * The present bit is always zero for page structure faults on real
         * hardware.
         */
	if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
		svm->vmcb->control.exit_info_1 &= ~1;

	nested_svm_vmexit(svm);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 70 48.95% 2 40.00%
Paolo Bonzini 58 40.56% 1 20.00%
Avi Kivity 15 10.49% 2 40.00%
Total 143 100.00% 5 100.00%



static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
{
	WARN_ON(mmu_is_nested(vcpu));
	kvm_init_shadow_mmu(vcpu);
	vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
	vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
	vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
	vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
	vcpu->arch.mmu.shadow_root_level = get_npt_level();
	reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 79 79.00% 4 57.14%
Avi Kivity 10 10.00% 1 14.29%
Xiao Guangrong 7 7.00% 1 14.29%
Paolo Bonzini 4 4.00% 1 14.29%
Total 100 100.00% 7 100.00%



static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 24 100.00% 3 100.00%
Total 24 100.00% 3 100.00%



static int nested_svm_check_permissions(struct vcpu_svm *svm)
{
	if (!(svm->vcpu.arch.efer & EFER_SVME) ||
	    !is_paging(&svm->vcpu)) {
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
		return 1;
	}

	if (svm->vmcb->save.cpl) {
		kvm_inject_gp(&svm->vcpu, 0);
		return 1;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Alexander Graf 50 64.10% 1 20.00%
Joerg Roedel 19 24.36% 1 20.00%
Avi Kivity 7 8.97% 2 40.00%
Rusty Russell 2 2.56% 1 20.00%
Total 78 100.00% 5 100.00%



static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code)
{
	int vmexit;

	if (!is_guest_mode(&svm->vcpu))
		return 0;

	vmexit = nested_svm_intercept(svm);
	if (vmexit != NESTED_EXIT_DONE)
		return 0;

	svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
	svm->vmcb->control.exit_code_hi = 0;
	svm->vmcb->control.exit_info_1 = error_code;

	/*
         * FIXME: we should not write CR2 when L1 intercepts an L2 #PF exception.
         * The fix is to add the ancillary datum (CR2 or DR6) to structs
         * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6 can be
         * written only when inject_pending_event runs (DR6 would written here
         * too).  This should be conditional on a new capability---if the
         * capability is disabled, kvm_multiple_exception would write the
         * ancillary information to CR2 or DR6, for backwards ABI-compatibility.
         */
	if (svm->vcpu.arch.exception.nested_apf)
		svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
	else
		svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;

	svm->nested.exit_required = true;
	return vmexit;
}

Contributors
Person Tokens Prop Commits CommitProp
Wanpeng Li 45 31.25% 1 8.33%
Alexander Graf 37 25.69% 2 16.67%
Joerg Roedel 20 13.89% 3 25.00%
Avi Kivity 18 12.50% 1 8.33%
Gleb Natapov 15 10.42% 1 8.33%
Guillaume Thouvenin 4 2.78% 1 8.33%
Rusty Russell 2 1.39% 1 8.33%
Laurent Vivier 2 1.39% 1 8.33%
Paolo Bonzini 1 0.69% 1 8.33%
Total 144 100.00% 12 100.00%

/* This function returns true if it is save to enable the irq window */


static inline bool nested_svm_intr(struct vcpu_svm *svm)
{
	if (!is_guest_mode(&svm->vcpu))
		return true;

	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
		return true;

	if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
		return false;

	/*
         * if vmexit was already requested (by intercepted exception
         * for instance) do not overwrite it with "external interrupt"
         * vmexit.
         */
	if (svm->nested.exit_required)
		return false;

	svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
	svm->vmcb->control.exit_info_1 = 0;
	svm->vmcb->control.exit_info_2 = 0;

	if (svm->nested.intercept & 1ULL) {
		/*
                 * The #vmexit can't be emulated here directly because this
                 * code path runs with irqs and preemption disabled. A
                 * #vmexit emulation might sleep. Only signal request for
                 * the #vmexit here.
                 */
		svm->nested.exit_required = true;
		trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
		return false;
	}

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 68 47.89% 8 66.67%
Alexander Graf 58 40.85% 1 8.33%
Gleb Natapov 12 8.45% 1 8.33%
Avi Kivity 3 2.11% 1 8.33%
Guo Chao 1 0.70% 1 8.33%
Total 142 100.00% 12 100.00%

/* This function returns true if it is save to enable the nmi window */


static inline bool nested_svm_nmi(struct vcpu_svm *svm)
{
	if (!is_guest_mode(&svm->vcpu))
		return true;

	if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
		return true;

	svm->vmcb->control.exit_code = SVM_EXIT_NMI;
	svm->nested.exit_required = true;

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 24 35.82% 3 30.00%
Alexander Graf 16 23.88% 1 10.00%
Avi Kivity 15 22.39% 2 20.00%
Marcelo Tosatti 4 5.97% 1 10.00%
Rusty Russell 4 5.97% 1 10.00%
Dor Laor 3 4.48% 1 10.00%
Andrey Smetanin 1 1.49% 1 10.00%
Total 67 100.00% 10 100.00%



static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
{
	struct page *page;

	might_sleep();

	page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
	if (is_error_page(page))
		goto error;

	*_page = page;

	return kmap(page);

error:
	kvm_inject_gp(&svm->vcpu, 0);

	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 56 70.89% 4 66.67%
Avi Kivity 22 27.85% 1 16.67%
Paolo Bonzini 1 1.27% 1 16.67%
Total 79 100.00% 6 100.00%



static void nested_svm_unmap(struct page *page)
{
	kunmap(page);
	kvm_release_page_dirty(page);
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 21 100.00% 4 100.00%
Total 21 100.00% 4 100.00%



static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
{
	unsigned port, size, iopm_len;
	u16 val, mask;
	u8 start_bit;
	u64 gpa;

	if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
		return NESTED_EXIT_HOST;

	port = svm->vmcb->control.exit_info_1 >> 16;
	size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
		SVM_IOIO_SIZE_SHIFT;
	gpa  = svm->nested.vmcb_iopm + (port / 8);
	start_bit = port % 8;
	iopm_len = (start_bit + size > 8) ? 2 : 1;
	mask = (0xf >> (4 - size)) << start_bit;
	val = 0;

	if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
		return NESTED_EXIT_DONE;

	return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 94 58.75% 4 50.00%
Jan Kiszka 56 35.00% 1 12.50%
Paolo Bonzini 8 5.00% 2 25.00%
Alexander Graf 2 1.25% 1 12.50%
Total 160 100.00% 8 100.00%



static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
{
	u32 offset, msr, value;
	int write, mask;

	if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
		return NESTED_EXIT_HOST;

	msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
	offset = svm_msrpm_offset(msr);
	write  = svm->vmcb->control.exit_info_1 & 1;
	mask   = 1 << ((2 * (msr & 0xf)) + write);

	if (offset == MSR_INVALID)
		return NESTED_EXIT_DONE;

	/* Offset is in 32 bit units but need in 8 bit units */
	offset *= 4;

	if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
		return NESTED_EXIT_DONE;

	return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 111 77.08% 6 66.67%
Paolo Bonzini 24 16.67% 2 22.22%
Alexander Graf 9 6.25% 1 11.11%
Total 144 100.00% 9 100.00%

/* DB exceptions for our internal use must not cause vmexit */


static int nested_svm_intercept_db(struct vcpu_svm *svm)
{
	unsigned long dr6;

	/* if we're not singlestepping, it's not ours */
	if (!svm->nmi_singlestep)
		return NESTED_EXIT_DONE;

	/* if it's not a singlestep exception, it's not ours */
	if (kvm_get_dr(&svm->vcpu, 6, &dr6))
		return NESTED_EXIT_DONE;
	if (!(dr6 & DR6_BS))
		return NESTED_EXIT_DONE;

	/* if the guest is singlestepping, it should get the vmexit */
	if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
		disable_nmi_singlestep(svm);
		return NESTED_EXIT_DONE;
	}

	/* it's ours, the nested hypervisor must not see this one */
	return NESTED_EXIT_HOST;
}

Contributors
Person Tokens Prop Commits CommitProp
Ladi Prosek 80 100.00% 1 100.00%
Total 80 100.00% 1 100.00%



static int nested_svm_exit_special(struct vcpu_svm *svm)
{
	u32 exit_code = svm->vmcb->control.exit_code;

	switch (exit_code) {
	case SVM_EXIT_INTR:
	case SVM_EXIT_NMI:
	case SVM_EXIT_EXCP_BASE + MC_VECTOR:
		return NESTED_EXIT_HOST;
	case SVM_EXIT_NPF:
		/* For now we are always handling NPFs when using them */
		if (npt_enabled)
			return NESTED_EXIT_HOST;
		break;
	case SVM_EXIT_EXCP_BASE + PF_VECTOR:
		/* When we're shadowing, trap PFs, but not async PF */
		if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
			return NESTED_EXIT_HOST;
		break;
	default:
		break;
	}

	return NESTED_EXIT_CONTINUE;
}

Contributors
Person Tokens Prop Commits CommitProp
Alexander Graf 44 51.16% 1 11.11%
Joerg Roedel 23 26.74% 5 55.56%
Wanpeng Li 7 8.14% 1 11.11%
Gleb Natapov 6 6.98% 1 11.11%
Paolo Bonzini 6 6.98% 1 11.11%
Total 86 100.00% 9 100.00%

/*
 * If this function returns true, this #vmexit was already handled
 */


static int nested_svm_intercept(struct vcpu_svm *svm)
{
	u32 exit_code = svm->vmcb->control.exit_code;
	int vmexit = NESTED_EXIT_HOST;

	switch (exit_code) {
	case SVM_EXIT_MSR:
		vmexit = nested_svm_exit_handled_msr(svm);
		break;
	case SVM_EXIT_IOIO:
		vmexit = nested_svm_intercept_ioio(svm);
		break;
	case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
		u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
		if (svm->nested.intercept_cr & bit)
			vmexit = NESTED_EXIT_DONE;
		break;
	}
	case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
		u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
		if (svm->nested.intercept_dr & bit)
			vmexit = NESTED_EXIT_DONE;
		break;
	}
	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
		u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
		if (svm->nested.intercept_exceptions & excp_bits) {
			if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
				vmexit = nested_svm_intercept_db(svm);
			else
				vmexit = NESTED_EXIT_DONE;
		}
		/* async page fault always cause vmexit */
		else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
			 svm->vcpu.arch.exception.nested_apf != 0)
			vmexit = NESTED_EXIT_DONE;
		break;
	}
	case SVM_EXIT_ERR: {
		vmexit = NESTED_EXIT_DONE;
		break;
	}
	default: {
		u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
		if (svm->nested.intercept & exit_bits)
			vmexit = NESTED_EXIT_DONE;
	}
	}

	return vmexit;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 102 41.98% 10 55.56%
Alexander Graf 90 37.04% 1 5.56%
Gleb Natapov 18 7.41% 1 5.56%
Ladi Prosek 18 7.41% 1 5.56%
Wanpeng Li 7 2.88% 2 11.11%
Avi Kivity 4 1.65% 1 5.56%
Paolo Bonzini 3 1.23% 1 5.56%
Xiao Guangrong 1 0.41% 1 5.56%
Total 243 100.00% 18 100.00%



static int nested_svm_exit_handled(struct vcpu_svm *svm)
{
	int vmexit;

	vmexit = nested_svm_intercept(svm);

	if (vmexit == NESTED_EXIT_DONE)
		nested_svm_vmexit(svm);

	return vmexit;
}

Contributors
Person Tokens Prop Commits CommitProp
Alexander Graf 23 65.71% 1 25.00%
Joerg Roedel 12 34.29% 3 75.00%
Total 35 100.00% 4 100.00%



static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
{
	struct vmcb_control_area *dst  = &dst_vmcb->control;
	struct vmcb_control_area *from = &from_vmcb->control;

	dst->intercept_cr         = from->intercept_cr;
	dst->intercept_dr         = from->intercept_dr;
	dst->intercept_exceptions = from->intercept_exceptions;
	dst->intercept            = from->intercept;
	dst->iopm_base_pa         = from->iopm_base_pa;
	dst->msrpm_base_pa        = from->msrpm_base_pa;
	dst->tsc_offset           = from->tsc_offset;
	dst->asid                 = from->asid;
	dst->tlb_ctl              = from->tlb_ctl;
	dst->int_ctl              = from->int_ctl;
	dst->int_vector           = from->int_vector;
	dst->int_state            = from->int_state;
	dst->exit_code            = from->exit_code;
	dst->exit_code_hi         = from->exit_code_hi;
	dst->exit_info_1          = from->exit_info_1;
	dst->exit_info_2          = from->exit_info_2;
	dst->exit_int_info        = from->exit_int_info;
	dst->exit_int_info_err    = from->exit_int_info_err;
	dst->nested_ctl           = from->nested_ctl;
	dst->event_inj            = from->event_inj;
	dst->event_inj_err        = from->event_inj_err;
	dst->nested_cr3           = from->nested_cr3;
	dst->virt_ext              = from->virt_ext;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 193 87.33% 5 62.50%
Alexander Graf 26 11.76% 2 25.00%
Janakarajan Natarajan 2 0.90% 1 12.50%
Total 221 100.00% 8 100.00%



static int nested_svm_vmexit(struct vcpu_svm *svm)
{
	struct vmcb *nested_vmcb;
	struct vmcb *hsave = svm->nested.hsave;
	struct vmcb *vmcb = svm->vmcb;
	struct page *page;

	trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
				       vmcb->control.exit_info_1,
				       vmcb->control.exit_info_2,
				       vmcb->control.exit_int_info,
				       vmcb->control.exit_int_info_err,
				       KVM_ISA_SVM);

	nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
	if (!nested_vmcb)
		return 1;

	/* Exit Guest-Mode */
	leave_guest_mode(&svm->vcpu);
	svm->nested.vmcb = 0;

	/* Give the current vmcb to the guest */
	disable_gif(svm);

	nested_vmcb->save.es     = vmcb->save.es;
	nested_vmcb->save.cs     = vmcb->save.cs;
	nested_vmcb->save.ss     = vmcb->save.ss;
	nested_vmcb->save.ds     = vmcb->save.ds;
	nested_vmcb->save.gdtr   = vmcb->save.gdtr;
	nested_vmcb->save.idtr   = vmcb->save.idtr;
	nested_vmcb->save.efer   = svm->vcpu.arch.efer;
	nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
	nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
	nested_vmcb->save.cr2    = vmcb->save.cr2;
	nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
	nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
	nested_vmcb->save.rip    = vmcb->save.rip;
	nested_vmcb->save.rsp    = vmcb->save.rsp;
	nested_vmcb->save.rax    = vmcb->save.rax;
	nested_vmcb->save.dr7    = vmcb->save.dr7;
	nested_vmcb->save.dr6    = vmcb->save.dr6;
	nested_vmcb->save.cpl    = vmcb->save.cpl;

	nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
	nested_vmcb->control.int_vector        = vmcb->control.int_vector;
	nested_vmcb->control.int_state         = vmcb->control.int_state;
	nested_vmcb->control.exit_code         = vmcb->control.exit_code;
	nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
	nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
	nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
	nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
	nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;

	if (svm->nrips_enabled)
		nested_vmcb->control.next_rip  = vmcb->control.next_rip;

	/*
         * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
         * to make sure that we do not lose injected events. So check event_inj
         * here and copy it to exit_int_info if it is valid.
         * Exit_int_info and event_inj can't be both valid because the case
         * below only happens on a VMRUN instruction intercept which has
         * no valid exit_int_info set.
         */
	if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
		struct vmcb_control_area *nc = &nested_vmcb->control;

		nc->exit_int_info     = vmcb->control.event_inj;
		nc->exit_int_info_err = vmcb->control.event_inj_err;
	}

	nested_vmcb->control.tlb_ctl           = 0;
	nested_vmcb->control.event_inj         = 0;
	nested_vmcb->control.event_inj_err     = 0;

	/* We always set V_INTR_MASKING and remember the old value in hflags */
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
		nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;

	/* Restore the original control entries */
	copy_vmcb_control_area(vmcb, hsave);

	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);

	svm->nested.nested_cr3 = 0;

	/* Restore selected save entries */
	svm->vmcb->save.es = hsave->save.es;
	svm->vmcb->save.cs = hsave->save.cs;
	svm->vmcb->save.ss = hsave->save.ss;
	svm->vmcb->save.ds = hsave->save.ds;
	svm->vmcb->save.gdtr = hsave->save.gdtr;
	svm->vmcb->save.idtr = hsave->save.idtr;
	kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
	svm_set_efer(&svm->vcpu, hsave->save.efer);
	svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
	svm_set_cr4(&svm->vcpu, hsave->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = hsave->save.cr3;
		svm->vcpu.arch.cr3 = hsave->save.cr3;
	} else {
		(void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
	}
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
	svm->vmcb->save.dr7 = 0;
	svm->vmcb->save.cpl = 0;
	svm->vmcb->control.exit_int_info = 0;

	mark_all_dirty(svm->vmcb);

	nested_svm_unmap(page);

	nested_svm_uninit_mmu_context(&svm->vcpu);
	kvm_mmu_reset_context(&svm->vcpu);
	kvm_mmu_load(&svm->vcpu);

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Alexander Graf 454 49.73% 4 10.00%
Joerg Roedel 419 45.89% 29 72.50%
Avi Kivity 18 1.97% 3 7.50%
Jan Kiszka 11 1.20% 1 2.50%
Gleb Natapov 9 0.99% 2 5.00%
Stefan Hajnoczi 2 0.22% 1 2.50%
Total 913 100.00% 40 100.00%



static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
{
	/*
         * This function merges the msr permission bitmaps of kvm and the
         * nested vmcb. It is optimized in that it only merges the parts where
         * the kvm msr permission bitmap may contain zero bits
         */
	int i;

	if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
		return true;

	for (i = 0; i < MSRPM_OFFSETS; i++) {
		u32 value, p;
		u64 offset;

		if (msrpm_offsets[i] == 0xffffffff)
			break;

		p      = msrpm_offsets[i];
		offset = svm->nested.vmcb_msrpm + (p * 4);

		if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
			return false;

		svm->nested.msrpm[p] = svm->msrpm[p] | value;
	}

	svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 101 68.71% 6 60.00%
Alexander Graf 43 29.25% 2 20.00%
Paolo Bonzini 2 1.36% 1 10.00%
Guo Chao 1 0.68% 1 10.00%
Total 147 100.00% 10 100.00%



static bool nested_vmcb_checks(struct vmcb *vmcb)
{
	if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
		return false;

	if (vmcb->control.asid == 0)
		return false;

	if (vmcb->control.nested_ctl && !npt_enabled)
		return false;

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Joerg Roedel 62 100.00% 4 100.00%
Total 62 100.00% 4 100.00%



static bool nested_svm_vmrun(struct vcpu_svm *svm)
{
	struct vmcb *nested_vmcb;
	struct vmcb *hsave = svm->nested.hsave;
	struct vmcb *vmcb = svm->vmcb;
	struct page *page;
	u64 vmcb_gpa;

	vmcb_gpa = svm->vmcb->save.rax;

	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
	if (!nested_vmcb)
		return false;

	if (!nested_vmcb_checks(nested_vmcb)) {
		nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
		nested_vmcb->control.exit_code_hi = 0;
		nested_vmcb->control.exit_info_1  = 0;
		nested_vmcb->control.exit_info_2  = 0;

		nested_svm_unmap(page);

		return false;
	}

	trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
			       nested_vmcb->save.rip,
			       nested_vmcb->control.int_ctl,
			       nested_vmcb->control.event_inj,
			       nested_vmcb->control.nested_ctl);

	trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
				    nested_vmcb->control.intercept_cr >> 16,
				    nested_vmcb->control.intercept_exceptions,
				    nested_vmcb->control.intercept);

	/* Clear internal status */
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);

	/*
         * Save the old vmcb, so we don't need to pick what we save, but can
         * restore everything when a VMEXIT occurs
         */
	hsave->save.es     = vmcb->save.es;
	hsave->save.cs     = vmcb->save.cs;
	hsave->save.ss     = vmcb->save.ss;
	hsave->save.ds     = vmcb->save.ds;
	hsave->save.gdtr   = vmcb->save.gdtr;
	hsave->save.idtr   = vmcb->save.idtr;
	hsave->save.efer   = svm->vcpu.arch.efer;
	hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
	hsave->save.cr4    = svm->vcpu.arch.cr4;
	hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
	hsave->save.rip    = kvm_rip_read(&svm->vcpu);
	hsave->save.rsp    = vmcb->save.rsp;
	hsave->save.rax    = vmcb->save.rax;
	if (npt_enabled)
		hsave->save.cr3    = vmcb->save.cr3;
	else
		hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);

	copy_vmcb_control_area(hsave, vmcb);

	if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
		svm->vcpu.arch.hflags |= HF_HIF_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_HIF_MASK;

	if (nested_vmcb->control.nested_ctl) {
		kvm_mmu_unload(&svm->vcpu);
		svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
		nested_svm_init_mmu_context(&svm->vcpu);
	}

	/* Load the nested guest state */
	svm->vmcb->save.es = nested_vmcb->save.es;
	svm->vmcb->save.cs = nested_vmcb->save.cs;
	svm->vmcb->save.ss = nested_vmcb->save.ss;
	svm->vmcb->save.ds = nested_vmcb->save.ds;
	svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
	svm->vmcb->save.idtr = nested_vmcb->save.idtr;
	kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
	svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
	svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
	svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
		svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
	} else
		(void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
Person	Tokens	Prop	Commits	CommitProp
Joerg Roedel	20	100.00%	1	100.00%
Total	20	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.13 arch/x86/kvm/svm.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors
