kvm_host.h

/*
 * This work is licensed under the terms of the GNU GPL, version 2. See the
 * COPYING file in the top-level directory.
 *
 * Copyright 2011 various Linux Kernel contributors.
 * Copyright 2018 Joyent, Inc.
 */

#ifndef __KVM_HOST_H
#define	__KVM_HOST_H

#include <sys/types.h>
#include <sys/list.h>
#include <sys/mutex.h>
#include <sys/sunddi.h>

#include "kvm_types.h"
#include "kvm_impl.h"
#include "kvm_x86host.h"

#define	NSEC_PER_MSEC 1000000L
#define	NSEC_PER_SEC 1000000000L

#define	BITS_PER_LONG	(sizeof (unsigned long) * 8)

/*
 * vcpu->requests bit members
 */
#define	KVM_REQ_TLB_FLUSH		0
#define	KVM_REQ_REPORT_TPR_ACCESS	2
#define	KVM_REQ_MMU_RELOAD		3
#define	KVM_REQ_TRIPLE_FAULT		4
#define	KVM_REQ_PENDING_TIMER		5
#define	KVM_REQ_UNHALT			6
#define	KVM_REQ_MMU_SYNC		7
#define	KVM_REQ_KVMCLOCK_UPDATE		8
#define	KVM_REQ_KICK			9
#define	KVM_REQ_DEACTIVATE_FPU		10

#define	KVM_USERSPACE_IRQ_SOURCE_ID	0

struct kvm;
struct kvm_vcpu;

typedef struct kvm_user_return_notifier {
	void (*on_user_return)(struct kvm_vcpu *,
	    struct kvm_user_return_notifier *);
} kvm_user_return_notifier_t;

extern void kvm_user_return_notifier_register(struct kvm_vcpu *,
    struct kvm_user_return_notifier *);
extern void kvm_user_return_notifier_unregister(struct kvm_vcpu *,
    struct kvm_user_return_notifier *);
extern void kvm_fire_urn(struct kvm_vcpu *);

#define	KVM_NR_SHARED_MSRS 16

typedef struct kvm_shared_msrs_global {
	int nr;
	uint32_t msrs[KVM_NR_SHARED_MSRS];
} kvm_shared_msrs_global_t;

typedef struct kvm_shared_msrs {
	struct kvm_user_return_notifier urn;
	int registered;
	uint_t host_saved;
	struct kvm_shared_msr_values {
		uint64_t host;
		uint64_t curr;
	} values[KVM_NR_SHARED_MSRS];
} kvm_shared_msrs_t;

/*
 * It would be nice to use something smarter than a linear search, TBD...
 * Thankfully we dont expect many devices to register (famous last words :),
 * so until then it will suffice.  At least its abstracted so we can change
 * in one place.
 */
typedef struct kvm_io_bus {
	int			dev_count;
#define	NR_IOBUS_DEVS 200
	struct kvm_io_device	*devs[NR_IOBUS_DEVS];
} kvm_io_bus_t;

enum kvm_bus {
	KVM_MMIO_BUS,
	KVM_PIO_BUS,
	KVM_NR_BUSES
};

extern int kvm_io_bus_write(struct kvm *, enum kvm_bus, gpa_t,
    int, const void *);
extern int kvm_io_bus_read(struct kvm *, enum kvm_bus, gpa_t, int,
    void *);
extern int kvm_io_bus_register_dev(struct kvm *, enum kvm_bus,
    struct kvm_io_device *);
extern int kvm_io_bus_unregister_dev(struct kvm *, enum kvm_bus,
    struct kvm_io_device *);

#define	KVM_MAX_IRQ_ROUTES 1024

#define	KVM_RINGBUF_NENTRIES	512

#define	KVM_RINGBUF_TAG_CTXSAVE		1
#define	KVM_RINGBUF_TAG_CTXRESTORE	2
#define	KVM_RINGBUF_TAG_VMPTRLD		3
#define	KVM_RINGBUF_TAG_VCPUMIGRATE	4
#define	KVM_RINGBUF_TAG_VCPUCLEAR	5
#define	KVM_RINGBUF_TAG_VCPULOAD	6
#define	KVM_RINGBUF_TAG_VCPUPUT		7
#define	KVM_RINGBUF_TAG_RELOAD		8
#define	KVM_RINGBUF_TAG_EMUFAIL0	9
#define	KVM_RINGBUF_TAG_EMUFAIL1	10
#define	KVM_RINGBUF_TAG_EMUFAIL2	11
#define	KVM_RINGBUF_TAG_EMUXADD		12
#define	KVM_RINGBUF_TAG_MAX		12

typedef struct kvm_ringbuf_entry {
	uint32_t kvmre_tag;			/* tag for this entry */
	uint32_t kvmre_cpuid;			/* CPU of entry */
	uint64_t kvmre_thread;			/* thread for entry */
	uint64_t kvmre_tsc;			/* TSC at time of entry */
	uint64_t kvmre_payload;			/* payload for this entry */
} kvm_ringbuf_entry_t;

typedef struct kvm_ringbuf {
	kvm_ringbuf_entry_t kvmr_buf[KVM_RINGBUF_NENTRIES]; /* ring buffer */
	kvm_ringbuf_entry_t kvmr_taglast[KVM_RINGBUF_TAG_MAX + 1];
	uint32_t kvmr_tagcount[KVM_RINGBUF_TAG_MAX + 1]; /* count of tags */
	uint32_t kvmr_ent;			/* current entry */
} kvm_ringbuf_t;

extern void kvm_ringbuf_record(kvm_ringbuf_t *, uint32_t, uint64_t);

typedef struct kvm_vcpu {
	struct kvm *kvm;
	int vcpu_id;
	kmutex_t mutex;
	int   cpu;
	struct kvm_run *run;
	unsigned long requests;
	unsigned long guest_debug;
	int srcu_idx;

	int fpu_active;
	int guest_fpu_loaded;

	kmutex_t kvcpu_kick_lock;
	kcondvar_t kvcpu_kick_cv;
	kvm_vcpu_stats_t kvcpu_stats;
	kstat_t *kvcpu_kstat;
	kvm_ringbuf_t kvcpu_ringbuf;
	int sigset_active;
	sigset_t sigset;
	int mmio_needed;
	int mmio_read_completed;
	int mmio_is_write;
	int mmio_size;
	unsigned char mmio_data[8];
	gpa_t mmio_phys_addr;
	struct kvm_vcpu_arch arch;
	ddi_umem_cookie_t cookie;
	struct kvm_user_return_notifier *urn;
	struct ctxop *ctxop;
} kvm_vcpu_t;

typedef struct kvm_memory_slot {
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long flags;
	unsigned long *rmap;
	unsigned long *dirty_bitmap;
	size_t dirty_bitmap_sz;
	struct {
		unsigned long rmap_pde;
		int write_count;
	} *lpage_info[KVM_NR_PAGE_SIZES];
	size_t lpage_info_sz[KVM_NR_PAGE_SIZES];
	unsigned long userspace_addr;
	int user_alloc;
} kvm_memory_slot_t;

unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot);

typedef struct kvm_kernel_irq_routing_entry {
	uint32_t gsi;
	uint32_t type;
	int (*set)(struct kvm_kernel_irq_routing_entry *e,
	    struct kvm *kvm, int irq_source_id, int level);
	union {
		struct {
			unsigned irqchip;
			unsigned pin;
		} irqchip;
		struct msi_msg msi;
	};
	struct list_node link;
} kvm_kernel_irq_routing_entry_t;

typedef struct kvm_irq_routing_table {
	int chip[KVM_NR_IRQCHIPS][KVM_IOAPIC_NUM_PINS];
	struct kvm_kernel_irq_routing_entry *rt_entries;
	uint32_t nr_rt_entries;
	/*
	 * Array indexed by gsi. Each entry contains list of irq chips
	 * the gsi is connected to.
	 */
	list_t map[KVM_MAX_IRQ_ROUTES+1];
} kvm_irq_routing_table_t;

typedef struct kvm_memslots {
	int nmemslots;
	struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS +
					KVM_PRIVATE_MEM_SLOTS];
} kvm_memslots_t;

typedef struct kvm {
	kmutex_t mmu_lock;
	kmutex_t requests_lock;
	kmutex_t slots_lock;
	struct kvm_memslots *memslots;
	kmutex_t memslots_lock; /* linux uses rcu for this */
	/* the following was a read-copy update mechanism */
	/* we'll use a reader-writer lock, for now */
	krwlock_t kvm_rwlock;
	uint32_t bsp_vcpu_id;
	struct kvm_vcpu *bsp_vcpu;
	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
	volatile int online_vcpus;
	struct list_node vm_list;
	kmutex_t lock;
	struct kvm_io_bus *buses[KVM_NR_BUSES];
	kmutex_t buses_lock;
	struct kstat *kvm_kstat;
	kvm_stats_t kvm_stats;
	struct kvm_arch arch;
	volatile int users_count;
	struct kvm_coalesced_mmio_dev *coalesced_mmio_dev;
	struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
	ddi_umem_cookie_t mmio_cookie;

	kmutex_t irq_lock;
	struct kvm_irq_routing_table *irq_routing;
	int irq_routing_sz;
	list_t mask_notifier_list;
	list_t irq_ack_notifier_list;

	int kvmid;  /* unique identifier for this kvm */
	int kvm_clones;
	pid_t kvm_pid;			/* pid associated with this kvm */
	kmutex_t kvm_avllock;
	avl_tree_t kvm_avlmp;		/* avl tree for mmu to page_t mapping */
} kvm_t;


extern struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i);

#define	kvm_for_each_vcpu(idx, vcpup, kvm) \
	for (idx = 0, vcpup = kvm_get_vcpu(kvm, idx); \
	    idx < kvm->online_vcpus && vcpup; /* XXX - need protection */ \
	    vcpup = kvm_get_vcpu(kvm, ++idx))

extern int kvm_vcpu_init(struct kvm_vcpu *, struct kvm *, unsigned);
extern void kvm_vcpu_uninit(struct kvm_vcpu *);

extern void vcpu_load(struct kvm_vcpu *);
extern void vcpu_put(struct kvm_vcpu *);

extern int kvm_init(void *);
extern void kvm_exit(void);

extern void kvm_get_kvm(struct kvm *);
extern void kvm_put_kvm(struct kvm *);

#define	HPA_MSB ((sizeof (hpa_t) * 8) - 1)
#define	HPA_ERR_MASK ((hpa_t)1 << HPA_MSB)
static int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; }

extern page_t *bad_page;
extern void *bad_page_kma;
extern pfn_t bad_pfn;

extern int is_error_page(struct page *);
extern int is_error_pfn(pfn_t);
extern int kvm_is_error_hva(unsigned long);

extern int kvm_set_memory_region(struct kvm *,
    struct kvm_userspace_memory_region *, int);
extern int __kvm_set_memory_region(struct kvm *,
    struct kvm_userspace_memory_region *, int);
extern int kvm_arch_prepare_memory_region(struct kvm *,
    struct kvm_memory_slot *, struct kvm_memory_slot,
    struct kvm_userspace_memory_region *, int);
extern void kvm_arch_commit_memory_region(struct kvm *,
    struct kvm_userspace_memory_region *,
    struct kvm_memory_slot, int);

extern void kvm_disable_largepages(void);
extern void kvm_arch_flush_shadow(struct kvm *);
extern gfn_t unalias_gfn(struct kvm *, gfn_t);
extern gfn_t unalias_gfn_instantiation(struct kvm *, gfn_t);

extern page_t *gfn_to_page(struct kvm *, gfn_t);
extern unsigned long gfn_to_hva(struct kvm *, gfn_t);
extern void kvm_release_page_clean(struct page *);
extern void kvm_release_page_dirty(struct page *);
extern void kvm_set_page_dirty(struct page *);
extern void kvm_set_page_accessed(struct page *);

extern pfn_t gfn_to_pfn(struct kvm *, gfn_t);
extern pfn_t gfn_to_pfn_memslot(struct kvm *,
    struct kvm_memory_slot *, gfn_t);
extern int memslot_id(struct kvm *, gfn_t);
extern void kvm_release_pfn_dirty(pfn_t);
extern void kvm_release_pfn_clean(pfn_t);
extern void kvm_set_pfn_dirty(pfn_t);
extern void kvm_set_pfn_accessed(struct kvm *, pfn_t);
extern void kvm_get_pfn(struct kvm_vcpu *, pfn_t);

extern int kvm_read_guest_page(struct kvm *, gfn_t, void *, int, int);
extern int kvm_read_guest_atomic(struct kvm *, gpa_t, void *, unsigned long);
extern int kvm_read_guest(struct kvm *, gpa_t, void *, unsigned long);
extern int kvm_read_guest_virt_helper(gva_t, void *, unsigned int,
    struct kvm_vcpu *, uint32_t, uint32_t *);
extern int kvm_write_guest_page(struct kvm *, gfn_t, const void *, int, int);
extern int kvm_write_guest(struct kvm *, gpa_t, const void *, unsigned long);
extern int kvm_clear_guest_page(struct kvm *, gfn_t, int, int);
extern int kvm_clear_guest(struct kvm *, gpa_t, unsigned long);
extern struct kvm_memory_slot *gfn_to_memslot(struct kvm *, gfn_t);
extern int kvm_is_visible_gfn(struct kvm *, gfn_t);
extern unsigned long kvm_host_page_size(struct kvm *, gfn_t);
extern void mark_page_dirty(struct kvm *, gfn_t);

extern void kvm_vcpu_block(struct kvm_vcpu *);
extern void kvm_vcpu_on_spin(struct kvm_vcpu *);
extern void kvm_resched(struct kvm_vcpu *);
extern void kvm_load_guest_fpu(struct kvm_vcpu *);
extern void kvm_put_guest_fpu(struct kvm_vcpu *);
extern void kvm_flush_remote_tlbs(struct kvm *);
extern void kvm_reload_remote_mmus(struct kvm *);

extern long kvm_arch_dev_ioctl(struct file *, unsigned int, unsigned long);
extern long kvm_arch_vcpu_ioctl(struct file *, unsigned int, unsigned long);
extern int kvm_dev_ioctl_check_extension(long, int *);
extern int kvm_get_dirty_log(struct kvm *, struct kvm_dirty_log *, int *);
extern int kvm_vm_ioctl_get_dirty_log(struct kvm *, struct kvm_dirty_log *);

extern int kvm_vm_ioctl_get_msr_index_list(struct kvm *, uintptr_t);
extern int kvm_vm_ioctl_set_memory_region(struct kvm *,
    struct kvm_userspace_memory_region *, int);
extern int kvm_vm_ioctl_set_tss_addr(struct kvm *, uintptr_t);
extern int kvm_vm_ioctl_get_irqchip(struct kvm *, struct kvm_irqchip *);
extern int kvm_vm_ioctl_set_irqchip(struct kvm *, struct kvm_irqchip *);

extern int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *, struct kvm_fpu *);
extern int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *, struct kvm_fpu *);

extern int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *, struct kvm_regs *);
extern int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *, struct kvm_regs *);
extern int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *, struct kvm_sregs *);
extern int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *, struct kvm_sregs *);
extern int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *,
    struct kvm_mp_state *);
extern int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *,
    struct kvm_mp_state *);
extern int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *,
    struct kvm_guest_debug *);
extern int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *);

extern int kvm_vcpu_ioctl_get_msrs(struct kvm_vcpu *, struct kvm_msrs *, int *);
extern int kvm_vcpu_ioctl_set_msrs(struct kvm_vcpu *, struct kvm_msrs *, int *);
extern int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *, uint64_t *);
extern int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *, struct kvm_cpuid2 *);
extern int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *, struct kvm_cpuid2 *,
    int *, intptr_t);
extern int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *,
    struct kvm_lapic_state *);
extern int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *,
    struct kvm_lapic_state *);
extern int kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *,
    struct kvm_vcpu_events *);
extern int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *,
    struct kvm_vcpu_events *);
extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *, struct kvm_interrupt *);
extern int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *);
extern int kvm_vm_ioctl_get_pit2(struct kvm *, struct kvm_pit_state2 *);
extern int kvm_vm_ioctl_set_pit2(struct kvm *, struct kvm_pit_state2 *);
extern int kvm_vm_ioctl_set_identity_map_addr(struct kvm *, uint64_t);
extern int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *,
    struct kvm_cpuid_entry2 *);

extern int kvm_arch_init(void *);
extern void kvm_arch_exit(void);

extern int kvm_arch_vcpu_init(struct kvm_vcpu *);
extern void kvm_arch_vcpu_uninit(struct kvm_vcpu *);

extern void kvm_arch_vcpu_free(struct kvm_vcpu *);
extern void kvm_arch_vcpu_load(struct kvm_vcpu *, int);
extern void kvm_arch_vcpu_put(struct kvm_vcpu *);
extern struct kvm_vcpu * kvm_arch_vcpu_create(struct kvm *, unsigned int);
extern int kvm_arch_vcpu_setup(struct kvm_vcpu *);
extern void kvm_arch_vcpu_destroy(struct kvm_vcpu *);

extern int kvm_arch_vcpu_reset(struct kvm_vcpu *);
extern int kvm_arch_hardware_setup(void);
extern void kvm_arch_hardware_unsetup(void);
extern void kvm_arch_check_processor_compat(void *);
extern int kvm_arch_vcpu_runnable(struct kvm_vcpu *);

extern void kvm_free_physmem(struct kvm *);

extern struct  kvm *kvm_arch_create_vm(void);
extern void kvm_arch_destroy_vm(struct kvm *);
extern void kvm_arch_destroy_vm_comps(struct kvm *);
extern void kvm_free_all_assigned_devices(struct kvm *);
extern void kvm_arch_sync_events(struct kvm *);

extern int kvm_cpu_has_pending_timer(struct kvm_vcpu *);
extern void kvm_vcpu_kick(struct kvm_vcpu *);

extern int kvm_is_mmio_pfn(pfn_t);

typedef struct kvm_irq_ack_notifier {
	list_t link;
	unsigned gsi;
	void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
} kvm_irq_ack_notifier_t;

#define	KVM_ASSIGNED_MSIX_PENDING		0x1
typedef struct kvm_guest_msix_entry {
	uint32_t vector;
	unsigned short entry;
	unsigned short flags;
} kvm_guest_msix_entry_t;

typedef struct kvm_assigned_dev_kernel {
	struct kvm_irq_ack_notifier ack_notifier;
	list_t interrupt_work;
	list_t list;
	int assigned_dev_id;
	int host_segnr;
	int host_busnr;
	int host_devfn;
	unsigned int entries_nr;
	int host_irq;
	unsigned char host_irq_disabled;
	struct msix_entry *host_msix_entries;
	int guest_irq;
	struct kvm_guest_msix_entry *guest_msix_entries;
	unsigned long irq_requested_type;
	int irq_source_id;
	int flags;
	struct pci_dev *dev;
	struct kvm *kvm;
	kmutex_t assigned_dev_lock;
} kvm_assigned_dev_kernel_t;

typedef struct kvm_irq_mask_notifier {
	void (*func)(struct kvm_irq_mask_notifier *kimn, int masked);
	int irq;
	struct list_node link;
} kvm_irq_mask_notifier_t;

extern void kvm_register_irq_mask_notifier(struct kvm *, int,
    struct kvm_irq_mask_notifier *);
extern void kvm_unregister_irq_mask_notifier(struct kvm *, int,
    struct kvm_irq_mask_notifier *);
extern void kvm_fire_mask_notifiers(struct kvm *, int, int);

extern int kvm_set_irq(struct kvm *, int, uint32_t, int);
extern void kvm_notify_acked_irq(struct kvm *, unsigned, unsigned);
extern void kvm_register_irq_ack_notifier(struct kvm *,
    struct kvm_irq_ack_notifier *);
extern void kvm_unregister_irq_ack_notifier(struct kvm *,
    struct kvm_irq_ack_notifier *);
extern int kvm_request_irq_source_id(struct kvm *);
extern void kvm_free_irq_source_id(struct kvm *, int);

/* For vcpu->arch.iommu_flags */
#define	KVM_IOMMU_CACHE_COHERENCY	0x1

extern void kvm_guest_enter(struct kvm_vcpu *);
extern void kvm_guest_exit(struct kvm_vcpu *);

#ifndef KVM_ARCH_HAS_UNALIAS_INSTANTIATION
#define	unalias_gfn_instantiation unalias_gfn
#endif

extern int kvm_setup_default_irq_routing(struct kvm *);
extern int kvm_set_irq_routing(struct kvm *,
    const struct kvm_irq_routing_entry *,
    unsigned, unsigned);
extern void kvm_free_irq_routing(struct kvm *);

extern int kvm_vcpu_is_bsp(struct kvm_vcpu *);

extern void kvm_sigprocmask(int how, sigset_t *, sigset_t *);

#define	offset_in_page(p)	((unsigned long)(p) & ~PAGEMASK)

#define	page_to_pfn(page) (page->p_pagenum)

/* LDT or TSS descriptor in the GDT. 16 bytes. */
struct ldttss_desc64 {
	unsigned short limit0;
	unsigned short base0;
	unsigned base1 : 8, type : 5, dpl : 2, p : 1;
	unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8;
	uint32_t base3;
	uint32_t zero1;
} __attribute__((packed));

typedef struct ldttss_desc64 ldttss_desc64_t;

#endif /* __KVM_HOST_H */