#include /* Needed for the macros */ #include /* Needed by all modules */ #include /* Needed for pr_info() */ #include /* for memset() and memcpy() */ #include #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0) #include #else #include #endif #include #include #include #include #include #include #include #define MSR_PM_BASE1 0x00000000 #define MSR_PM_BASE2 0xc0000000 #define MSR_PM_BASE3 0xc0010000 #define SVM_MSR_VM_HSAVE_PA 0xc0010117 #define UNLOAD_HV_MAGIC 0x40067420 #define SVM_SUPPORT_LEAF 0x80000001 #define SVM_SUPPORT_BIT 2 #define SVM_MSR_VM_CR 0xc0010114 #define VM_CR_SVMDIS_BIT 4 #define VMEXIT_CPUID 0x72 #define VMEXIT_MSRPROT 0x7C /* VECTOR 8 bits = 0x0D (GP) */ /* TYPE 3 bits = 0x3 (Fault) */ /* EV 1 bit = 0x1 (Error Code Valid) */ /* reserved 19 bits */ /* V 1 bit = 0x1 (Valid) */ /* ERRORCODE 32 bits = 0x0 */ #define GP0_EVENTINJ (0x0D | (0x3 << 8) | (0x1 << 11) | (0x1 << 31)) #ifndef page_to_phys #define page_to_phys(page) PFN_PHYS(page_to_pfn(page)) #endif /* The hypervisor doesn't handle nested virtualization, so attempting to * virtualize a kernel currently using virtualization features will probably * result in an immediate crash. If you know better you can tell the module * to attempt the virtualization anyway */ static int ignore_svm_enabled = 0; module_param(ignore_svm_enabled, int, 0); static struct { struct page* msrpm_page; u64 msrpm_pa; bool prev_cpuid_fault_cap; bool prev_svm_enabled; } shared_data; inline static void vmsave(u64 vmcb_pa) { asm volatile ("vmsave" : : "a" (vmcb_pa) : "memory"); } inline static void vmload(u64 vmcb_pa) { asm volatile ("vmload" : : "a" (vmcb_pa) : "memory"); } inline static void do_cpuid(u32* eax, u32* ebx, u32* ecx, u32* edx) { asm volatile ("cpuid" : "+a"(*eax), "=b"(*ebx), "+c"(*ecx), "=d"(*edx) : : "memory"); } inline static void do_cpuid_leaf(u32 leaf, u32* eax, u32* ebx, u32* ecx, u32* edx) { asm volatile ("cpuid" : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) : "a"(leaf) : "memory"); } inline static void do_wrmsr(u32 msr, u64 val) { asm volatile ("wrmsr" : : "c"(msr), "a"((u32)val), "d" ((u32)(val >> 32)) : "memory"); } inline static u64 do_rdmsr(u32 msr) { u32 low, high; asm volatile ("rdmsr" : "=a"(low), "=d" (high) : "c"(msr)); return low | ((u64)high) << 32; } /* Raw writes and reads for cr registers. The kernel has it's own safer * versions, however, all the writes we do should use values provided by the * kernel, so these consistent raw versions are cleaner to use. */ #define define_cr_ops(cr) \ inline static u64 do_read_ ## cr (void) {\ u64 val; \ asm volatile ("movq %%" #cr ", %0" : "=r" (val) : :); \ return val; \ } \ inline static void do_write_ ## cr (u64 val) {\ asm volatile ("movq %0, %%" #cr : :"r" (val) :); \ } define_cr_ops(cr0); define_cr_ops(cr2); define_cr_ops(cr3); define_cr_ops(cr4); /* Checks the relevant CPUID bits and MSR registers to determine SVM support * since the hypervisor doesn't use features like NPT or decode assists, * very few checks are needed */ static bool check_svm_support(void) { u32 eax, ebx, ecx, edx; /* check for "AuthenticAMD" on leaf 0 */ do_cpuid_leaf(0, &eax, &ebx, &ecx, &edx); if (!(ebx == 'htuA' && edx == 'itne' && ecx == 'DMAc')) return false; /* check for SVM support */ do_cpuid_leaf(SVM_SUPPORT_LEAF, &eax, &ebx, &ecx, &edx); if (!(ecx & (1U << SVM_SUPPORT_BIT))) return false; /* check if SVM is disabled by the BIOS */ if (do_rdmsr(SVM_MSR_VM_CR) & BIT_ULL(VM_CR_SVMDIS_BIT)) return false; return true; } /* capture_context saves all GPR's, as well as RIP and RFLAGS as they are at * it's return, except for RAX, which is saved as 0, but returned as 1 */ noinline_for_stack u64 capture_context(struct InitialContext* registers); /* run_vm repeatedly runs the VMRUN instruction, effectively continuing to * run the guest (the OS), when an intercepted event happens, it calls the * handle_vm_exit function with the current state of the guest. If the call * returns a non-zero value it begins the process of devirtualizing the * processor */ __attribute__((noreturn)) void run_vm(struct ProcessorStatus* status); /* called when the guest would execute a CPUID instruction*/ static u64 handle_cpuid(struct ProcessorStatus* status) { u32 eax = status->guest_registers.rax; u32 ebx; u32 ecx = status->guest_registers.rcx; u32 edx; if(status->vmcb.state_area.cpl == 0) { switch (eax) { case UNLOAD_HV_MAGIC: status->vmcb.state_area.rip += 2; return 1; /* tell the hypervisor to devirtualize the processor */ } } else { if(status->cpuid_fault) { /* if CPUID should fault */ status->vmcb.control_area.eventinj = GP0_EVENTINJ; /* inject GP */ return 0; } } /* otherwise, run CPUID and increase RIP */ do_cpuid(&eax, &ebx, &ecx, &edx); status->guest_registers.rax = eax; status->guest_registers.rbx = ebx; status->guest_registers.rcx = ecx; status->guest_registers.rdx = edx; status->vmcb.state_area.rip += 2; return 0; } /* called when the guest would execute a read or write to a protected MSR */ static u64 handle_msr(struct ProcessorStatus* status) { /* the msr the guest is trying to access */ u32 msr = status->guest_registers.rcx; bool write_access = (status->vmcb.control_area.exitinfo1 != 0); /* to emulate cpuid_fault, writes and reads to the corresponding MSR are * intercepted */ if (write_access) { u64 value = (status->guest_registers.rdx << 32) | (u32)status->guest_registers.rax; #ifdef MSR_K7_HWCR_CPUID_USER_DIS_BIT /* If the MSR to enable / disable cpuid_fault is being written to */ if(msr == MSR_K7_HWCR) { /* update whether the hypervisor should inject a fault on CPUID */ status->cpuid_fault = value & BIT_ULL(MSR_K7_HWCR_CPUID_USER_DIS_BIT); /* write the value to the register, with the cpuid_fault bit clear */ do_wrmsr(msr, value & ~BIT_ULL(MSR_K7_HWCR_CPUID_USER_DIS_BIT)); } #else /* Older kernels will attempt to enable cpuid_fault using the intel * MSR */ if(msr == MSR_MISC_FEATURES_ENABLES) { status->cpuid_fault = value & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT; } #endif else { /* If we don't care about the MSR, do the write directly */ do_wrmsr(msr, value); } } else { u64 value; #ifdef MSR_K7_HWCR_CPUID_USER_DIS_BIT /* If the MSR to enable / disable cpuid_fault is being read */ if(msr == MSR_K7_HWCR) { value = do_rdmsr(msr); /* reflect the current cpuid_fault status to the guest */ if(status->cpuid_fault) value |= BIT_ULL(MSR_K7_HWCR_CPUID_USER_DIS_BIT); } #else /* Older kernels will attempt to enable cpuid_faulting using the intel * MSR */ if(msr == MSR_MISC_FEATURES_ENABLES) { value = 0; if(status->cpuid_fault) value |= MSR_MISC_FEATURES_ENABLES_CPUID_FAULT; } #endif else { /* If we don't care about the MSR, do the read directly */ value = do_rdmsr(msr); } status->guest_registers.rax = (u32)value; status->guest_registers.rdx = (u32)(value >> 32); } status->vmcb.state_area.rip += 2; return 0; } /* consistent functions to write segment selectors */ #define define_sel_ops(sel) \ inline static u64 read_ ## sel ## _sel(void) {\ u16 sel; \ asm volatile ("movw %%" #sel ", %0" : "=r" (sel) : :); \ return sel; \ } \ inline static void write_ ## sel ## _sel(u16 sel) {\ asm volatile ("movw %0, %%" #sel : :"r" (sel) :); \ } define_sel_ops(ds); define_sel_ops(es); define_sel_ops(ss); inline static u64 read_cs_sel(void) { u16 cs; asm volatile("movw %%cs, %0" : "=r"(cs) : :); return cs; } /* a direct move into cs is not a valid instruction, so the value is written * via a far jump */ inline static void write_cs_sel(u16 cs) { asm volatile("movzwq %0,%%rax\n\t" "pushq %%rax\n\t" "leaq curr_ip(%%rip),%%rax\n\t" "pushq %%rax\n\t" "lretq\n\t" "curr_ip:": : "r"(cs) : "rax"); }; /* To end the virtualization the host needs to put itself the guest was just * in, this effectively does the opposite to setup_guest_state */ static void restore_host_state(struct ProcessorStatus* status) { struct VMCB* guest_vmcb = &status->vmcb; struct desc_ptr gdt_ptr; struct desc_ptr idt_ptr; /* Load other state that might have changed */ vmload(status->vmcb_pa); /* Restore the hsave address before the processor was virtualized */ do_wrmsr(SVM_MSR_VM_HSAVE_PA, status->prev_hsave_pa); do_wrmsr(MSR_EFER, guest_vmcb->state_area.efer); idt_ptr.address = guest_vmcb->state_area.idtr.base; idt_ptr.size = guest_vmcb->state_area.idtr.limit; gdt_ptr.address = guest_vmcb->state_area.gdtr.base; gdt_ptr.size = guest_vmcb->state_area.gdtr.limit; /* Restore gdt and idt. Done for correctness */ native_load_gdt(&gdt_ptr); native_load_idt(&idt_ptr); /* Load segment selectors from the guest. Also for correctness */ write_es_sel(guest_vmcb->state_area.es.selector); write_ss_sel(guest_vmcb->state_area.ss.selector); write_ds_sel(guest_vmcb->state_area.ds.selector); write_cs_sel(guest_vmcb->state_area.cs.selector); /* run_vm will resume running the OS by setting this rip and rsp */ status->guest_registers.rax = guest_vmcb->state_area.rip; status->guest_registers.rsp = guest_vmcb->state_area.rsp; /* will be used by the OS to cleanup the per processor data */ status->guest_registers.rbx = (u64)status; } /* Everytime an intercepted instruction is excecuted (currently only * wrmsr/rdmsr to the cpuid_fault register, cpuid and runvm (required by * the processor)), the guest's state is saved and the hypervisor calls * this function */ u64 handle_vm_exit(struct ProcessorStatus* status) { status->guest_registers.rsp = status->vmcb.state_area.rsp; status->guest_registers.rax = status->vmcb.state_area.rax; u64 exitcode = status->vmcb.control_area.exitcode; u64 result = 0; switch(exitcode) { case VMEXIT_CPUID: result = handle_cpuid(status); break; case VMEXIT_MSRPROT: result = handle_msr(status); break; } if(result) { /* begin the devirtualization process */ restore_host_state(status); } status->vmcb.state_area.rsp = status->guest_registers.rsp; status->vmcb.state_area.rax = status->guest_registers.rax; return result; /* tell run_vm if it should keep running the guest */ } /* Used to initialize the segment registers (cs, ds, es, ss) in the VMCB */ static void set_descriptor_from_gdt(struct desc_ptr* gdtr, u16 selector, struct SegmentDescriptor* desc) { desc->selector = selector; u64 idx = selector & ~0b111; /* Mask out the TI and RPL bits */ if(!idx) { desc->attrib = 0; desc->limit = 0; desc->base = 0; return; } /* Read the 8 bytes of the descriptor from the gdt, there's also 16-byte * descriptors (e.g. TSS, LDT), but the function is not used for them, so * they're not handled */ u64 raw = *(u64*) (gdtr->address + idx); /* Concatenate the attribute bits */ desc->attrib = ((raw & (0xFFL << 40)) >> 40) | ((raw & (0xFL << 52)) >> 44); /* Concatenate the limit bits */ desc->limit = (raw & 0xFFFF) | ((raw & (0xFL << 48)) >> 32); /* If the granularity is 4kb, store the effective limit */ if(raw & (0x1L << 55)) desc->limit = (desc->limit << 12) | 0xFFFL; /* Concatenate the base address bits */ desc->base = ((raw & (0xFFFFL << 16)) >> 16) | ((raw & (0xFFL << 32)) >> 16) | ((raw & (0xFFL << 56)) >> 32); } /* This setups all of the state needed to run a vm with the current status * of the processor */ static void setup_guest_state(struct page* host_status_page, struct InitialContext* initial_context) { u64 host_status_pa = page_to_phys(host_status_page); u64 guest_vmcb_pa = host_status_pa + HOST_VMCB_OFFSET; u64 host_vmcb_pa = host_status_pa + HOST_HOST_VMCB_OFFSET; u64 host_save_pa = host_status_pa + HOST_HOST_SAVE_OFFSET; struct ProcessorStatus* host_status = page_address(host_status_page); host_status->vmcb_pa = guest_vmcb_pa; host_status->host_vmcb_pa = host_vmcb_pa; /* used to cleanup after the processor is devirtualized */ host_status->page_ptr = host_status_page; host_status->guest_registers = initial_context->regs; struct VMCB* guest_vmcb = &host_status->vmcb; struct desc_ptr gdt_ptr; struct desc_ptr idt_ptr; native_store_gdt(&gdt_ptr); store_idt(&idt_ptr); /* intercept MSRs defined by MSR prot */ guest_vmcb->control_area.intercept_vector3.msr_prot = 1; guest_vmcb->control_area.msrpm_base_pa = shared_data.msrpm_pa; /* intercept the CPUID instruction */ guest_vmcb->control_area.intercept_vector3.cpuid_inst = 1; /* intercept the VMRUN instruction (required to run VMRUN) */ guest_vmcb->control_area.intercept_vector4.vmrun_inst = 1; /* the guest uses it's own address space, 1 is the safest value to use */ guest_vmcb->control_area.guest_asid = 1; u32 eax, ebx, ecx, edx; do_cpuid_leaf(0x80000008, &eax, &ebx, &ecx, &edx); if(ebx & (1 << 3)) { do_cpuid_leaf(0x8000000a, &eax, &ebx, &ecx, &edx); if(edx & (1 << 24)) { guest_vmcb->control_area.virtualization_control0.invlpg_tlbsync_enable = 1; } } guest_vmcb->state_area.gdtr.base = gdt_ptr.address; guest_vmcb->state_area.gdtr.limit = gdt_ptr.size; guest_vmcb->state_area.idtr.base = idt_ptr.address; guest_vmcb->state_area.idtr.limit = idt_ptr.size; u16 cs_selector = read_cs_sel(); set_descriptor_from_gdt(&gdt_ptr, cs_selector, &guest_vmcb->state_area.cs); u16 ds_selector = read_ds_sel(); set_descriptor_from_gdt(&gdt_ptr, ds_selector, &guest_vmcb->state_area.ds); u16 es_selector = read_es_sel(); set_descriptor_from_gdt(&gdt_ptr, es_selector, &guest_vmcb->state_area.es); u16 ss_selector = read_ss_sel(); set_descriptor_from_gdt(&gdt_ptr, ss_selector, &guest_vmcb->state_area.ss); guest_vmcb->state_area.efer = do_rdmsr(MSR_EFER); guest_vmcb->state_area.cr0 = do_read_cr0(); guest_vmcb->state_area.cr2 = do_read_cr2(); guest_vmcb->state_area.cr3 = do_read_cr3(); guest_vmcb->state_area.cr4 = do_read_cr4(); guest_vmcb->state_area.rflags = initial_context->rflags; guest_vmcb->state_area.rsp = initial_context->regs.rsp; guest_vmcb->state_area.rip = initial_context->rip; guest_vmcb->state_area.rax = initial_context->regs.rax; vmsave(guest_vmcb_pa); /* Save the hsave address before the hypervisor is loaded */ /* (If the kvm_amd kernel module is loaded, this value will already be * setup and changing it and trying to run a kvm vm will crash the OS) */ host_status->prev_hsave_pa = do_rdmsr(SVM_MSR_VM_HSAVE_PA); do_wrmsr(SVM_MSR_VM_HSAVE_PA, host_save_pa); /* save another copy of the host state, used to provide a more consistent * host state at vmexit */ vmsave(host_vmcb_pa); } static int virtualize_processor(void* processor_number) { struct page* host_status_page; int order = get_order(sizeof(struct ProcessorStatus)); host_status_page = alloc_pages(GFP_KERNEL, order); if (!host_status_page) goto host_status_page_failed; unsigned long flags; get_cpu(); local_irq_save(flags); memset(page_address(host_status_page), 0, PAGE_SIZE * (1 << order)); struct InitialContext context; /* Capture the context to be used to initialize the guest, since it * returns 1 but the stored state has 0 as it's return, when the guest * starts execution at the stored state it will not enter into the if. * */ if(capture_context(&context)) { do_wrmsr(MSR_EFER, do_rdmsr(MSR_EFER) | EFER_SVME); /* Enable SVME */ setup_guest_state(host_status_page, &context); struct ProcessorStatus* host_status = page_address(host_status_page); /* Kinda ugly, effectively sets a CR3 for the host that will * always be safe to access the hypervisor's code from * */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0) leave_mm(); #else leave_mm(smp_processor_id()); #endif run_vm(host_status); BUG(); } /* Only the guest should run this code */ put_cpu(); pr_info("Processor %d virtualized\n", (u32)(u64)processor_number); return 0; host_status_page_failed: local_irq_restore(flags); put_cpu(); return -ENOMEM; } static int check_svm_on_cpu(void*) { return do_rdmsr(MSR_EFER) & EFER_SVME; } static bool check_svm_status(void) { u32 svm_enabled = 0; cpus_read_lock(); u32 cpu; for_each_online_cpu(cpu) { svm_enabled |= smp_call_on_cpu(cpu, check_svm_on_cpu, NULL, true); } cpus_read_unlock(); return svm_enabled; } static int devirtualize_processor(void* processor_number) { int order = get_order(sizeof(struct ProcessorStatus)); unsigned long flags; get_cpu(); local_irq_save(flags); u64 rax = UNLOAD_HV_MAGIC; u64 rbx, rcx, rdx; asm volatile("cpuid":"+a"(rax),"=b"(rbx),"=c"(rcx),"=d"(rdx)::"memory"); struct ProcessorStatus* status = NULL; if(rax != UNLOAD_HV_MAGIC) { status = (struct ProcessorStatus*)rbx; struct VMCB* guest_vmcb = &status->vmcb; do_write_cr3(guest_vmcb->state_area.cr3); do_write_cr4(guest_vmcb->state_area.cr4); do_write_cr2(guest_vmcb->state_area.cr2); do_write_cr0(guest_vmcb->state_area.cr0); } if(!shared_data.prev_svm_enabled) do_wrmsr(MSR_EFER, do_rdmsr(MSR_EFER) & ~EFER_SVME); local_irq_restore(flags); put_cpu(); if(status)__free_pages(status->page_ptr, order); pr_info("Processor %d devirtualized\n", (u32)(u64)processor_number); return 0; } /* called when a Power Management (e.g. suspend) event is about to happen */ static int handle_pm_notification(struct notifier_block* nb, unsigned long action, void *data) { if(action == PM_HIBERNATION_PREPARE || action == PM_SUSPEND_PREPARE) { /* on suspend or hibernation svm is disabled, so * all processors need to be devirtualized */ cpus_read_lock(); u32 cpu; for_each_online_cpu(cpu) { smp_call_on_cpu(cpu, devirtualize_processor, (void*)(u64)cpu, true); } cpus_read_unlock(); /* return the cpuid_fault bit capability to it's previous value */ if(!shared_data.prev_cpuid_fault_cap) clear_bit(X86_FEATURE_CPUID_FAULT, (unsigned long *)(boot_cpu_data.x86_capability)); } if(action == PM_POST_HIBERNATION || action == PM_POST_SUSPEND) { /* revirtualize all processors after suspend or hibernation */ cpus_read_lock(); u32 cpu; for_each_online_cpu(cpu) { smp_call_on_cpu(cpu, virtualize_processor, (void*)(u64)cpu, true); } cpus_read_unlock(); shared_data.prev_cpuid_fault_cap = test_bit(X86_FEATURE_CPUID_FAULT, (unsigned long *)(boot_cpu_data.x86_capability)); set_bit(X86_FEATURE_CPUID_FAULT, (unsigned long *)(boot_cpu_data.x86_capability)); } return 0; } static struct notifier_block pm_notifier = { .notifier_call = handle_pm_notification }; static int __init init_emulation(void) { if (!check_svm_support()) { pr_info("Processor not supported\n"); return -ENODEV; } shared_data.prev_svm_enabled = check_svm_status(); if(shared_data.prev_svm_enabled && !ignore_svm_enabled) { pr_info("SVM already enabled\n"); return -ENODEV; } shared_data.msrpm_page = alloc_pages(GFP_KERNEL, 1); if (!shared_data.msrpm_page) goto msrpm_failed; shared_data.msrpm_pa = page_to_phys(shared_data.msrpm_page); memset(page_address(shared_data.msrpm_page), 0, 0x2000); unsigned char* section_1_start = (unsigned char*) page_address(shared_data.msrpm_page); unsigned char* section_3_start = (unsigned char*) page_address(shared_data.msrpm_page) + 0x1000; #ifdef MSR_K7_HWCR_CPUID_USER_DIS_BIT set_bit((MSR_K7_HWCR - MSR_PM_BASE3) * 2, (unsigned long*) section_3_start); set_bit((MSR_K7_HWCR - MSR_PM_BASE3) * 2 + 1, (unsigned long*) section_3_start); #else /* on older kernels intercept the intel register instead */ set_bit((MSR_MISC_FEATURES_ENABLES - MSR_PM_BASE1) * 2, (unsigned long*) section_1_start); set_bit((MSR_MISC_FEATURES_ENABLES - MSR_PM_BASE1) * 2 + 1, (unsigned long*) section_1_start); #endif cpus_read_lock(); u32 cpu; for_each_online_cpu(cpu) { smp_call_on_cpu(cpu, virtualize_processor, (void*)(u64)cpu, true); } cpus_read_unlock(); shared_data.prev_cpuid_fault_cap = test_bit(X86_FEATURE_CPUID_FAULT, (unsigned long *)(boot_cpu_data.x86_capability)); set_bit(X86_FEATURE_CPUID_FAULT, (unsigned long *)(boot_cpu_data.x86_capability)); register_pm_notifier(&pm_notifier); return 0; msrpm_failed: return -ENOMEM; } static void __exit deinit_emulation(void) { unregister_pm_notifier(&pm_notifier); if(!shared_data.prev_cpuid_fault_cap) clear_bit(X86_FEATURE_CPUID_FAULT, (unsigned long *)(boot_cpu_data.x86_capability)); cpus_read_lock(); u32 cpu; for_each_online_cpu(cpu) { smp_call_on_cpu(cpu, devirtualize_processor, (void*)(u64)cpu, true); } cpus_read_unlock(); if (shared_data.msrpm_page) __free_pages(shared_data.msrpm_page, 1); } module_init(init_emulation); module_exit(deinit_emulation); MODULE_LICENSE("GPL");