; $Id: HWACCMR0A.asm 35333 2010-12-27 12:10:56Z vboxsync $ ;; @file ; VMXM - R0 vmx helpers ; ; ; Copyright (C) 2006-2007 Oracle Corporation ; ; This file is part of VirtualBox Open Source Edition (OSE), as ; available from http://www.virtualbox.org. This file is free software; ; you can redistribute it and/or modify it under the terms of the GNU ; General Public License (GPL) as published by the Free Software ; Foundation, in version 2 as it comes in the "COPYING" file of the ; VirtualBox OSE distribution. VirtualBox OSE is distributed in the ; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. ; ;******************************************************************************* ;* Header Files * ;******************************************************************************* %include "VBox/asmdefs.mac" %include "VBox/err.mac" %include "VBox/hwacc_vmx.mac" %include "VBox/cpum.mac" %include "VBox/x86.mac" %include "HWACCMInternal.mac" %ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely. %macro vmwrite 2, int3 %endmacro %define vmlaunch int3 %define vmresume int3 %define vmsave int3 %define vmload int3 %define vmrun int3 %define clgi int3 %define stgi int3 %macro invlpga 2, int3 %endmacro %endif ;******************************************************************************* ;* Defined Constants And Macros * ;******************************************************************************* %ifdef RT_ARCH_AMD64 %define MAYBE_64_BIT %endif %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL %define MAYBE_64_BIT %else %ifdef RT_OS_DARWIN %ifdef RT_ARCH_AMD64 ;; ; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't ; risk loading a stale LDT value or something invalid. %define HWACCM_64_BIT_USE_NULL_SEL %endif %endif %endif ;; The offset of the XMM registers in X86FXSTATE. ; Use define because I'm too lazy to convert the struct. %define XMM_OFF_IN_X86FXSTATE 160 ;; This is too risky wrt. stability, performance and correctness. ;%define VBOX_WITH_DR6_EXPERIMENT 1 ;; @def MYPUSHAD ; Macro generating an equivalent to pushad ;; @def MYPOPAD ; Macro generating an equivalent to popad ;; @def MYPUSHSEGS ; Macro saving all segment registers on the stack. ; @param 1 full width register name ; @param 2 16-bit register name for \a 1. ;; @def MYPOPSEGS ; Macro restoring all segment registers on the stack ; @param 1 full width register name ; @param 2 16-bit register name for \a 1. %ifdef MAYBE_64_BIT ; Save a host and load the corresponding guest MSR (trashes rdx & rcx) %macro LOADGUESTMSR 2 mov rcx, %1 rdmsr push rdx push rax mov edx, dword [xSI + %2 + 4] mov eax, dword [xSI + %2] wrmsr %endmacro ; Save a guest and load the corresponding host MSR (trashes rdx & rcx) ; Only really useful for gs kernel base as that one can be changed behind our back (swapgs) %macro LOADHOSTMSREX 2 mov rcx, %1 rdmsr mov dword [xSI + %2], eax mov dword [xSI + %2 + 4], edx pop rax pop rdx wrmsr %endmacro ; Load the corresponding host MSR (trashes rdx & rcx) %macro LOADHOSTMSR 1 mov rcx, %1 pop rax pop rdx wrmsr %endmacro %endif %ifdef ASM_CALL64_GCC %macro MYPUSHAD64 0 push r15 push r14 push r13 push r12 push rbx %endmacro %macro MYPOPAD64 0 pop rbx pop r12 pop r13 pop r14 pop r15 %endmacro %else ; ASM_CALL64_MSC %macro MYPUSHAD64 0 push r15 push r14 push r13 push r12 push rbx push rsi push rdi %endmacro %macro MYPOPAD64 0 pop rdi pop rsi pop rbx pop r12 pop r13 pop r14 pop r15 %endmacro %endif ; trashes, rax, rdx & rcx %macro MYPUSHSEGS64 2 %ifndef HWACCM_64_BIT_USE_NULL_SEL mov %2, es push %1 mov %2, ds push %1 %endif ; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode, Solaris OTOH doesn't and we must save it. mov ecx, MSR_K8_FS_BASE rdmsr push rdx push rax %ifndef HWACCM_64_BIT_USE_NULL_SEL push fs %endif ; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel. The same happens on exit mov ecx, MSR_K8_GS_BASE rdmsr push rdx push rax %ifndef HWACCM_64_BIT_USE_NULL_SEL push gs %endif %endmacro ; trashes, rax, rdx & rcx %macro MYPOPSEGS64 2 ; Note: do not step through this code with a debugger! %ifndef HWACCM_64_BIT_USE_NULL_SEL xor eax, eax mov ds, ax mov es, ax mov fs, ax mov gs, ax %endif %ifndef HWACCM_64_BIT_USE_NULL_SEL pop gs %endif pop rax pop rdx mov ecx, MSR_K8_GS_BASE wrmsr %ifndef HWACCM_64_BIT_USE_NULL_SEL pop fs %endif pop rax pop rdx mov ecx, MSR_K8_FS_BASE wrmsr ; Now it's safe to step again %ifndef HWACCM_64_BIT_USE_NULL_SEL pop %1 mov ds, %2 pop %1 mov es, %2 %endif %endmacro %macro MYPUSHAD32 0 pushad %endmacro %macro MYPOPAD32 0 popad %endmacro %macro MYPUSHSEGS32 2 push ds push es push fs push gs %endmacro %macro MYPOPSEGS32 2 pop gs pop fs pop es pop ds %endmacro ;******************************************************************************* ;* External Symbols * ;******************************************************************************* %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL extern NAME(SUPR0AbsIs64bit) extern NAME(SUPR0Abs64bitKernelCS) extern NAME(SUPR0Abs64bitKernelSS) extern NAME(SUPR0Abs64bitKernelDS) extern NAME(SUPR0AbsKernelCS) %endif %ifdef VBOX_WITH_KERNEL_USING_XMM extern NAME(CPUMIsGuestFPUStateActive) %endif ;******************************************************************************* ;* Global Variables * ;******************************************************************************* %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL BEGINDATA ;; ; Store the SUPR0AbsIs64bit absolute value here so we can cmp/test without ; needing to clobber a register. (This trick doesn't quite work for PE btw. ; but that's not relevant atm.) GLOBALNAME g_fVMXIs64bitHost dd NAME(SUPR0AbsIs64bit) %endif BEGINCODE ;/** ; * Executes VMWRITE, 64-bit value. ; * ; * @returns VBox status code ; * @param idxField x86: [ebp + 08h] msc: rcx gcc: rdi VMCS index ; * @param u64Data x86: [ebp + 0ch] msc: rdx gcc: rsi VM field value ; */ ALIGNCODE(16) BEGINPROC VMXWriteVMCS64 %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC and edi, 0ffffffffh xor rax, rax vmwrite rdi, rsi %else and ecx, 0ffffffffh xor rax, rax vmwrite rcx, rdx %endif %else ; RT_ARCH_X86 mov ecx, [esp + 4] ; idxField lea edx, [esp + 8] ; &u64Data %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL vmwrite ecx, [edx] ; low dword jz .done jc .done inc ecx xor eax, eax vmwrite ecx, [edx + 4] ; high dword .done: %endif ; RT_ARCH_X86 jnc .valid_vmcs mov eax, VERR_VMX_INVALID_VMCS_PTR ret .valid_vmcs: jnz .the_end mov eax, VERR_VMX_INVALID_VMCS_FIELD .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: and edx, 0ffffffffh and ecx, 0ffffffffh xor eax, eax vmwrite rcx, [rdx] mov r8d, VERR_VMX_INVALID_VMCS_FIELD cmovz eax, r8d mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXWriteVMCS64 ;/** ; * Executes VMREAD, 64-bit value ; * ; * @returns VBox status code ; * @param idxField VMCS index ; * @param pData Ptr to store VM field value ; */ ;DECLASM(int) VMXReadVMCS64(uint32_t idxField, uint64_t *pData); ALIGNCODE(16) BEGINPROC VMXReadVMCS64 %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC and edi, 0ffffffffh xor rax, rax vmread [rsi], rdi %else and ecx, 0ffffffffh xor rax, rax vmread [rdx], rcx %endif %else ; RT_ARCH_X86 mov ecx, [esp + 4] ; idxField mov edx, [esp + 8] ; pData %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL vmread [edx], ecx ; low dword jz .done jc .done inc ecx xor eax, eax vmread [edx + 4], ecx ; high dword .done: %endif ; RT_ARCH_X86 jnc .valid_vmcs mov eax, VERR_VMX_INVALID_VMCS_PTR ret .valid_vmcs: jnz .the_end mov eax, VERR_VMX_INVALID_VMCS_FIELD .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: and edx, 0ffffffffh and ecx, 0ffffffffh xor eax, eax vmread [rdx], rcx mov r8d, VERR_VMX_INVALID_VMCS_FIELD cmovz eax, r8d mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXReadVMCS64 ;/** ; * Executes VMREAD, 32-bit value. ; * ; * @returns VBox status code ; * @param idxField VMCS index ; * @param pu32Data Ptr to store VM field value ; */ ;DECLASM(int) VMXReadVMCS32(uint32_t idxField, uint32_t *pu32Data); ALIGNCODE(16) BEGINPROC VMXReadVMCS32 %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC and edi, 0ffffffffh xor rax, rax vmread r10, rdi mov [rsi], r10d %else and ecx, 0ffffffffh xor rax, rax vmread r10, rcx mov [rdx], r10d %endif %else ; RT_ARCH_X86 mov ecx, [esp + 4] ; idxField mov edx, [esp + 8] ; pu32Data %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL xor eax, eax vmread [edx], ecx %endif ; RT_ARCH_X86 jnc .valid_vmcs mov eax, VERR_VMX_INVALID_VMCS_PTR ret .valid_vmcs: jnz .the_end mov eax, VERR_VMX_INVALID_VMCS_FIELD .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: and edx, 0ffffffffh and ecx, 0ffffffffh xor eax, eax vmread r10, rcx mov [rdx], r10d mov r8d, VERR_VMX_INVALID_VMCS_FIELD cmovz eax, r8d mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXReadVMCS32 ;/** ; * Executes VMWRITE, 32-bit value. ; * ; * @returns VBox status code ; * @param idxField VMCS index ; * @param u32Data Ptr to store VM field value ; */ ;DECLASM(int) VMXWriteVMCS32(uint32_t idxField, uint32_t u32Data); ALIGNCODE(16) BEGINPROC VMXWriteVMCS32 %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC and edi, 0ffffffffh and esi, 0ffffffffh xor rax, rax vmwrite rdi, rsi %else and ecx, 0ffffffffh and edx, 0ffffffffh xor rax, rax vmwrite rcx, rdx %endif %else ; RT_ARCH_X86 mov ecx, [esp + 4] ; idxField mov edx, [esp + 8] ; u32Data %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL xor eax, eax vmwrite ecx, edx %endif ; RT_ARCH_X86 jnc .valid_vmcs mov eax, VERR_VMX_INVALID_VMCS_PTR ret .valid_vmcs: jnz .the_end mov eax, VERR_VMX_INVALID_VMCS_FIELD .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: and edx, 0ffffffffh and ecx, 0ffffffffh xor eax, eax vmwrite rcx, rdx mov r8d, VERR_VMX_INVALID_VMCS_FIELD cmovz eax, r8d mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXWriteVMCS32 ;/** ; * Executes VMXON ; * ; * @returns VBox status code ; * @param HCPhysVMXOn Physical address of VMXON structure ; */ ;DECLASM(int) VMXEnable(RTHCPHYS HCPhysVMXOn); BEGINPROC VMXEnable %ifdef RT_ARCH_AMD64 xor rax, rax %ifdef ASM_CALL64_GCC push rdi %else push rcx %endif vmxon [rsp] %else ; RT_ARCH_X86 %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL xor eax, eax vmxon [esp + 4] %endif ; RT_ARCH_X86 jnc .good mov eax, VERR_VMX_INVALID_VMXON_PTR jmp .the_end .good: jnz .the_end mov eax, VERR_VMX_GENERIC .the_end: %ifdef RT_ARCH_AMD64 add rsp, 8 %endif ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: lea rdx, [rsp + 4] ; &HCPhysVMXOn. and edx, 0ffffffffh xor eax, eax vmxon [rdx] mov r8d, VERR_INVALID_PARAMETER cmovz eax, r8d mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXEnable ;/** ; * Executes VMXOFF ; */ ;DECLASM(void) VMXDisable(void); BEGINPROC VMXDisable %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL vmxoff .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: vmxoff jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXDisable ;/** ; * Executes VMCLEAR ; * ; * @returns VBox status code ; * @param HCPhysVMCS Physical address of VM control structure ; */ ;DECLASM(int) VMXClearVMCS(RTHCPHYS HCPhysVMCS); ALIGNCODE(16) BEGINPROC VMXClearVMCS %ifdef RT_ARCH_AMD64 xor rax, rax %ifdef ASM_CALL64_GCC push rdi %else push rcx %endif vmclear [rsp] %else ; RT_ARCH_X86 %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL xor eax, eax vmclear [esp + 4] %endif ; RT_ARCH_X86 jnc .the_end mov eax, VERR_VMX_INVALID_VMCS_PTR .the_end: %ifdef RT_ARCH_AMD64 add rsp, 8 %endif ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: lea rdx, [rsp + 4] ; &HCPhysVMCS and edx, 0ffffffffh xor eax, eax vmclear [rdx] mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ENDPROC VMXClearVMCS ;/** ; * Executes VMPTRLD ; * ; * @returns VBox status code ; * @param HCPhysVMCS Physical address of VMCS structure ; */ ;DECLASM(int) VMXActivateVMCS(RTHCPHYS HCPhysVMCS); ALIGNCODE(16) BEGINPROC VMXActivateVMCS %ifdef RT_ARCH_AMD64 xor rax, rax %ifdef ASM_CALL64_GCC push rdi %else push rcx %endif vmptrld [rsp] %else %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL xor eax, eax vmptrld [esp + 4] %endif jnc .the_end mov eax, VERR_VMX_INVALID_VMCS_PTR .the_end: %ifdef RT_ARCH_AMD64 add rsp, 8 %endif ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: lea rdx, [rsp + 4] ; &HCPhysVMCS and edx, 0ffffffffh xor eax, eax vmptrld [rdx] mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXActivateVMCS ;/** ; * Executes VMPTRST ; * ; * @returns VBox status code ; * @param [esp + 04h] gcc:rdi msc:rcx Param 1 - First parameter - Address that will receive the current pointer ; */ ;DECLASM(int) VMXGetActivateVMCS(RTHCPHYS *pVMCS); BEGINPROC VMXGetActivateVMCS %ifdef RT_OS_OS2 mov eax, VERR_NOT_SUPPORTED ret %else %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC vmptrst qword [rdi] %else vmptrst qword [rcx] %endif %else %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL vmptrst qword [esp+04h] %endif xor eax, eax .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: lea rdx, [rsp + 4] ; &HCPhysVMCS and edx, 0ffffffffh vmptrst qword [rdx] xor eax, eax jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL %endif ENDPROC VMXGetActivateVMCS ;/** ; * Invalidate a page using invept ; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush ; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer ; */ ;DECLASM(int) VMXR0InvEPT(VMX_FLUSH enmFlush, uint64_t *pDescriptor); BEGINPROC VMXR0InvEPT %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC and edi, 0ffffffffh xor rax, rax ; invept rdi, qword [rsi] DB 0x66, 0x0F, 0x38, 0x80, 0x3E %else and ecx, 0ffffffffh xor rax, rax ; invept rcx, qword [rdx] DB 0x66, 0x0F, 0x38, 0x80, 0xA %endif %else %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL mov ecx, [esp + 4] mov edx, [esp + 8] xor eax, eax ; invept ecx, qword [edx] DB 0x66, 0x0F, 0x38, 0x80, 0xA %endif jnc .valid_vmcs mov eax, VERR_VMX_INVALID_VMCS_PTR ret .valid_vmcs: jnz .the_end mov eax, VERR_INVALID_PARAMETER .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: and esp, 0ffffffffh mov ecx, [rsp + 4] ; enmFlush mov edx, [rsp + 8] ; pDescriptor xor eax, eax ; invept rcx, qword [rdx] DB 0x66, 0x0F, 0x38, 0x80, 0xA mov r8d, VERR_INVALID_PARAMETER cmovz eax, r8d mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXR0InvEPT ;/** ; * Invalidate a page using invvpid ; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush ; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer ; */ ;DECLASM(int) VMXR0InvVPID(VMX_FLUSH enmFlush, uint64_t *pDescriptor); BEGINPROC VMXR0InvVPID %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC and edi, 0ffffffffh xor rax, rax ;invvpid rdi, qword [rsi] DB 0x66, 0x0F, 0x38, 0x81, 0x3E %else and ecx, 0ffffffffh xor rax, rax ; invvpid rcx, qword [rdx] DB 0x66, 0x0F, 0x38, 0x81, 0xA %endif %else %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL cmp byte [NAME(g_fVMXIs64bitHost)], 0 jz .legacy_mode db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .legacy_mode: %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL mov ecx, [esp + 4] mov edx, [esp + 8] xor eax, eax ; invvpid ecx, qword [edx] DB 0x66, 0x0F, 0x38, 0x81, 0xA %endif jnc .valid_vmcs mov eax, VERR_VMX_INVALID_VMCS_PTR ret .valid_vmcs: jnz .the_end mov eax, VERR_INVALID_PARAMETER .the_end: ret %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: and esp, 0ffffffffh mov ecx, [rsp + 4] ; enmFlush mov edx, [rsp + 8] ; pDescriptor xor eax, eax ; invvpid rcx, qword [rdx] DB 0x66, 0x0F, 0x38, 0x81, 0xA mov r8d, VERR_INVALID_PARAMETER cmovz eax, r8d mov r9d, VERR_VMX_INVALID_VMCS_PTR cmovc eax, r9d jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL ENDPROC VMXR0InvVPID %if GC_ARCH_BITS == 64 ;; ; Executes INVLPGA ; ; @param pPageGC msc:rcx gcc:rdi x86:[esp+04] Virtual page to invalidate ; @param uASID msc:rdx gcc:rsi x86:[esp+0C] Tagged TLB id ; ;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID); BEGINPROC SVMR0InvlpgA %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC mov rax, rdi mov rcx, rsi %else mov rax, rcx mov rcx, rdx %endif %else mov eax, [esp + 4] mov ecx, [esp + 0Ch] %endif invlpga [xAX], ecx ret ENDPROC SVMR0InvlpgA %else ; GC_ARCH_BITS != 64 ;; ; Executes INVLPGA ; ; @param pPageGC msc:ecx gcc:edi x86:[esp+04] Virtual page to invalidate ; @param uASID msc:edx gcc:esi x86:[esp+08] Tagged TLB id ; ;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID); BEGINPROC SVMR0InvlpgA %ifdef RT_ARCH_AMD64 %ifdef ASM_CALL64_GCC movzx rax, edi mov ecx, esi %else ; from http://www.cs.cmu.edu/~fp/courses/15213-s06/misc/asm64-handout.pdf: ; ``Perhaps unexpectedly, instructions that move or generate 32-bit register ; values also set the upper 32 bits of the register to zero. Consequently ; there is no need for an instruction movzlq.'' mov eax, ecx mov ecx, edx %endif %else mov eax, [esp + 4] mov ecx, [esp + 8] %endif invlpga [xAX], ecx ret ENDPROC SVMR0InvlpgA %endif ; GC_ARCH_BITS != 64 %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ;/** ; * Gets 64-bit GDTR and IDTR on darwin. ; * @param pGdtr Where to store the 64-bit GDTR. ; * @param pIdtr Where to store the 64-bit IDTR. ; */ ;DECLASM(void) hwaccmR0Get64bitGDTRandIDTR(PX86XDTR64 pGdtr, PX86XDTR64 pIdtr); ALIGNCODE(16) BEGINPROC hwaccmR0Get64bitGDTRandIDTR db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .the_end: ret ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: and esp, 0ffffffffh mov ecx, [rsp + 4] ; pGdtr mov edx, [rsp + 8] ; pIdtr sgdt [rcx] sidt [rdx] jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 ENDPROC hwaccmR0Get64bitGDTRandIDTR ;/** ; * Gets 64-bit CR3 on darwin. ; * @returns CR3 ; */ ;DECLASM(uint64_t) hwaccmR0Get64bitCR3(void); ALIGNCODE(16) BEGINPROC hwaccmR0Get64bitCR3 db 0xea ; jmp far .sixtyfourbit_mode dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS) .the_end: ret ALIGNCODE(16) BITS 64 .sixtyfourbit_mode: mov rax, cr3 mov rdx, rax shr rdx, 32 jmp far [.fpret wrt rip] .fpret: ; 16:32 Pointer to .the_end. dd .the_end, NAME(SUPR0AbsKernelCS) BITS 32 ENDPROC hwaccmR0Get64bitCR3 %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL %ifdef VBOX_WITH_KERNEL_USING_XMM ;; ; Wrapper around vmx.pfnStartVM that preserves host XMM registers and ; load the guest ones when necessary. ; ; @cproto DECLASM(int) hwaccmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu, PFNHWACCMVMXSTARTVM pfnStartVM); ; ; @returns eax ; ; @param fResumeVM msc:rcx ; @param pCtx msc:rdx ; @param pVMCSCache msc:r8 ; @param pVM msc:r9 ; @param pVCpu msc:[rbp+30h] ; @param pfnStartVM msc:[rbp+38h] ; ; @remarks This is essentially the same code as hwaccmR0SVMRunWrapXMM, only the parameters differ a little bit. ; ; ASSUMING 64-bit and windows for now. ALIGNCODE(16) BEGINPROC hwaccmR0VMXStartVMWrapXMM push xBP mov xBP, xSP sub xSP, 0a0h + 040h ; Don't bother optimizing the frame size. ; spill input parameters. mov [xBP + 010h], rcx ; fResumeVM mov [xBP + 018h], rdx ; pCtx mov [xBP + 020h], r8 ; pVMCSCache mov [xBP + 028h], r9 ; pVM ; Ask CPUM whether we've started using the FPU yet. mov rcx, [xBP + 30h] ; pVCpu call NAME(CPUMIsGuestFPUStateActive) test al, al jnz .guest_fpu_state_active ; No need to mess with XMM registers just call the start routine and return. mov r11, [xBP + 38h] ; pfnStartVM mov r10, [xBP + 30h] ; pVCpu mov [xSP + 020h], r10 mov rcx, [xBP + 010h] ; fResumeVM mov rdx, [xBP + 018h] ; pCtx mov r8, [xBP + 020h] ; pVMCSCache mov r9, [xBP + 028h] ; pVM call r11 leave ret ALIGNCODE(8) .guest_fpu_state_active: ; Save the host XMM registers. movdqa [rsp + 040h + 000h], xmm6 movdqa [rsp + 040h + 010h], xmm7 movdqa [rsp + 040h + 020h], xmm8 movdqa [rsp + 040h + 030h], xmm9 movdqa [rsp + 040h + 040h], xmm10 movdqa [rsp + 040h + 050h], xmm11 movdqa [rsp + 040h + 060h], xmm12 movdqa [rsp + 040h + 070h], xmm13 movdqa [rsp + 040h + 080h], xmm14 movdqa [rsp + 040h + 090h], xmm15 ; Load the full guest XMM register state. mov r10, [xBP + 018h] ; pCtx lea r10, [r10 + XMM_OFF_IN_X86FXSTATE] movdqa xmm0, [r10 + 000h] movdqa xmm1, [r10 + 010h] movdqa xmm2, [r10 + 020h] movdqa xmm3, [r10 + 030h] movdqa xmm4, [r10 + 040h] movdqa xmm5, [r10 + 050h] movdqa xmm6, [r10 + 060h] movdqa xmm7, [r10 + 070h] movdqa xmm8, [r10 + 080h] movdqa xmm9, [r10 + 090h] movdqa xmm10, [r10 + 0a0h] movdqa xmm11, [r10 + 0b0h] movdqa xmm12, [r10 + 0c0h] movdqa xmm13, [r10 + 0d0h] movdqa xmm14, [r10 + 0e0h] movdqa xmm15, [r10 + 0f0h] ; Make the call (same as in the other case ). mov r11, [xBP + 38h] ; pfnStartVM mov r10, [xBP + 30h] ; pVCpu mov [xSP + 020h], r10 mov rcx, [xBP + 010h] ; fResumeVM mov rdx, [xBP + 018h] ; pCtx mov r8, [xBP + 020h] ; pVMCSCache mov r9, [xBP + 028h] ; pVM call r11 ; Save the guest XMM registers. mov r10, [xBP + 018h] ; pCtx lea r10, [r10 + XMM_OFF_IN_X86FXSTATE] movdqa [r10 + 000h], xmm0 movdqa [r10 + 010h], xmm1 movdqa [r10 + 020h], xmm2 movdqa [r10 + 030h], xmm3 movdqa [r10 + 040h], xmm4 movdqa [r10 + 050h], xmm5 movdqa [r10 + 060h], xmm6 movdqa [r10 + 070h], xmm7 movdqa [r10 + 080h], xmm8 movdqa [r10 + 090h], xmm9 movdqa [r10 + 0a0h], xmm10 movdqa [r10 + 0b0h], xmm11 movdqa [r10 + 0c0h], xmm12 movdqa [r10 + 0d0h], xmm13 movdqa [r10 + 0e0h], xmm14 movdqa [r10 + 0f0h], xmm15 ; Load the host XMM registers. movdqa xmm6, [rsp + 040h + 000h] movdqa xmm7, [rsp + 040h + 010h] movdqa xmm8, [rsp + 040h + 020h] movdqa xmm9, [rsp + 040h + 030h] movdqa xmm10, [rsp + 040h + 040h] movdqa xmm11, [rsp + 040h + 050h] movdqa xmm12, [rsp + 040h + 060h] movdqa xmm13, [rsp + 040h + 070h] movdqa xmm14, [rsp + 040h + 080h] movdqa xmm15, [rsp + 040h + 090h] leave ret ENDPROC hwaccmR0VMXStartVMWrapXMM ;; ; Wrapper around svm.pfnVMRun that preserves host XMM registers and ; load the guest ones when necessary. ; ; @cproto DECLASM(int) hwaccmR0SVMRunWrapXMM(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu, PFNHWACCMSVMVMRUN pfnVMRun); ; ; @returns eax ; ; @param pVMCBHostPhys msc:rcx ; @param pVMCBPhys msc:rdx ; @param pCtx msc:r8 ; @param pVM msc:r9 ; @param pVCpu msc:[rbp+30h] ; @param pfnVMRun msc:[rbp+38h] ; ; @remarks This is essentially the same code as hwaccmR0VMXStartVMWrapXMM, only the parameters differ a little bit. ; ; ASSUMING 64-bit and windows for now. ALIGNCODE(16) BEGINPROC hwaccmR0SVMRunWrapXMM push xBP mov xBP, xSP sub xSP, 0a0h + 040h ; Don't bother optimizing the frame size. ; spill input parameters. mov [xBP + 010h], rcx ; pVMCBHostPhys mov [xBP + 018h], rdx ; pVMCBPhys mov [xBP + 020h], r8 ; pCtx mov [xBP + 028h], r9 ; pVM ; Ask CPUM whether we've started using the FPU yet. mov rcx, [xBP + 30h] ; pVCpu call NAME(CPUMIsGuestFPUStateActive) test al, al jnz .guest_fpu_state_active ; No need to mess with XMM registers just call the start routine and return. mov r11, [xBP + 38h] ; pfnVMRun mov r10, [xBP + 30h] ; pVCpu mov [xSP + 020h], r10 mov rcx, [xBP + 010h] ; pVMCBHostPhys mov rdx, [xBP + 018h] ; pVMCBPhys mov r8, [xBP + 020h] ; pCtx mov r9, [xBP + 028h] ; pVM call r11 leave ret ALIGNCODE(8) .guest_fpu_state_active: ; Save the host XMM registers. movdqa [rsp + 040h + 000h], xmm6 movdqa [rsp + 040h + 010h], xmm7 movdqa [rsp + 040h + 020h], xmm8 movdqa [rsp + 040h + 030h], xmm9 movdqa [rsp + 040h + 040h], xmm10 movdqa [rsp + 040h + 050h], xmm11 movdqa [rsp + 040h + 060h], xmm12 movdqa [rsp + 040h + 070h], xmm13 movdqa [rsp + 040h + 080h], xmm14 movdqa [rsp + 040h + 090h], xmm15 ; Load the full guest XMM register state. mov r10, [xBP + 020h] ; pCtx lea r10, [r10 + XMM_OFF_IN_X86FXSTATE] movdqa xmm0, [r10 + 000h] movdqa xmm1, [r10 + 010h] movdqa xmm2, [r10 + 020h] movdqa xmm3, [r10 + 030h] movdqa xmm4, [r10 + 040h] movdqa xmm5, [r10 + 050h] movdqa xmm6, [r10 + 060h] movdqa xmm7, [r10 + 070h] movdqa xmm8, [r10 + 080h] movdqa xmm9, [r10 + 090h] movdqa xmm10, [r10 + 0a0h] movdqa xmm11, [r10 + 0b0h] movdqa xmm12, [r10 + 0c0h] movdqa xmm13, [r10 + 0d0h] movdqa xmm14, [r10 + 0e0h] movdqa xmm15, [r10 + 0f0h] ; Make the call (same as in the other case ). mov r11, [xBP + 38h] ; pfnVMRun mov r10, [xBP + 30h] ; pVCpu mov [xSP + 020h], r10 mov rcx, [xBP + 010h] ; pVMCBHostPhys mov rdx, [xBP + 018h] ; pVMCBPhys mov r8, [xBP + 020h] ; pCtx mov r9, [xBP + 028h] ; pVM call r11 ; Save the guest XMM registers. mov r10, [xBP + 020h] ; pCtx lea r10, [r10 + XMM_OFF_IN_X86FXSTATE] movdqa [r10 + 000h], xmm0 movdqa [r10 + 010h], xmm1 movdqa [r10 + 020h], xmm2 movdqa [r10 + 030h], xmm3 movdqa [r10 + 040h], xmm4 movdqa [r10 + 050h], xmm5 movdqa [r10 + 060h], xmm6 movdqa [r10 + 070h], xmm7 movdqa [r10 + 080h], xmm8 movdqa [r10 + 090h], xmm9 movdqa [r10 + 0a0h], xmm10 movdqa [r10 + 0b0h], xmm11 movdqa [r10 + 0c0h], xmm12 movdqa [r10 + 0d0h], xmm13 movdqa [r10 + 0e0h], xmm14 movdqa [r10 + 0f0h], xmm15 ; Load the host XMM registers. movdqa xmm6, [rsp + 040h + 000h] movdqa xmm7, [rsp + 040h + 010h] movdqa xmm8, [rsp + 040h + 020h] movdqa xmm9, [rsp + 040h + 030h] movdqa xmm10, [rsp + 040h + 040h] movdqa xmm11, [rsp + 040h + 050h] movdqa xmm12, [rsp + 040h + 060h] movdqa xmm13, [rsp + 040h + 070h] movdqa xmm14, [rsp + 040h + 080h] movdqa xmm15, [rsp + 040h + 090h] leave ret ENDPROC hwaccmR0SVMRunWrapXMM %endif ; VBOX_WITH_KERNEL_USING_XMM ; ; The default setup of the StartVM routines. ; %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL %define MY_NAME(name) name %+ _32 %else %define MY_NAME(name) name %endif %ifdef RT_ARCH_AMD64 %define MYPUSHAD MYPUSHAD64 %define MYPOPAD MYPOPAD64 %define MYPUSHSEGS MYPUSHSEGS64 %define MYPOPSEGS MYPOPSEGS64 %else %define MYPUSHAD MYPUSHAD32 %define MYPOPAD MYPOPAD32 %define MYPUSHSEGS MYPUSHSEGS32 %define MYPOPSEGS MYPOPSEGS32 %endif %include "HWACCMR0Mixed.mac" %ifdef VBOX_WITH_HYBRID_32BIT_KERNEL ; ; Write the wrapper procedures. ; ; These routines are probably being too paranoid about selector ; restoring, but better safe than sorry... ; ; DECLASM(int) VMXR0StartVM32(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache /*, PVM pVM, PVMCPU pVCpu*/); ALIGNCODE(16) BEGINPROC VMXR0StartVM32 cmp byte [NAME(g_fVMXIs64bitHost)], 0 je near NAME(VMXR0StartVM32_32) ; stack frame push esi push edi push fs push gs ; jmp far .thunk64 db 0xea dd .thunk64, NAME(SUPR0Abs64bitKernelCS) ALIGNCODE(16) BITS 64 .thunk64: sub esp, 20h mov edi, [rsp + 20h + 14h] ; fResume mov esi, [rsp + 20h + 18h] ; pCtx mov edx, [rsp + 20h + 1Ch] ; pCache call NAME(VMXR0StartVM32_64) add esp, 20h jmp far [.fpthunk32 wrt rip] .fpthunk32: ; 16:32 Pointer to .thunk32. dd .thunk32, NAME(SUPR0AbsKernelCS) BITS 32 ALIGNCODE(16) .thunk32: pop gs pop fs pop edi pop esi ret ENDPROC VMXR0StartVM32 ; DECLASM(int) VMXR0StartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache /*, PVM pVM, PVMCPU pVCpu*/); ALIGNCODE(16) BEGINPROC VMXR0StartVM64 cmp byte [NAME(g_fVMXIs64bitHost)], 0 je .not_in_long_mode ; stack frame push esi push edi push fs push gs ; jmp far .thunk64 db 0xea dd .thunk64, NAME(SUPR0Abs64bitKernelCS) ALIGNCODE(16) BITS 64 .thunk64: sub esp, 20h mov edi, [rsp + 20h + 14h] ; fResume mov esi, [rsp + 20h + 18h] ; pCtx mov edx, [rsp + 20h + 1Ch] ; pCache call NAME(VMXR0StartVM64_64) add esp, 20h jmp far [.fpthunk32 wrt rip] .fpthunk32: ; 16:32 Pointer to .thunk32. dd .thunk32, NAME(SUPR0AbsKernelCS) BITS 32 ALIGNCODE(16) .thunk32: pop gs pop fs pop edi pop esi ret .not_in_long_mode: mov eax, VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE ret ENDPROC VMXR0StartVM64 ;DECLASM(int) SVMR0VMRun(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx /*, PVM pVM, PVMCPU pVCpu*/); ALIGNCODE(16) BEGINPROC SVMR0VMRun cmp byte [NAME(g_fVMXIs64bitHost)], 0 je near NAME(SVMR0VMRun_32) ; stack frame push esi push edi push fs push gs ; jmp far .thunk64 db 0xea dd .thunk64, NAME(SUPR0Abs64bitKernelCS) ALIGNCODE(16) BITS 64 .thunk64: sub esp, 20h mov rdi, [rsp + 20h + 14h] ; pVMCBHostPhys mov rsi, [rsp + 20h + 1Ch] ; pVMCBPhys mov edx, [rsp + 20h + 24h] ; pCtx call NAME(SVMR0VMRun_64) add esp, 20h jmp far [.fpthunk32 wrt rip] .fpthunk32: ; 16:32 Pointer to .thunk32. dd .thunk32, NAME(SUPR0AbsKernelCS) BITS 32 ALIGNCODE(16) .thunk32: pop gs pop fs pop edi pop esi ret ENDPROC SVMR0VMRun ; DECLASM(int) SVMR0VMRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx /*, PVM pVM, PVMCPU pVCpu*/); ALIGNCODE(16) BEGINPROC SVMR0VMRun64 cmp byte [NAME(g_fVMXIs64bitHost)], 0 je .not_in_long_mode ; stack frame push esi push edi push fs push gs ; jmp far .thunk64 db 0xea dd .thunk64, NAME(SUPR0Abs64bitKernelCS) ALIGNCODE(16) BITS 64 .thunk64: sub esp, 20h mov rdi, [rbp + 20h + 14h] ; pVMCBHostPhys mov rsi, [rbp + 20h + 1Ch] ; pVMCBPhys mov edx, [rbp + 20h + 24h] ; pCtx call NAME(SVMR0VMRun64_64) add esp, 20h jmp far [.fpthunk32 wrt rip] .fpthunk32: ; 16:32 Pointer to .thunk32. dd .thunk32, NAME(SUPR0AbsKernelCS) BITS 32 ALIGNCODE(16) .thunk32: pop gs pop fs pop edi pop esi ret .not_in_long_mode: mov eax, VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE ret ENDPROC SVMR0VMRun64 ; ; Do it a second time pretending we're a 64-bit host. ; ; This *HAS* to be done at the very end of the file to avoid restoring ; macros. So, add new code *BEFORE* this mess. ; BITS 64 %undef RT_ARCH_X86 %define RT_ARCH_AMD64 %undef ASM_CALL64_MSC %define ASM_CALL64_GCC %define xS 8 %define xSP rsp %define xBP rbp %define xAX rax %define xBX rbx %define xCX rcx %define xDX rdx %define xDI rdi %define xSI rsi %define MY_NAME(name) name %+ _64 %define MYPUSHAD MYPUSHAD64 %define MYPOPAD MYPOPAD64 %define MYPUSHSEGS MYPUSHSEGS64 %define MYPOPSEGS MYPOPSEGS64 %include "HWACCMR0Mixed.mac" %endif ; VBOX_WITH_HYBRID_32BIT_KERNEL