HMR0A.asm@ 56296

Last change on this file since 56296 was 55738, checked in by vboxsync, 10 years ago
CPUMR0A.asm,HMR0A.asm: Microsoft documents doesn't extend the non-volatile status code XMM to the high YMM part, the high part is volatile. Use xsave/xrstor to speed up the loading and saving of guest XMM, YMM and ZMM registers.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 48.5 KB

Line
1	; $Id: HMR0A.asm 55738 2015-05-07 18:42:49Z vboxsync $
2	;; @file
3	; HM - Ring-0 VMX, SVM world-switch and helper routines
4	;
5
6	;
7	; Copyright (C) 2006-2015 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.virtualbox.org. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17
18	;*******************************************************************************
19	;* Header Files *
20	;*******************************************************************************
21	%include "VBox/asmdefs.mac"
22	%include "VBox/err.mac"
23	%include "VBox/vmm/hm_vmx.mac"
24	%include "VBox/vmm/cpum.mac"
25	%include "VBox/vmm/vm.mac"
26	%include "iprt/x86.mac"
27	%include "HMInternal.mac"
28
29	%ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely.
30	%macro vmwrite 2,
31	int3
32	%endmacro
33	%define vmlaunch int3
34	%define vmresume int3
35	%define vmsave int3
36	%define vmload int3
37	%define vmrun int3
38	%define clgi int3
39	%define stgi int3
40	%macro invlpga 2,
41	int3
42	%endmacro
43	%endif
44
45	;*******************************************************************************
46	;* Defined Constants And Macros *
47	;*******************************************************************************
48	%ifdef RT_ARCH_AMD64
49	%define MAYBE_64_BIT
50	%endif
51	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
52	%define MAYBE_64_BIT
53	%else
54	%ifdef RT_OS_DARWIN
55	%ifdef RT_ARCH_AMD64
56	;;
57	; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
58	; risk loading a stale LDT value or something invalid.
59	%define HM_64_BIT_USE_NULL_SEL
60	%endif
61	%endif
62	%endif
63
64	%ifdef RT_ARCH_AMD64
65	%define VBOX_SKIP_RESTORE_SEG
66	%endif
67
68	;; The offset of the XMM registers in X86FXSTATE.
69	; Use define because I'm too lazy to convert the struct.
70	%define XMM_OFF_IN_X86FXSTATE 160
71
72
73	;; @def MYPUSHAD
74	; Macro generating an equivalent to pushad
75
76	;; @def MYPOPAD
77	; Macro generating an equivalent to popad
78
79	;; @def MYPUSHSEGS
80	; Macro saving all segment registers on the stack.
81	; @param 1 full width register name
82	; @param 2 16-bit register name for \a 1.
83
84	;; @def MYPOPSEGS
85	; Macro restoring all segment registers on the stack
86	; @param 1 full width register name
87	; @param 2 16-bit register name for \a 1.
88
89	%ifdef ASM_CALL64_GCC
90	%macro MYPUSHAD64 0
91	push r15
92	push r14
93	push r13
94	push r12
95	push rbx
96	%endmacro
97	%macro MYPOPAD64 0
98	pop rbx
99	pop r12
100	pop r13
101	pop r14
102	pop r15
103	%endmacro
104
105	%else ; ASM_CALL64_MSC
106	%macro MYPUSHAD64 0
107	push r15
108	push r14
109	push r13
110	push r12
111	push rbx
112	push rsi
113	push rdi
114	%endmacro
115	%macro MYPOPAD64 0
116	pop rdi
117	pop rsi
118	pop rbx
119	pop r12
120	pop r13
121	pop r14
122	pop r15
123	%endmacro
124	%endif
125
126	%ifdef VBOX_SKIP_RESTORE_SEG
127	%macro MYPUSHSEGS64 2
128	%endmacro
129
130	%macro MYPOPSEGS64 2
131	%endmacro
132	%else ; !VBOX_SKIP_RESTORE_SEG
133	; trashes, rax, rdx & rcx
134	%macro MYPUSHSEGS64 2
135	%ifndef HM_64_BIT_USE_NULL_SEL
136	mov %2, es
137	push %1
138	mov %2, ds
139	push %1
140	%endif
141
142	; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode, Solaris OTOH doesn't and we must save it.
143	mov ecx, MSR_K8_FS_BASE
144	rdmsr
145	push rdx
146	push rax
147	%ifndef HM_64_BIT_USE_NULL_SEL
148	push fs
149	%endif
150
151	; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel. The same happens on exit
152	mov ecx, MSR_K8_GS_BASE
153	rdmsr
154	push rdx
155	push rax
156	%ifndef HM_64_BIT_USE_NULL_SEL
157	push gs
158	%endif
159	%endmacro
160
161	; trashes, rax, rdx & rcx
162	%macro MYPOPSEGS64 2
163	; Note: do not step through this code with a debugger!
164	%ifndef HM_64_BIT_USE_NULL_SEL
165	xor eax, eax
166	mov ds, ax
167	mov es, ax
168	mov fs, ax
169	mov gs, ax
170	%endif
171
172	%ifndef HM_64_BIT_USE_NULL_SEL
173	pop gs
174	%endif
175	pop rax
176	pop rdx
177	mov ecx, MSR_K8_GS_BASE
178	wrmsr
179
180	%ifndef HM_64_BIT_USE_NULL_SEL
181	pop fs
182	%endif
183	pop rax
184	pop rdx
185	mov ecx, MSR_K8_FS_BASE
186	wrmsr
187	; Now it's safe to step again
188
189	%ifndef HM_64_BIT_USE_NULL_SEL
190	pop %1
191	mov ds, %2
192	pop %1
193	mov es, %2
194	%endif
195	%endmacro
196	%endif ; VBOX_SKIP_RESTORE_SEG
197
198	%macro MYPUSHAD32 0
199	pushad
200	%endmacro
201	%macro MYPOPAD32 0
202	popad
203	%endmacro
204
205	%macro MYPUSHSEGS32 2
206	push ds
207	push es
208	push fs
209	push gs
210	%endmacro
211	%macro MYPOPSEGS32 2
212	pop gs
213	pop fs
214	pop es
215	pop ds
216	%endmacro
217
218
219	;*******************************************************************************
220	;* External Symbols *
221	;*******************************************************************************
222	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
223	extern NAME(SUPR0AbsIs64bit)
224	extern NAME(SUPR0Abs64bitKernelCS)
225	extern NAME(SUPR0Abs64bitKernelSS)
226	extern NAME(SUPR0Abs64bitKernelDS)
227	extern NAME(SUPR0AbsKernelCS)
228	%endif
229	%ifdef VBOX_WITH_KERNEL_USING_XMM
230	extern NAME(CPUMIsGuestFPUStateActive)
231	%endif
232
233
234	;*******************************************************************************
235	;* Global Variables *
236	;*******************************************************************************
237	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
238	BEGINDATA
239	;;
240	; Store the SUPR0AbsIs64bit absolute value here so we can cmp/test without
241	; needing to clobber a register. (This trick doesn't quite work for PE btw.
242	; but that's not relevant atm.)
243	GLOBALNAME g_fVMXIs64bitHost
244	dd NAME(SUPR0AbsIs64bit)
245	%endif
246
247
248	BEGINCODE
249
250
251	;/**
252	; * Restores host-state fields.
253	; *
254	; * @returns VBox status code
255	; * @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags.
256	; * @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct.
257	; */
258	ALIGNCODE(16)
259	BEGINPROC VMXRestoreHostState
260	%ifdef RT_ARCH_AMD64
261	%ifndef ASM_CALL64_GCC
262	; Use GCC's input registers since we'll be needing both rcx and rdx further
263	; down with the wrmsr instruction. Use the R10 and R11 register for saving
264	; RDI and RSI since MSC preserve the two latter registers.
265	mov r10, rdi
266	mov r11, rsi
267	mov rdi, rcx
268	mov rsi, rdx
269	%endif
270
271	test edi, VMX_RESTORE_HOST_GDTR
272	jz .test_idtr
273	lgdt [rsi + VMXRESTOREHOST.HostGdtr]
274
275	.test_idtr:
276	test edi, VMX_RESTORE_HOST_IDTR
277	jz .test_ds
278	lidt [rsi + VMXRESTOREHOST.HostIdtr]
279
280	.test_ds:
281	test edi, VMX_RESTORE_HOST_SEL_DS
282	jz .test_es
283	mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
284	mov ds, eax
285
286	.test_es:
287	test edi, VMX_RESTORE_HOST_SEL_ES
288	jz .test_tr
289	mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
290	mov es, eax
291
292	.test_tr:
293	test edi, VMX_RESTORE_HOST_SEL_TR
294	jz .test_fs
295	; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
296	mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
297	mov ax, dx
298	and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
299	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
300	test edi, VMX_RESTORE_HOST_GDT_READ_ONLY
301	jnz .gdt_readonly
302	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
303	ltr dx
304	jmp short .test_fs
305	.gdt_readonly:
306	mov rcx, cr0
307	mov r9, rcx
308	and rcx, ~X86_CR0_WP
309	mov cr0, rcx
310	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
311	ltr dx
312	mov cr0, r9
313
314	.test_fs:
315	;
316	; When restoring the selector values for FS and GS, we'll temporarily trash
317	; the base address (at least the high 32-bit bits, but quite possibly the
318	; whole base address), the wrmsr will restore it correctly. (VT-x actually
319	; restores the base correctly when leaving guest mode, but not the selector
320	; value, so there is little problem with interrupts being enabled prior to
321	; this restore job.)
322	; We'll disable ints once for both FS and GS as that's probably faster.
323	;
324	test edi, VMX_RESTORE_HOST_SEL_FS \| VMX_RESTORE_HOST_SEL_GS
325	jz .restore_success
326	pushfq
327	cli ; (see above)
328
329	test edi, VMX_RESTORE_HOST_SEL_FS
330	jz .test_gs
331	mov ax, word [rsi + VMXRESTOREHOST.uHostSelFS]
332	mov fs, eax
333	mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
334	mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
335	mov ecx, MSR_K8_FS_BASE
336	wrmsr
337
338	.test_gs:
339	test edi, VMX_RESTORE_HOST_SEL_GS
340	jz .restore_flags
341	mov ax, word [rsi + VMXRESTOREHOST.uHostSelGS]
342	mov gs, eax
343	mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
344	mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
345	mov ecx, MSR_K8_GS_BASE
346	wrmsr
347
348	.restore_flags:
349	popfq
350
351	.restore_success:
352	mov eax, VINF_SUCCESS
353	%ifndef ASM_CALL64_GCC
354	; Restore RDI and RSI on MSC.
355	mov rdi, r10
356	mov rsi, r11
357	%endif
358	%else ; RT_ARCH_X86
359	mov eax, VERR_NOT_IMPLEMENTED
360	%endif
361	ret
362	ENDPROC VMXRestoreHostState
363
364
365	;/**
366	; * Dispatches an NMI to the host.
367	; */
368	ALIGNCODE(16)
369	BEGINPROC VMXDispatchHostNmi
370	int 2 ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
371	ret
372	ENDPROC VMXDispatchHostNmi
373
374
375	;/**
376	; * Executes VMWRITE, 64-bit value.
377	; *
378	; * @returns VBox status code
379	; * @param idxField x86: [ebp + 08h] msc: rcx gcc: rdi VMCS index
380	; * @param u64Data x86: [ebp + 0ch] msc: rdx gcc: rsi VM field value
381	; */
382	ALIGNCODE(16)
383	BEGINPROC VMXWriteVmcs64
384	%ifdef RT_ARCH_AMD64
385	%ifdef ASM_CALL64_GCC
386	and edi, 0ffffffffh
387	xor rax, rax
388	vmwrite rdi, rsi
389	%else
390	and ecx, 0ffffffffh
391	xor rax, rax
392	vmwrite rcx, rdx
393	%endif
394	%else ; RT_ARCH_X86
395	mov ecx, [esp + 4] ; idxField
396	lea edx, [esp + 8] ; &u64Data
397	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
398	cmp byte [NAME(g_fVMXIs64bitHost)], 0
399	jz .legacy_mode
400	db 0xea ; jmp far .sixtyfourbit_mode
401	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
402	.legacy_mode:
403	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
404	vmwrite ecx, [edx] ; low dword
405	jz .done
406	jc .done
407	inc ecx
408	xor eax, eax
409	vmwrite ecx, [edx + 4] ; high dword
410	.done:
411	%endif ; RT_ARCH_X86
412	jnc .valid_vmcs
413	mov eax, VERR_VMX_INVALID_VMCS_PTR
414	ret
415	.valid_vmcs:
416	jnz .the_end
417	mov eax, VERR_VMX_INVALID_VMCS_FIELD
418	.the_end:
419	ret
420
421	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
422	ALIGNCODE(16)
423	BITS 64
424	.sixtyfourbit_mode:
425	and edx, 0ffffffffh
426	and ecx, 0ffffffffh
427	xor eax, eax
428	vmwrite rcx, [rdx]
429	mov r8d, VERR_VMX_INVALID_VMCS_FIELD
430	cmovz eax, r8d
431	mov r9d, VERR_VMX_INVALID_VMCS_PTR
432	cmovc eax, r9d
433	jmp far [.fpret wrt rip]
434	.fpret: ; 16:32 Pointer to .the_end.
435	dd .the_end, NAME(SUPR0AbsKernelCS)
436	BITS 32
437	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
438	ENDPROC VMXWriteVmcs64
439
440
441	;/**
442	; * Executes VMREAD, 64-bit value
443	; *
444	; * @returns VBox status code
445	; * @param idxField VMCS index
446	; * @param pData Ptr to store VM field value
447	; */
448	;DECLASM(int) VMXReadVmcs64(uint32_t idxField, uint64_t *pData);
449	ALIGNCODE(16)
450	BEGINPROC VMXReadVmcs64
451	%ifdef RT_ARCH_AMD64
452	%ifdef ASM_CALL64_GCC
453	and edi, 0ffffffffh
454	xor rax, rax
455	vmread [rsi], rdi
456	%else
457	and ecx, 0ffffffffh
458	xor rax, rax
459	vmread [rdx], rcx
460	%endif
461	%else ; RT_ARCH_X86
462	mov ecx, [esp + 4] ; idxField
463	mov edx, [esp + 8] ; pData
464	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
465	cmp byte [NAME(g_fVMXIs64bitHost)], 0
466	jz .legacy_mode
467	db 0xea ; jmp far .sixtyfourbit_mode
468	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
469	.legacy_mode:
470	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
471	vmread [edx], ecx ; low dword
472	jz .done
473	jc .done
474	inc ecx
475	xor eax, eax
476	vmread [edx + 4], ecx ; high dword
477	.done:
478	%endif ; RT_ARCH_X86
479	jnc .valid_vmcs
480	mov eax, VERR_VMX_INVALID_VMCS_PTR
481	ret
482	.valid_vmcs:
483	jnz .the_end
484	mov eax, VERR_VMX_INVALID_VMCS_FIELD
485	.the_end:
486	ret
487
488	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
489	ALIGNCODE(16)
490	BITS 64
491	.sixtyfourbit_mode:
492	and edx, 0ffffffffh
493	and ecx, 0ffffffffh
494	xor eax, eax
495	vmread [rdx], rcx
496	mov r8d, VERR_VMX_INVALID_VMCS_FIELD
497	cmovz eax, r8d
498	mov r9d, VERR_VMX_INVALID_VMCS_PTR
499	cmovc eax, r9d
500	jmp far [.fpret wrt rip]
501	.fpret: ; 16:32 Pointer to .the_end.
502	dd .the_end, NAME(SUPR0AbsKernelCS)
503	BITS 32
504	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
505	ENDPROC VMXReadVmcs64
506
507
508	;/**
509	; * Executes VMREAD, 32-bit value.
510	; *
511	; * @returns VBox status code
512	; * @param idxField VMCS index
513	; * @param pu32Data Ptr to store VM field value
514	; */
515	;DECLASM(int) VMXReadVmcs32(uint32_t idxField, uint32_t *pu32Data);
516	ALIGNCODE(16)
517	BEGINPROC VMXReadVmcs32
518	%ifdef RT_ARCH_AMD64
519	%ifdef ASM_CALL64_GCC
520	and edi, 0ffffffffh
521	xor rax, rax
522	vmread r10, rdi
523	mov [rsi], r10d
524	%else
525	and ecx, 0ffffffffh
526	xor rax, rax
527	vmread r10, rcx
528	mov [rdx], r10d
529	%endif
530	%else ; RT_ARCH_X86
531	mov ecx, [esp + 4] ; idxField
532	mov edx, [esp + 8] ; pu32Data
533	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
534	cmp byte [NAME(g_fVMXIs64bitHost)], 0
535	jz .legacy_mode
536	db 0xea ; jmp far .sixtyfourbit_mode
537	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
538	.legacy_mode:
539	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
540	xor eax, eax
541	vmread [edx], ecx
542	%endif ; RT_ARCH_X86
543	jnc .valid_vmcs
544	mov eax, VERR_VMX_INVALID_VMCS_PTR
545	ret
546	.valid_vmcs:
547	jnz .the_end
548	mov eax, VERR_VMX_INVALID_VMCS_FIELD
549	.the_end:
550	ret
551
552	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
553	ALIGNCODE(16)
554	BITS 64
555	.sixtyfourbit_mode:
556	and edx, 0ffffffffh
557	and ecx, 0ffffffffh
558	xor eax, eax
559	vmread r10, rcx
560	mov [rdx], r10d
561	mov r8d, VERR_VMX_INVALID_VMCS_FIELD
562	cmovz eax, r8d
563	mov r9d, VERR_VMX_INVALID_VMCS_PTR
564	cmovc eax, r9d
565	jmp far [.fpret wrt rip]
566	.fpret: ; 16:32 Pointer to .the_end.
567	dd .the_end, NAME(SUPR0AbsKernelCS)
568	BITS 32
569	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
570	ENDPROC VMXReadVmcs32
571
572
573	;/**
574	; * Executes VMWRITE, 32-bit value.
575	; *
576	; * @returns VBox status code
577	; * @param idxField VMCS index
578	; * @param u32Data Ptr to store VM field value
579	; */
580	;DECLASM(int) VMXWriteVmcs32(uint32_t idxField, uint32_t u32Data);
581	ALIGNCODE(16)
582	BEGINPROC VMXWriteVmcs32
583	%ifdef RT_ARCH_AMD64
584	%ifdef ASM_CALL64_GCC
585	and edi, 0ffffffffh
586	and esi, 0ffffffffh
587	xor rax, rax
588	vmwrite rdi, rsi
589	%else
590	and ecx, 0ffffffffh
591	and edx, 0ffffffffh
592	xor rax, rax
593	vmwrite rcx, rdx
594	%endif
595	%else ; RT_ARCH_X86
596	mov ecx, [esp + 4] ; idxField
597	mov edx, [esp + 8] ; u32Data
598	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
599	cmp byte [NAME(g_fVMXIs64bitHost)], 0
600	jz .legacy_mode
601	db 0xea ; jmp far .sixtyfourbit_mode
602	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
603	.legacy_mode:
604	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
605	xor eax, eax
606	vmwrite ecx, edx
607	%endif ; RT_ARCH_X86
608	jnc .valid_vmcs
609	mov eax, VERR_VMX_INVALID_VMCS_PTR
610	ret
611	.valid_vmcs:
612	jnz .the_end
613	mov eax, VERR_VMX_INVALID_VMCS_FIELD
614	.the_end:
615	ret
616
617	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
618	ALIGNCODE(16)
619	BITS 64
620	.sixtyfourbit_mode:
621	and edx, 0ffffffffh
622	and ecx, 0ffffffffh
623	xor eax, eax
624	vmwrite rcx, rdx
625	mov r8d, VERR_VMX_INVALID_VMCS_FIELD
626	cmovz eax, r8d
627	mov r9d, VERR_VMX_INVALID_VMCS_PTR
628	cmovc eax, r9d
629	jmp far [.fpret wrt rip]
630	.fpret: ; 16:32 Pointer to .the_end.
631	dd .the_end, NAME(SUPR0AbsKernelCS)
632	BITS 32
633	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
634	ENDPROC VMXWriteVmcs32
635
636
637	;/**
638	; * Executes VMXON
639	; *
640	; * @returns VBox status code
641	; * @param HCPhysVMXOn Physical address of VMXON structure
642	; */
643	;DECLASM(int) VMXEnable(RTHCPHYS HCPhysVMXOn);
644	BEGINPROC VMXEnable
645	%ifdef RT_ARCH_AMD64
646	xor rax, rax
647	%ifdef ASM_CALL64_GCC
648	push rdi
649	%else
650	push rcx
651	%endif
652	vmxon [rsp]
653	%else ; RT_ARCH_X86
654	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
655	cmp byte [NAME(g_fVMXIs64bitHost)], 0
656	jz .legacy_mode
657	db 0xea ; jmp far .sixtyfourbit_mode
658	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
659	.legacy_mode:
660	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
661	xor eax, eax
662	vmxon [esp + 4]
663	%endif ; RT_ARCH_X86
664	jnc .good
665	mov eax, VERR_VMX_INVALID_VMXON_PTR
666	jmp .the_end
667
668	.good:
669	jnz .the_end
670	mov eax, VERR_VMX_VMXON_FAILED
671
672	.the_end:
673	%ifdef RT_ARCH_AMD64
674	add rsp, 8
675	%endif
676	ret
677
678	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
679	ALIGNCODE(16)
680	BITS 64
681	.sixtyfourbit_mode:
682	lea rdx, [rsp + 4] ; &HCPhysVMXOn.
683	and edx, 0ffffffffh
684	xor eax, eax
685	vmxon [rdx]
686	mov r8d, VERR_VMX_VMXON_FAILED
687	cmovz eax, r8d
688	mov r9d, VERR_VMX_INVALID_VMXON_PTR
689	cmovc eax, r9d
690	jmp far [.fpret wrt rip]
691	.fpret: ; 16:32 Pointer to .the_end.
692	dd .the_end, NAME(SUPR0AbsKernelCS)
693	BITS 32
694	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
695	ENDPROC VMXEnable
696
697
698	;/**
699	; * Executes VMXOFF
700	; */
701	;DECLASM(void) VMXDisable(void);
702	BEGINPROC VMXDisable
703	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
704	cmp byte [NAME(g_fVMXIs64bitHost)], 0
705	jz .legacy_mode
706	db 0xea ; jmp far .sixtyfourbit_mode
707	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
708	.legacy_mode:
709	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
710	vmxoff
711	.the_end:
712	ret
713
714	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
715	ALIGNCODE(16)
716	BITS 64
717	.sixtyfourbit_mode:
718	vmxoff
719	jmp far [.fpret wrt rip]
720	.fpret: ; 16:32 Pointer to .the_end.
721	dd .the_end, NAME(SUPR0AbsKernelCS)
722	BITS 32
723	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
724	ENDPROC VMXDisable
725
726
727	;/**
728	; * Executes VMCLEAR
729	; *
730	; * @returns VBox status code
731	; * @param HCPhysVmcs Physical address of VM control structure
732	; */
733	;DECLASM(int) VMXClearVmcs(RTHCPHYS HCPhysVmcs);
734	ALIGNCODE(16)
735	BEGINPROC VMXClearVmcs
736	%ifdef RT_ARCH_AMD64
737	xor rax, rax
738	%ifdef ASM_CALL64_GCC
739	push rdi
740	%else
741	push rcx
742	%endif
743	vmclear [rsp]
744	%else ; RT_ARCH_X86
745	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
746	cmp byte [NAME(g_fVMXIs64bitHost)], 0
747	jz .legacy_mode
748	db 0xea ; jmp far .sixtyfourbit_mode
749	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
750	.legacy_mode:
751	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
752	xor eax, eax
753	vmclear [esp + 4]
754	%endif ; RT_ARCH_X86
755	jnc .the_end
756	mov eax, VERR_VMX_INVALID_VMCS_PTR
757	.the_end:
758	%ifdef RT_ARCH_AMD64
759	add rsp, 8
760	%endif
761	ret
762
763	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
764	ALIGNCODE(16)
765	BITS 64
766	.sixtyfourbit_mode:
767	lea rdx, [rsp + 4] ; &HCPhysVmcs
768	and edx, 0ffffffffh
769	xor eax, eax
770	vmclear [rdx]
771	mov r9d, VERR_VMX_INVALID_VMCS_PTR
772	cmovc eax, r9d
773	jmp far [.fpret wrt rip]
774	.fpret: ; 16:32 Pointer to .the_end.
775	dd .the_end, NAME(SUPR0AbsKernelCS)
776	BITS 32
777	%endif
778	ENDPROC VMXClearVmcs
779
780
781	;/**
782	; * Executes VMPTRLD
783	; *
784	; * @returns VBox status code
785	; * @param HCPhysVmcs Physical address of VMCS structure
786	; */
787	;DECLASM(int) VMXActivateVmcs(RTHCPHYS HCPhysVmcs);
788	ALIGNCODE(16)
789	BEGINPROC VMXActivateVmcs
790	%ifdef RT_ARCH_AMD64
791	xor rax, rax
792	%ifdef ASM_CALL64_GCC
793	push rdi
794	%else
795	push rcx
796	%endif
797	vmptrld [rsp]
798	%else
799	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
800	cmp byte [NAME(g_fVMXIs64bitHost)], 0
801	jz .legacy_mode
802	db 0xea ; jmp far .sixtyfourbit_mode
803	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
804	.legacy_mode:
805	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
806	xor eax, eax
807	vmptrld [esp + 4]
808	%endif
809	jnc .the_end
810	mov eax, VERR_VMX_INVALID_VMCS_PTR
811	.the_end:
812	%ifdef RT_ARCH_AMD64
813	add rsp, 8
814	%endif
815	ret
816
817	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
818	ALIGNCODE(16)
819	BITS 64
820	.sixtyfourbit_mode:
821	lea rdx, [rsp + 4] ; &HCPhysVmcs
822	and edx, 0ffffffffh
823	xor eax, eax
824	vmptrld [rdx]
825	mov r9d, VERR_VMX_INVALID_VMCS_PTR
826	cmovc eax, r9d
827	jmp far [.fpret wrt rip]
828	.fpret: ; 16:32 Pointer to .the_end.
829	dd .the_end, NAME(SUPR0AbsKernelCS)
830	BITS 32
831	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
832	ENDPROC VMXActivateVmcs
833
834
835	;/**
836	; * Executes VMPTRST
837	; *
838	; * @returns VBox status code
839	; * @param [esp + 04h] gcc:rdi msc:rcx Param 1 - First parameter - Address that will receive the current pointer
840	; */
841	;DECLASM(int) VMXGetActivatedVmcs(RTHCPHYS *pVMCS);
842	BEGINPROC VMXGetActivatedVmcs
843	%ifdef RT_OS_OS2
844	mov eax, VERR_NOT_SUPPORTED
845	ret
846	%else
847	%ifdef RT_ARCH_AMD64
848	%ifdef ASM_CALL64_GCC
849	vmptrst qword [rdi]
850	%else
851	vmptrst qword [rcx]
852	%endif
853	%else
854	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
855	cmp byte [NAME(g_fVMXIs64bitHost)], 0
856	jz .legacy_mode
857	db 0xea ; jmp far .sixtyfourbit_mode
858	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
859	.legacy_mode:
860	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
861	vmptrst qword [esp+04h]
862	%endif
863	xor eax, eax
864	.the_end:
865	ret
866
867	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
868	ALIGNCODE(16)
869	BITS 64
870	.sixtyfourbit_mode:
871	lea rdx, [rsp + 4] ; &HCPhysVmcs
872	and edx, 0ffffffffh
873	vmptrst qword [rdx]
874	xor eax, eax
875	jmp far [.fpret wrt rip]
876	.fpret: ; 16:32 Pointer to .the_end.
877	dd .the_end, NAME(SUPR0AbsKernelCS)
878	BITS 32
879	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
880	%endif
881	ENDPROC VMXGetActivatedVmcs
882
883	;/**
884	; * Invalidate a page using invept
885	; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush
886	; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer
887	; */
888	;DECLASM(int) VMXR0InvEPT(VMX_FLUSH enmFlush, uint64_t *pDescriptor);
889	BEGINPROC VMXR0InvEPT
890	%ifdef RT_ARCH_AMD64
891	%ifdef ASM_CALL64_GCC
892	and edi, 0ffffffffh
893	xor rax, rax
894	; invept rdi, qword [rsi]
895	DB 0x66, 0x0F, 0x38, 0x80, 0x3E
896	%else
897	and ecx, 0ffffffffh
898	xor rax, rax
899	; invept rcx, qword [rdx]
900	DB 0x66, 0x0F, 0x38, 0x80, 0xA
901	%endif
902	%else
903	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
904	cmp byte [NAME(g_fVMXIs64bitHost)], 0
905	jz .legacy_mode
906	db 0xea ; jmp far .sixtyfourbit_mode
907	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
908	.legacy_mode:
909	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
910	mov ecx, [esp + 4]
911	mov edx, [esp + 8]
912	xor eax, eax
913	; invept ecx, qword [edx]
914	DB 0x66, 0x0F, 0x38, 0x80, 0xA
915	%endif
916	jnc .valid_vmcs
917	mov eax, VERR_VMX_INVALID_VMCS_PTR
918	ret
919	.valid_vmcs:
920	jnz .the_end
921	mov eax, VERR_INVALID_PARAMETER
922	.the_end:
923	ret
924
925	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
926	ALIGNCODE(16)
927	BITS 64
928	.sixtyfourbit_mode:
929	and esp, 0ffffffffh
930	mov ecx, [rsp + 4] ; enmFlush
931	mov edx, [rsp + 8] ; pDescriptor
932	xor eax, eax
933	; invept rcx, qword [rdx]
934	DB 0x66, 0x0F, 0x38, 0x80, 0xA
935	mov r8d, VERR_INVALID_PARAMETER
936	cmovz eax, r8d
937	mov r9d, VERR_VMX_INVALID_VMCS_PTR
938	cmovc eax, r9d
939	jmp far [.fpret wrt rip]
940	.fpret: ; 16:32 Pointer to .the_end.
941	dd .the_end, NAME(SUPR0AbsKernelCS)
942	BITS 32
943	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
944	ENDPROC VMXR0InvEPT
945
946
947	;/**
948	; * Invalidate a page using invvpid
949	; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush
950	; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer
951	; */
952	;DECLASM(int) VMXR0InvVPID(VMX_FLUSH enmFlush, uint64_t *pDescriptor);
953	BEGINPROC VMXR0InvVPID
954	%ifdef RT_ARCH_AMD64
955	%ifdef ASM_CALL64_GCC
956	and edi, 0ffffffffh
957	xor rax, rax
958	; invvpid rdi, qword [rsi]
959	DB 0x66, 0x0F, 0x38, 0x81, 0x3E
960	%else
961	and ecx, 0ffffffffh
962	xor rax, rax
963	; invvpid rcx, qword [rdx]
964	DB 0x66, 0x0F, 0x38, 0x81, 0xA
965	%endif
966	%else
967	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
968	cmp byte [NAME(g_fVMXIs64bitHost)], 0
969	jz .legacy_mode
970	db 0xea ; jmp far .sixtyfourbit_mode
971	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
972	.legacy_mode:
973	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
974	mov ecx, [esp + 4]
975	mov edx, [esp + 8]
976	xor eax, eax
977	; invvpid ecx, qword [edx]
978	DB 0x66, 0x0F, 0x38, 0x81, 0xA
979	%endif
980	jnc .valid_vmcs
981	mov eax, VERR_VMX_INVALID_VMCS_PTR
982	ret
983	.valid_vmcs:
984	jnz .the_end
985	mov eax, VERR_INVALID_PARAMETER
986	.the_end:
987	ret
988
989	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
990	ALIGNCODE(16)
991	BITS 64
992	.sixtyfourbit_mode:
993	and esp, 0ffffffffh
994	mov ecx, [rsp + 4] ; enmFlush
995	mov edx, [rsp + 8] ; pDescriptor
996	xor eax, eax
997	; invvpid rcx, qword [rdx]
998	DB 0x66, 0x0F, 0x38, 0x81, 0xA
999	mov r8d, VERR_INVALID_PARAMETER
1000	cmovz eax, r8d
1001	mov r9d, VERR_VMX_INVALID_VMCS_PTR
1002	cmovc eax, r9d
1003	jmp far [.fpret wrt rip]
1004	.fpret: ; 16:32 Pointer to .the_end.
1005	dd .the_end, NAME(SUPR0AbsKernelCS)
1006	BITS 32
1007	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
1008	ENDPROC VMXR0InvVPID
1009
1010
1011	%if GC_ARCH_BITS == 64
1012	;;
1013	; Executes INVLPGA
1014	;
1015	; @param pPageGC msc:rcx gcc:rdi x86:[esp+04] Virtual page to invalidate
1016	; @param uASID msc:rdx gcc:rsi x86:[esp+0C] Tagged TLB id
1017	;
1018	;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
1019	BEGINPROC SVMR0InvlpgA
1020	%ifdef RT_ARCH_AMD64
1021	%ifdef ASM_CALL64_GCC
1022	mov rax, rdi
1023	mov rcx, rsi
1024	%else
1025	mov rax, rcx
1026	mov rcx, rdx
1027	%endif
1028	%else
1029	mov eax, [esp + 4]
1030	mov ecx, [esp + 0Ch]
1031	%endif
1032	invlpga [xAX], ecx
1033	ret
1034	ENDPROC SVMR0InvlpgA
1035
1036	%else ; GC_ARCH_BITS != 64
1037	;;
1038	; Executes INVLPGA
1039	;
1040	; @param pPageGC msc:ecx gcc:edi x86:[esp+04] Virtual page to invalidate
1041	; @param uASID msc:edx gcc:esi x86:[esp+08] Tagged TLB id
1042	;
1043	;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
1044	BEGINPROC SVMR0InvlpgA
1045	%ifdef RT_ARCH_AMD64
1046	%ifdef ASM_CALL64_GCC
1047	movzx rax, edi
1048	mov ecx, esi
1049	%else
1050	; from http://www.cs.cmu.edu/~fp/courses/15213-s06/misc/asm64-handout.pdf:
1051	; ``Perhaps unexpectedly, instructions that move or generate 32-bit register
1052	; values also set the upper 32 bits of the register to zero. Consequently
1053	; there is no need for an instruction movzlq.''
1054	mov eax, ecx
1055	mov ecx, edx
1056	%endif
1057	%else
1058	mov eax, [esp + 4]
1059	mov ecx, [esp + 8]
1060	%endif
1061	invlpga [xAX], ecx
1062	ret
1063	ENDPROC SVMR0InvlpgA
1064
1065	%endif ; GC_ARCH_BITS != 64
1066
1067	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1068
1069	;/**
1070	; * Gets 64-bit GDTR and IDTR on darwin.
1071	; * @param pGdtr Where to store the 64-bit GDTR.
1072	; * @param pIdtr Where to store the 64-bit IDTR.
1073	; */
1074	;DECLASM(void) HMR0Get64bitGdtrAndIdtr(PX86XDTR64 pGdtr, PX86XDTR64 pIdtr);
1075	ALIGNCODE(16)
1076	BEGINPROC HMR0Get64bitGdtrAndIdtr
1077	db 0xea ; jmp far .sixtyfourbit_mode
1078	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
1079	.the_end:
1080	ret
1081
1082	ALIGNCODE(16)
1083	BITS 64
1084	.sixtyfourbit_mode:
1085	and esp, 0ffffffffh
1086	mov ecx, [rsp + 4] ; pGdtr
1087	mov edx, [rsp + 8] ; pIdtr
1088	sgdt [rcx]
1089	sidt [rdx]
1090	jmp far [.fpret wrt rip]
1091	.fpret: ; 16:32 Pointer to .the_end.
1092	dd .the_end, NAME(SUPR0AbsKernelCS)
1093	BITS 32
1094	ENDPROC HMR0Get64bitGdtrAndIdtr
1095
1096
1097	;/**
1098	; * Gets 64-bit CR3 on darwin.
1099	; * @returns CR3
1100	; */
1101	;DECLASM(uint64_t) HMR0Get64bitCR3(void);
1102	ALIGNCODE(16)
1103	BEGINPROC HMR0Get64bitCR3
1104	db 0xea ; jmp far .sixtyfourbit_mode
1105	dd .sixtyfourbit_mode, NAME(SUPR0Abs64bitKernelCS)
1106	.the_end:
1107	ret
1108
1109	ALIGNCODE(16)
1110	BITS 64
1111	.sixtyfourbit_mode:
1112	mov rax, cr3
1113	mov rdx, rax
1114	shr rdx, 32
1115	jmp far [.fpret wrt rip]
1116	.fpret: ; 16:32 Pointer to .the_end.
1117	dd .the_end, NAME(SUPR0AbsKernelCS)
1118	BITS 32
1119	ENDPROC HMR0Get64bitCR3
1120
1121	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
1122
1123	%ifdef VBOX_WITH_KERNEL_USING_XMM
1124
1125	;;
1126	; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
1127	; load the guest ones when necessary.
1128	;
1129	; @cproto DECLASM(int) HMR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
1130	;
1131	; @returns eax
1132	;
1133	; @param fResumeVM msc:rcx
1134	; @param pCtx msc:rdx
1135	; @param pVMCSCache msc:r8
1136	; @param pVM msc:r9
1137	; @param pVCpu msc:[rbp+30h]
1138	; @param pfnStartVM msc:[rbp+38h]
1139	;
1140	; @remarks This is essentially the same code as HMR0SVMRunWrapXMM, only the parameters differ a little bit.
1141	;
1142	; ASSUMING 64-bit and windows for now.
1143	ALIGNCODE(16)
1144	BEGINPROC HMR0VMXStartVMWrapXMM
1145	push xBP
1146	mov xBP, xSP
1147	sub xSP, 0a0h + 040h ; Don't bother optimizing the frame size.
1148
1149	; spill input parameters.
1150	mov [xBP + 010h], rcx ; fResumeVM
1151	mov [xBP + 018h], rdx ; pCtx
1152	mov [xBP + 020h], r8 ; pVMCSCache
1153	mov [xBP + 028h], r9 ; pVM
1154
1155	; Ask CPUM whether we've started using the FPU yet.
1156	mov rcx, [xBP + 30h] ; pVCpu
1157	call NAME(CPUMIsGuestFPUStateActive)
1158	test al, al
1159	jnz .guest_fpu_state_active
1160
1161	; No need to mess with XMM registers just call the start routine and return.
1162	mov r11, [xBP + 38h] ; pfnStartVM
1163	mov r10, [xBP + 30h] ; pVCpu
1164	mov [xSP + 020h], r10
1165	mov rcx, [xBP + 010h] ; fResumeVM
1166	mov rdx, [xBP + 018h] ; pCtx
1167	mov r8, [xBP + 020h] ; pVMCSCache
1168	mov r9, [xBP + 028h] ; pVM
1169	call r11
1170
1171	leave
1172	ret
1173
1174	ALIGNCODE(8)
1175	.guest_fpu_state_active:
1176	; Save the non-volatile host XMM registers.
1177	movdqa [rsp + 040h + 000h], xmm6
1178	movdqa [rsp + 040h + 010h], xmm7
1179	movdqa [rsp + 040h + 020h], xmm8
1180	movdqa [rsp + 040h + 030h], xmm9
1181	movdqa [rsp + 040h + 040h], xmm10
1182	movdqa [rsp + 040h + 050h], xmm11
1183	movdqa [rsp + 040h + 060h], xmm12
1184	movdqa [rsp + 040h + 070h], xmm13
1185	movdqa [rsp + 040h + 080h], xmm14
1186	movdqa [rsp + 040h + 090h], xmm15
1187
1188	mov r10, [xBP + 018h] ; pCtx
1189	mov eax, [r10 + CPUMCTX.fXStateMask]
1190	test eax, eax
1191	jz .guest_fpu_state_manually
1192
1193	;
1194	; Using XSAVE to load the guest XMM, YMM and ZMM registers.
1195	;
1196	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1197	xor edx, edx
1198	mov r10, [r10 + CPUMCTX.pXStateR0]
1199	xrstor [r10]
1200
1201	; Make the call (same as in the other case ).
1202	mov r11, [xBP + 38h] ; pfnStartVM
1203	mov r10, [xBP + 30h] ; pVCpu
1204	mov [xSP + 020h], r10
1205	mov rcx, [xBP + 010h] ; fResumeVM
1206	mov rdx, [xBP + 018h] ; pCtx
1207	mov r8, [xBP + 020h] ; pVMCSCache
1208	mov r9, [xBP + 028h] ; pVM
1209	call r11
1210
1211	mov r11d, eax ; save return value (xsave below uses eax)
1212
1213	; Save the guest XMM registers.
1214	mov r10, [xBP + 018h] ; pCtx
1215	mov eax, [r10 + CPUMCTX.fXStateMask]
1216	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1217	xor edx, edx
1218	mov r10, [r10 + CPUMCTX.pXStateR0]
1219	xsave [r10]
1220
1221	mov eax, r11d ; restore return value.
1222
1223	.restore_non_volatile_host_xmm_regs:
1224	; Load the non-volatile host XMM registers.
1225	movdqa xmm6, [rsp + 040h + 000h]
1226	movdqa xmm7, [rsp + 040h + 010h]
1227	movdqa xmm8, [rsp + 040h + 020h]
1228	movdqa xmm9, [rsp + 040h + 030h]
1229	movdqa xmm10, [rsp + 040h + 040h]
1230	movdqa xmm11, [rsp + 040h + 050h]
1231	movdqa xmm12, [rsp + 040h + 060h]
1232	movdqa xmm13, [rsp + 040h + 070h]
1233	movdqa xmm14, [rsp + 040h + 080h]
1234	movdqa xmm15, [rsp + 040h + 090h]
1235	leave
1236	ret
1237
1238	;
1239	; No XSAVE, load and save the guest XMM registers manually.
1240	;
1241	.guest_fpu_state_manually:
1242	; Load the full guest XMM register state.
1243	mov r10, [r10 + CPUMCTX.pXStateR0]
1244	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
1245	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
1246	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
1247	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
1248	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
1249	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
1250	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
1251	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
1252	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
1253	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
1254	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
1255	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
1256	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
1257	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
1258	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
1259	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
1260
1261	; Make the call (same as in the other case ).
1262	mov r11, [xBP + 38h] ; pfnStartVM
1263	mov r10, [xBP + 30h] ; pVCpu
1264	mov [xSP + 020h], r10
1265	mov rcx, [xBP + 010h] ; fResumeVM
1266	mov rdx, [xBP + 018h] ; pCtx
1267	mov r8, [xBP + 020h] ; pVMCSCache
1268	mov r9, [xBP + 028h] ; pVM
1269	call r11
1270
1271	; Save the guest XMM registers.
1272	mov r10, [xBP + 018h] ; pCtx
1273	mov r10, [r10 + CPUMCTX.pXStateR0]
1274	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1275	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1276	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1277	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1278	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1279	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1280	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1281	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1282	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1283	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1284	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1285	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1286	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1287	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1288	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1289	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1290	jmp .restore_non_volatile_host_xmm_regs
1291	ENDPROC HMR0VMXStartVMWrapXMM
1292
1293	;;
1294	; Wrapper around svm.pfnVMRun that preserves host XMM registers and
1295	; load the guest ones when necessary.
1296	;
1297	; @cproto DECLASM(int) HMR0SVMRunWrapXMM(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu, PFNHMSVMVMRUN pfnVMRun);
1298	;
1299	; @returns eax
1300	;
1301	; @param pVMCBHostPhys msc:rcx
1302	; @param pVMCBPhys msc:rdx
1303	; @param pCtx msc:r8
1304	; @param pVM msc:r9
1305	; @param pVCpu msc:[rbp+30h]
1306	; @param pfnVMRun msc:[rbp+38h]
1307	;
1308	; @remarks This is essentially the same code as HMR0VMXStartVMWrapXMM, only the parameters differ a little bit.
1309	;
1310	; ASSUMING 64-bit and windows for now.
1311	ALIGNCODE(16)
1312	BEGINPROC HMR0SVMRunWrapXMM
1313	push xBP
1314	mov xBP, xSP
1315	sub xSP, 0a0h + 040h ; Don't bother optimizing the frame size.
1316
1317	; spill input parameters.
1318	mov [xBP + 010h], rcx ; pVMCBHostPhys
1319	mov [xBP + 018h], rdx ; pVMCBPhys
1320	mov [xBP + 020h], r8 ; pCtx
1321	mov [xBP + 028h], r9 ; pVM
1322
1323	; Ask CPUM whether we've started using the FPU yet.
1324	mov rcx, [xBP + 30h] ; pVCpu
1325	call NAME(CPUMIsGuestFPUStateActive)
1326	test al, al
1327	jnz .guest_fpu_state_active
1328
1329	; No need to mess with XMM registers just call the start routine and return.
1330	mov r11, [xBP + 38h] ; pfnVMRun
1331	mov r10, [xBP + 30h] ; pVCpu
1332	mov [xSP + 020h], r10
1333	mov rcx, [xBP + 010h] ; pVMCBHostPhys
1334	mov rdx, [xBP + 018h] ; pVMCBPhys
1335	mov r8, [xBP + 020h] ; pCtx
1336	mov r9, [xBP + 028h] ; pVM
1337	call r11
1338
1339	leave
1340	ret
1341
1342	ALIGNCODE(8)
1343	.guest_fpu_state_active:
1344	; Save the non-volatile host XMM registers.
1345	movdqa [rsp + 040h + 000h], xmm6
1346	movdqa [rsp + 040h + 010h], xmm7
1347	movdqa [rsp + 040h + 020h], xmm8
1348	movdqa [rsp + 040h + 030h], xmm9
1349	movdqa [rsp + 040h + 040h], xmm10
1350	movdqa [rsp + 040h + 050h], xmm11
1351	movdqa [rsp + 040h + 060h], xmm12
1352	movdqa [rsp + 040h + 070h], xmm13
1353	movdqa [rsp + 040h + 080h], xmm14
1354	movdqa [rsp + 040h + 090h], xmm15
1355
1356	mov r10, [xBP + 020h] ; pCtx
1357	mov eax, [r10 + CPUMCTX.fXStateMask]
1358	test eax, eax
1359	jz .guest_fpu_state_manually
1360
1361	;
1362	; Using XSAVE.
1363	;
1364	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1365	xor edx, edx
1366	mov r10, [r10 + CPUMCTX.pXStateR0]
1367	xrstor [r10]
1368
1369	; Make the call (same as in the other case ).
1370	mov r11, [xBP + 38h] ; pfnVMRun
1371	mov r10, [xBP + 30h] ; pVCpu
1372	mov [xSP + 020h], r10
1373	mov rcx, [xBP + 010h] ; pVMCBHostPhys
1374	mov rdx, [xBP + 018h] ; pVMCBPhys
1375	mov r8, [xBP + 020h] ; pCtx
1376	mov r9, [xBP + 028h] ; pVM
1377	call r11
1378
1379	mov r11d, eax ; save return value (xsave below uses eax)
1380
1381	; Save the guest XMM registers.
1382	mov r10, [xBP + 020h] ; pCtx
1383	mov eax, [r10 + CPUMCTX.fXStateMask]
1384	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1385	xor edx, edx
1386	mov r10, [r10 + CPUMCTX.pXStateR0]
1387	xsave [r10]
1388
1389	mov eax, r11d ; restore return value.
1390
1391	.restore_non_volatile_host_xmm_regs:
1392	; Load the non-volatile host XMM registers.
1393	movdqa xmm6, [rsp + 040h + 000h]
1394	movdqa xmm7, [rsp + 040h + 010h]
1395	movdqa xmm8, [rsp + 040h + 020h]
1396	movdqa xmm9, [rsp + 040h + 030h]
1397	movdqa xmm10, [rsp + 040h + 040h]
1398	movdqa xmm11, [rsp + 040h + 050h]
1399	movdqa xmm12, [rsp + 040h + 060h]
1400	movdqa xmm13, [rsp + 040h + 070h]
1401	movdqa xmm14, [rsp + 040h + 080h]
1402	movdqa xmm15, [rsp + 040h + 090h]
1403	leave
1404	ret
1405
1406	;
1407	; No XSAVE, load and save the guest XMM registers manually.
1408	;
1409	.guest_fpu_state_manually:
1410	; Load the full guest XMM register state.
1411	mov r10, [r10 + CPUMCTX.pXStateR0]
1412	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
1413	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
1414	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
1415	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
1416	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
1417	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
1418	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
1419	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
1420	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
1421	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
1422	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
1423	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
1424	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
1425	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
1426	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
1427	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
1428
1429	; Make the call (same as in the other case ).
1430	mov r11, [xBP + 38h] ; pfnVMRun
1431	mov r10, [xBP + 30h] ; pVCpu
1432	mov [xSP + 020h], r10
1433	mov rcx, [xBP + 010h] ; pVMCBHostPhys
1434	mov rdx, [xBP + 018h] ; pVMCBPhys
1435	mov r8, [xBP + 020h] ; pCtx
1436	mov r9, [xBP + 028h] ; pVM
1437	call r11
1438
1439	; Save the guest XMM registers.
1440	mov r10, [xBP + 020h] ; pCtx
1441	mov r10, [r10 + CPUMCTX.pXStateR0]
1442	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1443	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1444	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1445	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1446	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1447	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1448	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1449	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1450	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1451	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1452	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1453	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1454	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1455	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1456	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1457	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1458	jmp .restore_non_volatile_host_xmm_regs
1459	ENDPROC HMR0SVMRunWrapXMM
1460
1461	%endif ; VBOX_WITH_KERNEL_USING_XMM
1462
1463	;
1464	; The default setup of the StartVM routines.
1465	;
1466	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1467	%define MY_NAME(name) name %+ _32
1468	%else
1469	%define MY_NAME(name) name
1470	%endif
1471	%ifdef RT_ARCH_AMD64
1472	%define MYPUSHAD MYPUSHAD64
1473	%define MYPOPAD MYPOPAD64
1474	%define MYPUSHSEGS MYPUSHSEGS64
1475	%define MYPOPSEGS MYPOPSEGS64
1476	%else
1477	%define MYPUSHAD MYPUSHAD32
1478	%define MYPOPAD MYPOPAD32
1479	%define MYPUSHSEGS MYPUSHSEGS32
1480	%define MYPOPSEGS MYPOPSEGS32
1481	%endif
1482
1483	%include "HMR0Mixed.mac"
1484
1485
1486	%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1487	;
1488	; Write the wrapper procedures.
1489	;
1490	; These routines are probably being too paranoid about selector
1491	; restoring, but better safe than sorry...
1492	;
1493
1494	; DECLASM(int) VMXR0StartVM32(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu);
1495	ALIGNCODE(16)
1496	BEGINPROC VMXR0StartVM32
1497	cmp byte [NAME(g_fVMXIs64bitHost)], 0
1498	je near NAME(VMXR0StartVM32_32)
1499
1500	; stack frame
1501	push esi
1502	push edi
1503	push fs
1504	push gs
1505
1506	; jmp far .thunk64
1507	db 0xea
1508	dd .thunk64, NAME(SUPR0Abs64bitKernelCS)
1509
1510	ALIGNCODE(16)
1511	BITS 64
1512	.thunk64:
1513	sub esp, 20h
1514	mov edi, [rsp + 20h + 14h] ; fResume
1515	mov esi, [rsp + 20h + 18h] ; pCtx
1516	mov edx, [rsp + 20h + 1Ch] ; pCache
1517	mov ecx, [rsp + 20h + 20h] ; pVM
1518	mov r8, [rsp + 20h + 24h] ; pVCpu
1519	call NAME(VMXR0StartVM32_64)
1520	add esp, 20h
1521	jmp far [.fpthunk32 wrt rip]
1522	.fpthunk32: ; 16:32 Pointer to .thunk32.
1523	dd .thunk32, NAME(SUPR0AbsKernelCS)
1524
1525	BITS 32
1526	ALIGNCODE(16)
1527	.thunk32:
1528	pop gs
1529	pop fs
1530	pop edi
1531	pop esi
1532	ret
1533	ENDPROC VMXR0StartVM32
1534
1535
1536	; DECLASM(int) VMXR0StartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu);
1537	ALIGNCODE(16)
1538	BEGINPROC VMXR0StartVM64
1539	cmp byte [NAME(g_fVMXIs64bitHost)], 0
1540	je .not_in_long_mode
1541
1542	; stack frame
1543	push esi
1544	push edi
1545	push fs
1546	push gs
1547
1548	; jmp far .thunk64
1549	db 0xea
1550	dd .thunk64, NAME(SUPR0Abs64bitKernelCS)
1551
1552	ALIGNCODE(16)
1553	BITS 64
1554	.thunk64:
1555	sub esp, 20h
1556	mov edi, [rsp + 20h + 14h] ; fResume
1557	mov esi, [rsp + 20h + 18h] ; pCtx
1558	mov edx, [rsp + 20h + 1Ch] ; pCache
1559	mov ecx, [rsp + 20h + 20h] ; pVM
1560	mov r8, [rsp + 20h + 24h] ; pVCpu
1561	call NAME(VMXR0StartVM64_64)
1562	add esp, 20h
1563	jmp far [.fpthunk32 wrt rip]
1564	.fpthunk32: ; 16:32 Pointer to .thunk32.
1565	dd .thunk32, NAME(SUPR0AbsKernelCS)
1566
1567	BITS 32
1568	ALIGNCODE(16)
1569	.thunk32:
1570	pop gs
1571	pop fs
1572	pop edi
1573	pop esi
1574	ret
1575
1576	.not_in_long_mode:
1577	mov eax, VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE
1578	ret
1579	ENDPROC VMXR0StartVM64
1580
1581	;DECLASM(int) SVMR0VMRun(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu);
1582	ALIGNCODE(16)
1583	BEGINPROC SVMR0VMRun
1584	cmp byte [NAME(g_fVMXIs64bitHost)], 0
1585	je near NAME(SVMR0VMRun_32)
1586
1587	; stack frame
1588	push esi
1589	push edi
1590	push fs
1591	push gs
1592
1593	; jmp far .thunk64
1594	db 0xea
1595	dd .thunk64, NAME(SUPR0Abs64bitKernelCS)
1596
1597	ALIGNCODE(16)
1598	BITS 64
1599	.thunk64:
1600	sub esp, 20h
1601	mov rdi, [rsp + 20h + 14h] ; pVMCBHostPhys
1602	mov rsi, [rsp + 20h + 1Ch] ; pVMCBPhys
1603	mov edx, [rsp + 20h + 24h] ; pCtx
1604	mov ecx, [rsp + 20h + 28h] ; pVM
1605	mov r8d, [rsp + 20h + 2Ch] ; pVCpu
1606	call NAME(SVMR0VMRun_64)
1607	add esp, 20h
1608	jmp far [.fpthunk32 wrt rip]
1609	.fpthunk32: ; 16:32 Pointer to .thunk32.
1610	dd .thunk32, NAME(SUPR0AbsKernelCS)
1611
1612	BITS 32
1613	ALIGNCODE(16)
1614	.thunk32:
1615	pop gs
1616	pop fs
1617	pop edi
1618	pop esi
1619	ret
1620	ENDPROC SVMR0VMRun
1621
1622
1623	; DECLASM(int) SVMR0VMRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu);
1624	ALIGNCODE(16)
1625	BEGINPROC SVMR0VMRun64
1626	cmp byte [NAME(g_fVMXIs64bitHost)], 0
1627	je .not_in_long_mode
1628
1629	; stack frame
1630	push esi
1631	push edi
1632	push fs
1633	push gs
1634
1635	; jmp far .thunk64
1636	db 0xea
1637	dd .thunk64, NAME(SUPR0Abs64bitKernelCS)
1638
1639	ALIGNCODE(16)
1640	BITS 64
1641	.thunk64:
1642	sub esp, 20h
1643	mov rdi, [rsp + 20h + 14h] ; pVMCBHostPhys
1644	mov rsi, [rsp + 20h + 1Ch] ; pVMCBPhys
1645	mov edx, [rsp + 20h + 24h] ; pCtx
1646	mov ecx, [rsp + 20h + 28h] ; pVM
1647	mov r8d, [rsp + 20h + 2Ch] ; pVCpu
1648	call NAME(SVMR0VMRun64_64)
1649	add esp, 20h
1650	jmp far [.fpthunk32 wrt rip]
1651	.fpthunk32: ; 16:32 Pointer to .thunk32.
1652	dd .thunk32, NAME(SUPR0AbsKernelCS)
1653
1654	BITS 32
1655	ALIGNCODE(16)
1656	.thunk32:
1657	pop gs
1658	pop fs
1659	pop edi
1660	pop esi
1661	ret
1662
1663	.not_in_long_mode:
1664	mov eax, VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE
1665	ret
1666	ENDPROC SVMR0VMRun64
1667
1668	;
1669	; Do it a second time pretending we're a 64-bit host.
1670	;
1671	; This HAS to be done at the very end of the file to avoid restoring
1672	; macros. So, add new code BEFORE this mess.
1673	;
1674	BITS 64
1675	%undef RT_ARCH_X86
1676	%define RT_ARCH_AMD64
1677	%undef ASM_CALL64_MSC
1678	%define ASM_CALL64_GCC
1679	%define xCB 8
1680	%define xSP rsp
1681	%define xBP rbp
1682	%define xAX rax
1683	%define xBX rbx
1684	%define xCX rcx
1685	%define xDX rdx
1686	%define xDI rdi
1687	%define xSI rsi
1688	%define MY_NAME(name) name %+ _64
1689	%define MYPUSHAD MYPUSHAD64
1690	%define MYPOPAD MYPOPAD64
1691	%define MYPUSHSEGS MYPUSHSEGS64
1692	%define MYPOPSEGS MYPOPSEGS64
1693
1694	%include "HMR0Mixed.mac"
1695	%endif ; VBOX_WITH_HYBRID_32BIT_KERNEL
1696

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 56296

Download in other formats: