VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 94125

Last change on this file since 94125 was 93115, checked in by vboxsync, 3 years ago

scm --update-copyright-year
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 59.6 KB
Line 
1; $Id: HMR0A.asm 93115 2022-01-01 11:31:46Z vboxsync $
2;; @file
3; HM - Ring-0 VMX, SVM world-switch and helper routines.
4;
5
6;
7; Copyright (C) 2006-2022 Oracle Corporation
8;
9; This file is part of VirtualBox Open Source Edition (OSE), as
10; available from http://www.virtualbox.org. This file is free software;
11; you can redistribute it and/or modify it under the terms of the GNU
12; General Public License (GPL) as published by the Free Software
13; Foundation, in version 2 as it comes in the "COPYING" file of the
14; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16;
17
18;*********************************************************************************************************************************
19;* Header Files *
20;*********************************************************************************************************************************
21;%define RT_ASM_WITH_SEH64 - trouble with SEH, alignment and (probably) 2nd pass optimizations.
22%define RT_ASM_WITH_SEH64_ALT ; Use asmdefs.mac hackery for manually emitting unwind info.
23%include "VBox/asmdefs.mac"
24%include "VBox/err.mac"
25%include "VBox/vmm/hm_vmx.mac"
26%include "VBox/vmm/cpum.mac"
27%include "VBox/vmm/gvm.mac"
28%include "iprt/x86.mac"
29%include "HMInternal.mac"
30
31%ifndef RT_ARCH_AMD64
32 %error AMD64 only.
33%endif
34
35
36;*********************************************************************************************************************************
37;* Defined Constants And Macros *
38;*********************************************************************************************************************************
39;; The offset of the XMM registers in X86FXSTATE.
40; Use define because I'm too lazy to convert the struct.
41%define XMM_OFF_IN_X86FXSTATE 160
42
43;; Spectre filler for 64-bit mode.
44; Choosen to be an invalid address (also with 5 level paging).
45%define SPECTRE_FILLER 0x02204204207fffff
46
47;;
48; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation.
49;
50; @note This is normally done by hmR0VmxExportHostSegmentRegs and VMXRestoreHostState,
51; so much of this is untested code.
52; @{
53%define VMX_SKIP_GDTR
54%define VMX_SKIP_TR
55%define VBOX_SKIP_RESTORE_SEG
56%ifdef RT_OS_DARWIN
57 ; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
58 ; risk loading a stale LDT value or something invalid.
59 %define HM_64_BIT_USE_NULL_SEL
60 ; Darwin (Mavericks) uses IDTR limit to store the CPU number so we need to always restore it.
61 ; See @bugref{6875}.
62 %undef VMX_SKIP_IDTR
63%else
64 %define VMX_SKIP_IDTR
65%endif
66;; @}
67
68;; @def CALLEE_PRESERVED_REGISTER_COUNT
69; Number of registers pushed by PUSH_CALLEE_PRESERVED_REGISTERS
70%ifdef ASM_CALL64_GCC
71 %define CALLEE_PRESERVED_REGISTER_COUNT 5
72%else
73 %define CALLEE_PRESERVED_REGISTER_COUNT 7
74%endif
75
76;; @def PUSH_CALLEE_PRESERVED_REGISTERS
77; Macro for pushing all GPRs we must preserve for the caller.
78%macro PUSH_CALLEE_PRESERVED_REGISTERS 0
79 push r15
80 SEH64_PUSH_GREG r15
81 %assign cbFrame cbFrame + 8
82 %assign frm_saved_r15 -cbFrame
83
84 push r14
85 SEH64_PUSH_GREG r14
86 %assign cbFrame cbFrame + 8
87 %assign frm_saved_r14 -cbFrame
88
89 push r13
90 SEH64_PUSH_GREG r13
91 %assign cbFrame cbFrame + 8
92 %assign frm_saved_r13 -cbFrame
93
94 push r12
95 SEH64_PUSH_GREG r12
96 %assign cbFrame cbFrame + 8
97 %assign frm_saved_r12 -cbFrame
98
99 push rbx
100 SEH64_PUSH_GREG rbx
101 %assign cbFrame cbFrame + 8
102 %assign frm_saved_rbx -cbFrame
103
104 %ifdef ASM_CALL64_MSC
105 push rsi
106 SEH64_PUSH_GREG rsi
107 %assign cbFrame cbFrame + 8
108 %assign frm_saved_rsi -cbFrame
109
110 push rdi
111 SEH64_PUSH_GREG rdi
112 %assign cbFrame cbFrame + 8
113 %assign frm_saved_rdi -cbFrame
114 %endif
115%endmacro
116
117;; @def POP_CALLEE_PRESERVED_REGISTERS
118; Counterpart to PUSH_CALLEE_PRESERVED_REGISTERS for use in the epilogue.
119%macro POP_CALLEE_PRESERVED_REGISTERS 0
120 %ifdef ASM_CALL64_MSC
121 pop rdi
122 %assign cbFrame cbFrame - 8
123 %undef frm_saved_rdi
124
125 pop rsi
126 %assign cbFrame cbFrame - 8
127 %undef frm_saved_rsi
128 %endif
129 pop rbx
130 %assign cbFrame cbFrame - 8
131 %undef frm_saved_rbx
132
133 pop r12
134 %assign cbFrame cbFrame - 8
135 %undef frm_saved_r12
136
137 pop r13
138 %assign cbFrame cbFrame - 8
139 %undef frm_saved_r13
140
141 pop r14
142 %assign cbFrame cbFrame - 8
143 %undef frm_saved_r14
144
145 pop r15
146 %assign cbFrame cbFrame - 8
147 %undef frm_saved_r15
148%endmacro
149
150
151;; @def PUSH_RELEVANT_SEGMENT_REGISTERS
152; Macro saving all segment registers on the stack.
153; @param 1 Full width register name.
154; @param 2 16-bit register name for \a 1.
155; @cobbers rax, rdx, rcx
156%macro PUSH_RELEVANT_SEGMENT_REGISTERS 2
157 %ifndef VBOX_SKIP_RESTORE_SEG
158 %error untested code. probably does not work any more!
159 %ifndef HM_64_BIT_USE_NULL_SEL
160 mov %2, es
161 push %1
162 mov %2, ds
163 push %1
164 %endif
165
166 ; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode,
167 ; Solaris OTOH doesn't and we must save it.
168 mov ecx, MSR_K8_FS_BASE
169 rdmsr
170 push rdx
171 push rax
172 %ifndef HM_64_BIT_USE_NULL_SEL
173 push fs
174 %endif
175
176 ; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel.
177 ; The same happens on exit.
178 mov ecx, MSR_K8_GS_BASE
179 rdmsr
180 push rdx
181 push rax
182 %ifndef HM_64_BIT_USE_NULL_SEL
183 push gs
184 %endif
185 %endif ; !VBOX_SKIP_RESTORE_SEG
186%endmacro ; PUSH_RELEVANT_SEGMENT_REGISTERS
187
188;; @def POP_RELEVANT_SEGMENT_REGISTERS
189; Macro restoring all segment registers on the stack.
190; @param 1 Full width register name.
191; @param 2 16-bit register name for \a 1.
192; @cobbers rax, rdx, rcx
193%macro POP_RELEVANT_SEGMENT_REGISTERS 2
194 %ifndef VBOX_SKIP_RESTORE_SEG
195 %error untested code. probably does not work any more!
196 ; Note: do not step through this code with a debugger!
197 %ifndef HM_64_BIT_USE_NULL_SEL
198 xor eax, eax
199 mov ds, ax
200 mov es, ax
201 mov fs, ax
202 mov gs, ax
203 %endif
204
205 %ifndef HM_64_BIT_USE_NULL_SEL
206 pop gs
207 %endif
208 pop rax
209 pop rdx
210 mov ecx, MSR_K8_GS_BASE
211 wrmsr
212
213 %ifndef HM_64_BIT_USE_NULL_SEL
214 pop fs
215 %endif
216 pop rax
217 pop rdx
218 mov ecx, MSR_K8_FS_BASE
219 wrmsr
220 ; Now it's safe to step again
221
222 %ifndef HM_64_BIT_USE_NULL_SEL
223 pop %1
224 mov ds, %2
225 pop %1
226 mov es, %2
227 %endif
228 %endif ; !VBOX_SKIP_RESTORE_SEG
229%endmacro ; POP_RELEVANT_SEGMENT_REGISTERS
230
231
232;*********************************************************************************************************************************
233;* External Symbols *
234;*********************************************************************************************************************************
235%ifdef VBOX_WITH_KERNEL_USING_XMM
236extern NAME(CPUMIsGuestFPUStateActive)
237%endif
238
239
240BEGINCODE
241
242
243;;
244; Used on platforms with poor inline assembly support to retrieve all the
245; info from the CPU and put it in the @a pRestoreHost structure.
246;
247; @returns VBox status code
248; @param pRestoreHost msc: rcx gcc: rdi Pointer to the RestoreHost struct.
249; @param fHaveFsGsBase msc: dl gcc: sil Whether we can use rdfsbase or not.
250;
251ALIGNCODE(64)
252BEGINPROC hmR0VmxExportHostSegmentRegsAsmHlp
253%ifdef ASM_CALL64_MSC
254 %define pRestoreHost rcx
255%elifdef ASM_CALL64_GCC
256 %define pRestoreHost rdi
257%else
258 %error Unknown calling convension.
259%endif
260 SEH64_END_PROLOGUE
261
262 ; Start with the FS and GS base so we can trash DL/SIL.
263%ifdef ASM_CALL64_MSC
264 or dl, dl
265%else
266 or sil, sil
267%endif
268 jz .use_rdmsr_for_fs_and_gs_base
269 rdfsbase rax
270 mov [pRestoreHost + VMXRESTOREHOST.uHostFSBase], rax
271 rdgsbase rax
272 mov [pRestoreHost + VMXRESTOREHOST.uHostGSBase], rax
273.done_fs_and_gs_base:
274
275 ; TR, GDTR and IDTR
276 str [pRestoreHost + VMXRESTOREHOST.uHostSelTR]
277 sgdt [pRestoreHost + VMXRESTOREHOST.HostGdtr]
278 sidt [pRestoreHost + VMXRESTOREHOST.HostIdtr]
279
280 ; Segment registers.
281 xor eax, eax
282 mov eax, cs
283 mov [pRestoreHost + VMXRESTOREHOST.uHostSelCS], ax
284
285 mov eax, ss
286 mov [pRestoreHost + VMXRESTOREHOST.uHostSelSS], ax
287
288 mov eax, gs
289 mov [pRestoreHost + VMXRESTOREHOST.uHostSelGS], ax
290
291 mov eax, fs
292 mov [pRestoreHost + VMXRESTOREHOST.uHostSelFS], ax
293
294 mov eax, es
295 mov [pRestoreHost + VMXRESTOREHOST.uHostSelES], ax
296
297 mov eax, ds
298 mov [pRestoreHost + VMXRESTOREHOST.uHostSelDS], ax
299
300 ret
301
302ALIGNCODE(16)
303.use_rdmsr_for_fs_and_gs_base:
304%ifdef ASM_CALL64_MSC
305 mov r8, pRestoreHost
306%endif
307
308 mov ecx, MSR_K8_FS_BASE
309 rdmsr
310 shl rdx, 32
311 or rdx, rax
312 mov [r8 + VMXRESTOREHOST.uHostFSBase], rdx
313
314 mov ecx, MSR_K8_GS_BASE
315 rdmsr
316 shl rdx, 32
317 or rdx, rax
318 mov [r8 + VMXRESTOREHOST.uHostGSBase], rdx
319
320%ifdef ASM_CALL64_MSC
321 mov pRestoreHost, r8
322%endif
323 jmp .done_fs_and_gs_base
324%undef pRestoreHost
325ENDPROC hmR0VmxExportHostSegmentRegsAsmHlp
326
327
328;;
329; Restores host-state fields.
330;
331; @returns VBox status code
332; @param f32RestoreHost msc: ecx gcc: edi RestoreHost flags.
333; @param pRestoreHost msc: rdx gcc: rsi Pointer to the RestoreHost struct.
334;
335ALIGNCODE(64)
336BEGINPROC VMXRestoreHostState
337%ifndef ASM_CALL64_GCC
338 ; Use GCC's input registers since we'll be needing both rcx and rdx further
339 ; down with the wrmsr instruction. Use the R10 and R11 register for saving
340 ; RDI and RSI since MSC preserve the two latter registers.
341 mov r10, rdi
342 mov r11, rsi
343 mov rdi, rcx
344 mov rsi, rdx
345%endif
346 SEH64_END_PROLOGUE
347
348.restore_gdtr:
349 test edi, VMX_RESTORE_HOST_GDTR
350 jz .restore_idtr
351 lgdt [rsi + VMXRESTOREHOST.HostGdtr]
352
353.restore_idtr:
354 test edi, VMX_RESTORE_HOST_IDTR
355 jz .restore_ds
356 lidt [rsi + VMXRESTOREHOST.HostIdtr]
357
358.restore_ds:
359 test edi, VMX_RESTORE_HOST_SEL_DS
360 jz .restore_es
361 mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
362 mov ds, eax
363
364.restore_es:
365 test edi, VMX_RESTORE_HOST_SEL_ES
366 jz .restore_tr
367 mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
368 mov es, eax
369
370.restore_tr:
371 test edi, VMX_RESTORE_HOST_SEL_TR
372 jz .restore_fs
373 ; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
374 mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
375 mov ax, dx
376 and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
377 test edi, VMX_RESTORE_HOST_GDT_READ_ONLY | VMX_RESTORE_HOST_GDT_NEED_WRITABLE
378 jnz .gdt_readonly_or_need_writable
379 add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
380 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
381 ltr dx
382
383.restore_fs:
384 ;
385 ; When restoring the selector values for FS and GS, we'll temporarily trash
386 ; the base address (at least the high 32-bit bits, but quite possibly the
387 ; whole base address), the wrmsr will restore it correctly. (VT-x actually
388 ; restores the base correctly when leaving guest mode, but not the selector
389 ; value, so there is little problem with interrupts being enabled prior to
390 ; this restore job.)
391 ; We'll disable ints once for both FS and GS as that's probably faster.
392 ;
393 test edi, VMX_RESTORE_HOST_SEL_FS | VMX_RESTORE_HOST_SEL_GS
394 jz .restore_success
395 pushfq
396 cli ; (see above)
397
398 test edi, VMX_RESTORE_HOST_CAN_USE_WRFSBASE_AND_WRGSBASE
399 jz .restore_fs_using_wrmsr
400
401.restore_fs_using_wrfsbase:
402 test edi, VMX_RESTORE_HOST_SEL_FS
403 jz .restore_gs_using_wrgsbase
404 mov rax, qword [rsi + VMXRESTOREHOST.uHostFSBase]
405 mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
406 mov fs, ecx
407 wrfsbase rax
408
409.restore_gs_using_wrgsbase:
410 test edi, VMX_RESTORE_HOST_SEL_GS
411 jz .restore_flags
412 mov rax, qword [rsi + VMXRESTOREHOST.uHostGSBase]
413 mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
414 mov gs, ecx
415 wrgsbase rax
416
417.restore_flags:
418 popfq
419
420.restore_success:
421 mov eax, VINF_SUCCESS
422%ifndef ASM_CALL64_GCC
423 ; Restore RDI and RSI on MSC.
424 mov rdi, r10
425 mov rsi, r11
426%endif
427 ret
428
429ALIGNCODE(8)
430.gdt_readonly_or_need_writable:
431 test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
432 jnz .gdt_readonly_need_writable
433.gdt_readonly:
434 mov rcx, cr0
435 mov r9, rcx
436 add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
437 and rcx, ~X86_CR0_WP
438 mov cr0, rcx
439 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
440 ltr dx
441 mov cr0, r9
442 jmp .restore_fs
443
444ALIGNCODE(8)
445.gdt_readonly_need_writable:
446 add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw
447 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
448 lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
449 ltr dx
450 lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT
451 jmp .restore_fs
452
453ALIGNCODE(8)
454.restore_fs_using_wrmsr:
455 test edi, VMX_RESTORE_HOST_SEL_FS
456 jz .restore_gs_using_wrmsr
457 mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
458 mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
459 mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
460 mov fs, ecx
461 mov ecx, MSR_K8_FS_BASE
462 wrmsr
463
464.restore_gs_using_wrmsr:
465 test edi, VMX_RESTORE_HOST_SEL_GS
466 jz .restore_flags
467 mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
468 mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
469 mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
470 mov gs, ecx
471 mov ecx, MSR_K8_GS_BASE
472 wrmsr
473 jmp .restore_flags
474ENDPROC VMXRestoreHostState
475
476
477;;
478; Clears the MDS buffers using VERW.
479ALIGNCODE(16)
480BEGINPROC hmR0MdsClear
481 SEH64_END_PROLOGUE
482 sub xSP, xCB
483 mov [xSP], ds
484 verw [xSP]
485 add xSP, xCB
486 ret
487ENDPROC hmR0MdsClear
488
489
490;;
491; Dispatches an NMI to the host.
492;
493ALIGNCODE(16)
494BEGINPROC VMXDispatchHostNmi
495 ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
496 SEH64_END_PROLOGUE
497 int 2
498 ret
499ENDPROC VMXDispatchHostNmi
500
501
502;;
503; Common restore logic for success and error paths. We duplicate this because we
504; don't want to waste writing the VINF_SUCCESS return value to the stack in the
505; regular code path.
506;
507; @param 1 Zero if regular return, non-zero if error return. Controls label emission.
508; @param 2 fLoadSaveGuestXcr0 value
509; @param 3 The (HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY) + HM_WSF_IBPB_EXIT value.
510; The entry values are either all set or not at all, as we're too lazy to flesh out all the variants.
511; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
512;
513; @note Important that this does not modify cbFrame or rsp.
514%macro RESTORE_STATE_VMX 4
515 ; Restore base and limit of the IDTR & GDTR.
516 %ifndef VMX_SKIP_IDTR
517 lidt [rsp + cbFrame + frm_saved_idtr]
518 %endif
519 %ifndef VMX_SKIP_GDTR
520 lgdt [rsp + cbFrame + frm_saved_gdtr]
521 %endif
522
523 ; Save the guest state and restore the non-volatile registers. We use rcx=pGstCtx (&pVCpu->cpum.GstCtx) here.
524 mov [rsp + cbFrame + frm_guest_rcx], rcx
525 mov rcx, [rsp + cbFrame + frm_pGstCtx]
526
527 mov qword [rcx + CPUMCTX.eax], rax
528 mov qword [rcx + CPUMCTX.edx], rdx
529 rdtsc
530 mov qword [rcx + CPUMCTX.ebp], rbp
531 lea rbp, [rsp + cbFrame] ; re-establish the frame pointer as early as possible.
532 shl rdx, 20h
533 or rax, rdx ; TSC value in RAX
534 mov rdx, [rbp + frm_guest_rcx]
535 mov qword [rcx + CPUMCTX.ecx], rdx
536 mov rdx, SPECTRE_FILLER ; FILLER in RDX
537 mov qword [rcx + GVMCPU.hmr0 + HMR0PERVCPU.uTscExit - VMCPU.cpum.GstCtx], rax
538 mov qword [rcx + CPUMCTX.r8], r8
539 mov r8, rdx
540 mov qword [rcx + CPUMCTX.r9], r9
541 mov r9, rdx
542 mov qword [rcx + CPUMCTX.r10], r10
543 mov r10, rdx
544 mov qword [rcx + CPUMCTX.r11], r11
545 mov r11, rdx
546 mov qword [rcx + CPUMCTX.esi], rsi
547 %ifdef ASM_CALL64_MSC
548 mov rsi, [rbp + frm_saved_rsi]
549 %else
550 mov rsi, rdx
551 %endif
552 mov qword [rcx + CPUMCTX.edi], rdi
553 %ifdef ASM_CALL64_MSC
554 mov rdi, [rbp + frm_saved_rdi]
555 %else
556 mov rdi, rdx
557 %endif
558 mov qword [rcx + CPUMCTX.ebx], rbx
559 mov rbx, [rbp + frm_saved_rbx]
560 mov qword [rcx + CPUMCTX.r12], r12
561 mov r12, [rbp + frm_saved_r12]
562 mov qword [rcx + CPUMCTX.r13], r13
563 mov r13, [rbp + frm_saved_r13]
564 mov qword [rcx + CPUMCTX.r14], r14
565 mov r14, [rbp + frm_saved_r14]
566 mov qword [rcx + CPUMCTX.r15], r15
567 mov r15, [rbp + frm_saved_r15]
568
569 mov rax, cr2
570 mov qword [rcx + CPUMCTX.cr2], rax
571 mov rax, rdx
572
573 %if %4 != 0
574 ; Save the context pointer in r8 for the SSE save/restore.
575 mov r8, rcx
576 %endif
577
578 %if %3 & HM_WSF_IBPB_EXIT
579 ; Fight spectre (trashes rax, rdx and rcx).
580 %if %1 = 0 ; Skip this in failure branch (=> guru)
581 mov ecx, MSR_IA32_PRED_CMD
582 mov eax, MSR_IA32_PRED_CMD_F_IBPB
583 xor edx, edx
584 wrmsr
585 %endif
586 %endif
587
588 %ifndef VMX_SKIP_TR
589 ; Restore TSS selector; must mark it as not busy before using ltr!
590 ; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p).
591 %ifndef VMX_SKIP_GDTR
592 lgdt [rbp + frm_saved_gdtr]
593 %endif
594 movzx eax, word [rbp + frm_saved_tr]
595 mov ecx, eax
596 and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
597 add rax, [rbp + frm_saved_gdtr + 2] ; eax <- GDTR.address + descriptor offset
598 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
599 ltr cx
600 %endif
601 movzx edx, word [rbp + frm_saved_ldtr]
602 test edx, edx
603 jz %%skip_ldt_write
604 lldt dx
605%%skip_ldt_write:
606
607 %if %1 != 0
608.return_after_vmwrite_error:
609 %endif
610 ; Restore segment registers.
611 ;POP_RELEVANT_SEGMENT_REGISTERS rax, ax - currently broken.
612
613 %if %2 != 0
614 ; Restore the host XCR0.
615 xor ecx, ecx
616 mov eax, [rbp + frm_uHostXcr0]
617 mov edx, [rbp + frm_uHostXcr0 + 4]
618 xsetbv
619 %endif
620%endmacro ; RESTORE_STATE_VMX
621
622
623;;
624; hmR0VmxStartVm template
625;
626; @param 1 The suffix of the variation.
627; @param 2 fLoadSaveGuestXcr0 value
628; @param 3 The HM_WSF_IBPB_ENTRY + HM_WSF_IBPB_EXIT value.
629; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
630; Drivers shouldn't use AVX registers without saving+loading:
631; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
632; However the compiler docs have different idea:
633; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
634; We'll go with the former for now.
635;
636%macro hmR0VmxStartVmTemplate 4
637
638;;
639; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
640;
641; @returns VBox status code
642; @param pVmcsInfo msc:rcx, gcc:rdi Pointer to the VMCS info (for cached host RIP and RSP).
643; @param pVCpu msc:rdx, gcc:rsi The cross context virtual CPU structure of the calling EMT.
644; @param fResume msc:r8l, gcc:dl Whether to use vmlauch/vmresume.
645;
646ALIGNCODE(64)
647BEGINPROC RT_CONCAT(hmR0VmxStartVm,%1)
648 %ifdef VBOX_WITH_KERNEL_USING_XMM
649 %if %4 = 0
650 ;
651 ; The non-saving variant will currently check the two SSE preconditions and pick
652 ; the right variant to continue with. Later we can see if we can't manage to
653 ; move these decisions into hmR0VmxUpdateStartVmFunction().
654 ;
655 %ifdef ASM_CALL64_MSC
656 test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
657 %else
658 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
659 %endif
660 jz .save_xmm_no_need
661 %ifdef ASM_CALL64_MSC
662 cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
663 %else
664 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
665 %endif
666 je RT_CONCAT3(hmR0VmxStartVm,%1,_SseManual)
667 jmp RT_CONCAT3(hmR0VmxStartVm,%1,_SseXSave)
668.save_xmm_no_need:
669 %endif
670 %endif
671 push xBP
672 SEH64_PUSH_xBP
673 mov xBP, xSP
674 SEH64_SET_FRAME_xBP 0
675 pushf
676 cli
677
678 %define frm_fRFlags -008h
679 %define frm_pGstCtx -010h ; Where we stash guest CPU context for use after the vmrun.
680 %define frm_uHostXcr0 -020h ; 128-bit
681 %define frm_saved_gdtr -02ah ; 16+64: Only used when VMX_SKIP_GDTR isn't defined
682 %define frm_saved_tr -02ch ; 16-bit: Only used when VMX_SKIP_TR isn't defined
683 %define frm_MDS_seg -030h ; 16-bit: Temporary storage for the MDS flushing.
684 %define frm_saved_idtr -03ah ; 16+64: Only used when VMX_SKIP_IDTR isn't defined
685 %define frm_saved_ldtr -03ch ; 16-bit: always saved.
686 %define frm_rcError -040h ; 32-bit: Error status code (not used in the success path)
687 %define frm_guest_rcx -048h ; Temporary storage slot for guest RCX.
688 %if %4 = 0
689 %assign cbFrame 048h
690 %else
691 %define frm_saved_xmm6 -050h
692 %define frm_saved_xmm7 -060h
693 %define frm_saved_xmm8 -070h
694 %define frm_saved_xmm9 -080h
695 %define frm_saved_xmm10 -090h
696 %define frm_saved_xmm11 -0a0h
697 %define frm_saved_xmm12 -0b0h
698 %define frm_saved_xmm13 -0c0h
699 %define frm_saved_xmm14 -0d0h
700 %define frm_saved_xmm15 -0e0h
701 %define frm_saved_mxcsr -0f0h
702 %assign cbFrame 0f0h
703 %endif
704 %assign cbBaseFrame cbFrame
705 sub rsp, cbFrame - 8h
706 SEH64_ALLOCATE_STACK cbFrame
707
708 ; Save all general purpose host registers.
709 PUSH_CALLEE_PRESERVED_REGISTERS
710 ;PUSH_RELEVANT_SEGMENT_REGISTERS xAX, ax - currently broken
711 SEH64_END_PROLOGUE
712
713 ;
714 ; Unify the input parameter registers: r9=pVmcsInfo, rsi=pVCpu, bl=fResume, rdi=&pVCpu->cpum.GstCtx;
715 ;
716 %ifdef ASM_CALL64_GCC
717 mov r9, rdi ; pVmcsInfo
718 mov ebx, edx ; fResume
719 %else
720 mov r9, rcx ; pVmcsInfo
721 mov rsi, rdx ; pVCpu
722 mov ebx, r8d ; fResume
723 %endif
724 lea rdi, [rsi + VMCPU.cpum.GstCtx]
725 mov [rbp + frm_pGstCtx], rdi
726
727 %ifdef VBOX_STRICT
728 ;
729 ; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
730 ;
731 cmp byte [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fLoadSaveGuestXcr0], %2
732 mov eax, VERR_VMX_STARTVM_PRECOND_0
733 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
734
735 mov eax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fWorldSwitcher]
736 and eax, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT
737 cmp eax, %3
738 mov eax, VERR_VMX_STARTVM_PRECOND_1
739 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
740
741 %ifdef VBOX_WITH_KERNEL_USING_XMM
742 mov eax, VERR_VMX_STARTVM_PRECOND_2
743 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
744 %if %4 = 0
745 jnz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
746 %else
747 jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
748
749 mov eax, VERR_VMX_STARTVM_PRECOND_3
750 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
751 %if %4 = 1
752 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
753 %elif %4 = 2
754 je NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
755 %else
756 %error Invalid template parameter 4.
757 %endif
758 %endif
759 %endif
760 %endif ; VBOX_STRICT
761
762 %if %4 != 0
763 ; Save the non-volatile SSE host register state.
764 movdqa [rbp + frm_saved_xmm6 ], xmm6
765 movdqa [rbp + frm_saved_xmm7 ], xmm7
766 movdqa [rbp + frm_saved_xmm8 ], xmm8
767 movdqa [rbp + frm_saved_xmm9 ], xmm9
768 movdqa [rbp + frm_saved_xmm10], xmm10
769 movdqa [rbp + frm_saved_xmm11], xmm11
770 movdqa [rbp + frm_saved_xmm12], xmm12
771 movdqa [rbp + frm_saved_xmm13], xmm13
772 movdqa [rbp + frm_saved_xmm14], xmm14
773 movdqa [rbp + frm_saved_xmm15], xmm15
774 stmxcsr [rbp + frm_saved_mxcsr]
775
776 ; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx.
777 lea rcx, [rdi + CPUMCTX.XState]
778 %if %4 = 1 ; manual
779 movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h]
780 movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h]
781 movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h]
782 movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h]
783 movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h]
784 movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h]
785 movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h]
786 movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h]
787 movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h]
788 movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h]
789 movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h]
790 movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h]
791 movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h]
792 movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h]
793 movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h]
794 movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h]
795 ldmxcsr [rcx + X86FXSTATE.MXCSR]
796 %elif %4 = 2 ; use xrstor/xsave
797 mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
798 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
799 xor edx, edx
800 xrstor [rcx]
801 %else
802 %error invalid template parameter 4
803 %endif
804 %endif
805
806 %if %2 != 0
807 ; Save the host XCR0 and load the guest one if necessary.
808 ; Note! Trashes rax, rdx and rcx.
809 xor ecx, ecx
810 xgetbv ; save the host one on the stack
811 mov [rbp + frm_uHostXcr0], eax
812 mov [rbp + frm_uHostXcr0 + 4], edx
813
814 mov eax, [rdi + CPUMCTX.aXcr] ; load the guest one
815 mov edx, [rdi + CPUMCTX.aXcr + 4]
816 xor ecx, ecx ; paranoia; indicate that we must restore XCR0 (popped into ecx, thus 0)
817 xsetbv
818 %endif
819
820 ; Save host LDTR.
821 sldt word [rbp + frm_saved_ldtr]
822
823 %ifndef VMX_SKIP_TR
824 ; The host TR limit is reset to 0x67; save & restore it manually.
825 str word [rbp + frm_saved_tr]
826 %endif
827
828 %ifndef VMX_SKIP_GDTR
829 ; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
830 sgdt [rbp + frm_saved_gdtr]
831 %endif
832 %ifndef VMX_SKIP_IDTR
833 sidt [rbp + frm_saved_idtr]
834 %endif
835
836 ; Load CR2 if necessary (expensive as writing CR2 is a synchronizing instruction - (bird: still expensive on 10980xe)).
837 mov rcx, qword [rdi + CPUMCTX.cr2]
838 mov rdx, cr2
839 cmp rcx, rdx
840 je .skip_cr2_write
841 mov cr2, rcx
842.skip_cr2_write:
843
844 ; Set the vmlaunch/vmresume "return" host RIP and RSP values if they've changed (unlikly).
845 ; The vmwrite isn't quite for free (on an 10980xe at least), thus we check if anything changed
846 ; before writing here.
847 lea rcx, [NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) wrt rip]
848 cmp rcx, [r9 + VMXVMCSINFO.uHostRip]
849 jne .write_host_rip
850.wrote_host_rip:
851 cmp rsp, [r9 + VMXVMCSINFO.uHostRsp]
852 jne .write_host_rsp
853.wrote_host_rsp:
854
855 ;
856 ; Fight spectre and similar. Trashes rax, rcx, and rdx.
857 ;
858 %if %3 & (HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY) ; The eax:edx value is the same for the first two.
859 AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
860 mov eax, MSR_IA32_PRED_CMD_F_IBPB
861 xor edx, edx
862 %endif
863 %if %3 & HM_WSF_IBPB_ENTRY ; Indirect branch barrier.
864 mov ecx, MSR_IA32_PRED_CMD
865 wrmsr
866 %endif
867 %if %3 & HM_WSF_L1D_ENTRY ; Level 1 data cache flush.
868 mov ecx, MSR_IA32_FLUSH_CMD
869 wrmsr
870 %elif %3 & HM_WSF_MDS_ENTRY ; MDS flushing is included in L1D_FLUSH
871 mov word [rbp + frm_MDS_seg], ds
872 verw word [rbp + frm_MDS_seg]
873 %endif
874
875 ; Resume or start VM?
876 cmp bl, 0 ; fResume
877
878 ; Load guest general purpose registers.
879 mov rax, qword [rdi + CPUMCTX.eax]
880 mov rbx, qword [rdi + CPUMCTX.ebx]
881 mov rcx, qword [rdi + CPUMCTX.ecx]
882 mov rdx, qword [rdi + CPUMCTX.edx]
883 mov rbp, qword [rdi + CPUMCTX.ebp]
884 mov rsi, qword [rdi + CPUMCTX.esi]
885 mov r8, qword [rdi + CPUMCTX.r8]
886 mov r9, qword [rdi + CPUMCTX.r9]
887 mov r10, qword [rdi + CPUMCTX.r10]
888 mov r11, qword [rdi + CPUMCTX.r11]
889 mov r12, qword [rdi + CPUMCTX.r12]
890 mov r13, qword [rdi + CPUMCTX.r13]
891 mov r14, qword [rdi + CPUMCTX.r14]
892 mov r15, qword [rdi + CPUMCTX.r15]
893 mov rdi, qword [rdi + CPUMCTX.edi]
894
895 je .vmlaunch64_launch
896
897 vmresume
898 jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr)
899 jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed)
900 jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmresume detected a failure
901
902.vmlaunch64_launch:
903 vmlaunch
904 jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr)
905 jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed)
906 jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmlaunch detected a failure
907
908
909; Put these two outside the normal code path as they should rarely change.
910ALIGNCODE(8)
911.write_host_rip:
912 %ifdef VBOX_WITH_STATISTICS
913 inc qword [rsi + VMCPU.hm + HMCPU.StatVmxWriteHostRip]
914 %endif
915 mov [r9 + VMXVMCSINFO.uHostRip], rcx
916 mov eax, VMX_VMCS_HOST_RIP ;; @todo It is only strictly necessary to write VMX_VMCS_HOST_RIP when
917 vmwrite rax, rcx ;; the VMXVMCSINFO::pfnStartVM function changes (eventually
918 %ifdef VBOX_STRICT ;; take the Windows/SSE stuff into account then)...
919 jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed)
920 %endif
921 jmp .wrote_host_rip
922
923ALIGNCODE(8)
924.write_host_rsp:
925 %ifdef VBOX_WITH_STATISTICS
926 inc qword [rsi + VMCPU.hm + HMCPU.StatVmxWriteHostRsp]
927 %endif
928 mov [r9 + VMXVMCSINFO.uHostRsp], rsp
929 mov eax, VMX_VMCS_HOST_RSP
930 vmwrite rax, rsp
931 %ifdef VBOX_STRICT
932 jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed)
933 %endif
934 jmp .wrote_host_rsp
935
936ALIGNCODE(64)
937GLOBALNAME RT_CONCAT(hmR0VmxStartVmHostRIP,%1)
938 RESTORE_STATE_VMX 0, %2, %3, %4
939 mov eax, VINF_SUCCESS
940
941.vmstart64_end:
942 %if %4 != 0
943 mov r11d, eax ; save the return code.
944
945 ; Save the guest SSE state related to non-volatile and volatile SSE registers.
946 lea rcx, [r8 + CPUMCTX.XState]
947 %if %4 = 1 ; manual
948 stmxcsr [rcx + X86FXSTATE.MXCSR]
949 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
950 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
951 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
952 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
953 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
954 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
955 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
956 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
957 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
958 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
959 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
960 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
961 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
962 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
963 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
964 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
965 %elif %4 = 2 ; use xrstor/xsave
966 mov eax, [r8 + CPUMCTX.fXStateMask]
967 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
968 xor edx, edx
969 xsave [rcx]
970 %else
971 %error invalid template parameter 4
972 %endif
973
974 ; Restore the host non-volatile SSE register state.
975 ldmxcsr [rbp + frm_saved_mxcsr]
976 movdqa xmm6, [rbp + frm_saved_xmm6 ]
977 movdqa xmm7, [rbp + frm_saved_xmm7 ]
978 movdqa xmm8, [rbp + frm_saved_xmm8 ]
979 movdqa xmm9, [rbp + frm_saved_xmm9 ]
980 movdqa xmm10, [rbp + frm_saved_xmm10]
981 movdqa xmm11, [rbp + frm_saved_xmm11]
982 movdqa xmm12, [rbp + frm_saved_xmm12]
983 movdqa xmm13, [rbp + frm_saved_xmm13]
984 movdqa xmm14, [rbp + frm_saved_xmm14]
985 movdqa xmm15, [rbp + frm_saved_xmm15]
986
987 mov eax, r11d
988 %endif ; %4 != 0
989
990 lea rsp, [rbp + frm_fRFlags]
991 popf
992 leave
993 ret
994
995 ;
996 ; Error returns.
997 ;
998 %ifdef VBOX_STRICT
999.vmwrite_failed:
1000 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_FIELD
1001 jz .return_after_vmwrite_error
1002 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR
1003 jmp .return_after_vmwrite_error
1004 %endif
1005.vmxstart64_invalid_vmcs_ptr:
1006 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1007 jmp .vmstart64_error_return
1008.vmxstart64_start_failed:
1009 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_UNABLE_TO_START_VM
1010.vmstart64_error_return:
1011 RESTORE_STATE_VMX 1, %2, %3, %4
1012 mov eax, [rbp + frm_rcError]
1013 jmp .vmstart64_end
1014
1015 %ifdef VBOX_STRICT
1016 ; Precondition checks failed.
1017.precond_failure_return:
1018 POP_CALLEE_PRESERVED_REGISTERS
1019 %if cbFrame != cbBaseFrame
1020 %error Bad frame size value: cbFrame, expected cbBaseFrame
1021 %endif
1022 jmp .vmstart64_end
1023 %endif
1024
1025 %undef frm_fRFlags
1026 %undef frm_pGstCtx
1027 %undef frm_uHostXcr0
1028 %undef frm_saved_gdtr
1029 %undef frm_saved_tr
1030 %undef frm_fNoRestoreXcr0
1031 %undef frm_saved_idtr
1032 %undef frm_saved_ldtr
1033 %undef frm_rcError
1034 %undef frm_guest_rax
1035 %undef cbFrame
1036ENDPROC RT_CONCAT(hmR0VmxStartVm,%1)
1037 %ifdef ASM_FORMAT_ELF
1038size NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) NAME(RT_CONCAT(hmR0VmxStartVm,%1) %+ _EndProc) - NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1))
1039 %endif
1040
1041
1042%endmacro ; hmR0VmxStartVmTemplate
1043
1044%macro hmR0VmxStartVmSseTemplate 3
1045hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, 0 | 0 | 0 | 0 , %1
1046hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, 0 | 0 | 0 | 0 , %1
1047hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | 0 | 0 , %1
1048hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | 0 | 0 , %1
1049hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1050hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1051hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1052hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1053hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, 0 | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1054hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, 0 | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1055hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1056hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1057hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1058hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1059hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1060hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1061hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, 0 | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1062hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, 0 | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1063hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1064hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1065hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1066hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1067hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1068hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1069hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, 0 | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1070hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, 0 | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1071hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1072hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1073hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1074hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1075hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1076hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1077%endmacro
1078
1079hmR0VmxStartVmSseTemplate 0,,RT_NOTHING
1080%ifdef VBOX_WITH_KERNEL_USING_XMM
1081hmR0VmxStartVmSseTemplate 1,_SseManual,RT_NOTHING
1082hmR0VmxStartVmSseTemplate 2,_SseXSave,RT_NOTHING
1083%endif
1084
1085
1086;;
1087; hmR0SvmVmRun template
1088;
1089; @param 1 The suffix of the variation.
1090; @param 2 fLoadSaveGuestXcr0 value
1091; @param 3 The HM_WSF_IBPB_ENTRY + HM_WSF_IBPB_EXIT value.
1092; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
1093; Drivers shouldn't use AVX registers without saving+loading:
1094; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
1095; However the compiler docs have different idea:
1096; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
1097; We'll go with the former for now.
1098;
1099%macro hmR0SvmVmRunTemplate 4
1100
1101;;
1102; Prepares for and executes VMRUN (32-bit and 64-bit guests).
1103;
1104; @returns VBox status code
1105; @param pVM msc:rcx,gcc:rdi The cross context VM structure (unused).
1106; @param pVCpu msc:rdx,gcc:rsi The cross context virtual CPU structure of the calling EMT.
1107; @param HCPhysVmcb msc:r8, gcc:rdx Physical address of guest VMCB.
1108;
1109ALIGNCODE(64) ; This + immediate optimizations causes serious trouble for yasm and the SEH frames: prologue -28 bytes, must be <256
1110 ; So the SEH64_XXX stuff is currently not operational.
1111BEGINPROC RT_CONCAT(hmR0SvmVmRun,%1)
1112 %ifdef VBOX_WITH_KERNEL_USING_XMM
1113 %if %4 = 0
1114 ;
1115 ; The non-saving variant will currently check the two SSE preconditions and pick
1116 ; the right variant to continue with. Later we can see if we can't manage to
1117 ; move these decisions into hmR0SvmUpdateVmRunFunction().
1118 ;
1119 %ifdef ASM_CALL64_MSC
1120 test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1121 %else
1122 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1123 %endif
1124 jz .save_xmm_no_need
1125 %ifdef ASM_CALL64_MSC
1126 cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1127 %else
1128 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1129 %endif
1130 je RT_CONCAT3(hmR0SvmVmRun,%1,_SseManual)
1131 jmp RT_CONCAT3(hmR0SvmVmRun,%1,_SseXSave)
1132.save_xmm_no_need:
1133 %endif
1134 %endif
1135 push rbp
1136 SEH64_PUSH_xBP
1137 mov rbp, rsp
1138 SEH64_SET_FRAME_xBP 0
1139 pushf
1140 %assign cbFrame 30h
1141 %if %4 != 0
1142 %assign cbFrame cbFrame + 16 * 11 ; Reserve space for 10x 128-bit XMM registers and MXCSR (32-bit)
1143 %endif
1144 %assign cbBaseFrame cbFrame
1145 sub rsp, cbFrame - 8h ; We subtract 8 bytes for the above pushf
1146 SEH64_ALLOCATE_STACK cbFrame ; And we have CALLEE_PRESERVED_REGISTER_COUNT following it.
1147
1148 %define frm_fRFlags -008h
1149 %define frm_uHostXcr0 -018h ; 128-bit
1150 ;%define frm_fNoRestoreXcr0 -020h ; Non-zero if we should skip XCR0 restoring.
1151 %define frm_pGstCtx -028h ; Where we stash guest CPU context for use after the vmrun.
1152 %define frm_HCPhysVmcbHost -030h ; Where we stash HCPhysVmcbHost for the vmload after vmrun.
1153 %if %4 != 0
1154 %define frm_saved_xmm6 -040h
1155 %define frm_saved_xmm7 -050h
1156 %define frm_saved_xmm8 -060h
1157 %define frm_saved_xmm9 -070h
1158 %define frm_saved_xmm10 -080h
1159 %define frm_saved_xmm11 -090h
1160 %define frm_saved_xmm12 -0a0h
1161 %define frm_saved_xmm13 -0b0h
1162 %define frm_saved_xmm14 -0c0h
1163 %define frm_saved_xmm15 -0d0h
1164 %define frm_saved_mxcsr -0e0h
1165 %endif
1166
1167 ; Manual save and restore:
1168 ; - General purpose registers except RIP, RSP, RAX
1169 ;
1170 ; Trashed:
1171 ; - CR2 (we don't care)
1172 ; - LDTR (reset to 0)
1173 ; - DRx (presumably not changed at all)
1174 ; - DR7 (reset to 0x400)
1175
1176 ; Save all general purpose host registers.
1177 PUSH_CALLEE_PRESERVED_REGISTERS
1178 SEH64_END_PROLOGUE
1179 %if cbFrame != (cbBaseFrame + 8 * CALLEE_PRESERVED_REGISTER_COUNT)
1180 %error Bad cbFrame value
1181 %endif
1182
1183 ; Shuffle parameter registers so that r8=HCPhysVmcb and rsi=pVCpu. (rdx & rcx will soon be trashed.)
1184 %ifdef ASM_CALL64_GCC
1185 mov r8, rdx ; Put HCPhysVmcb in r8 like on MSC as rdx is trashed below.
1186 %else
1187 mov rsi, rdx ; Put pVCpu in rsi like on GCC as rdx is trashed below.
1188 ;mov rdi, rcx ; Put pVM in rdi like on GCC as rcx is trashed below.
1189 %endif
1190
1191 %ifdef VBOX_STRICT
1192 ;
1193 ; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
1194 ;
1195 cmp byte [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fLoadSaveGuestXcr0], %2
1196 mov eax, VERR_SVM_VMRUN_PRECOND_0
1197 jne .failure_return
1198
1199 mov eax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fWorldSwitcher]
1200 and eax, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT
1201 cmp eax, %3
1202 mov eax, VERR_SVM_VMRUN_PRECOND_1
1203 jne .failure_return
1204
1205 %ifdef VBOX_WITH_KERNEL_USING_XMM
1206 mov eax, VERR_SVM_VMRUN_PRECOND_2
1207 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1208 %if %4 = 0
1209 jnz .failure_return
1210 %else
1211 jz .failure_return
1212
1213 mov eax, VERR_SVM_VMRUN_PRECOND_3
1214 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1215 %if %4 = 1
1216 jne .failure_return
1217 %elif %4 = 2
1218 je .failure_return
1219 %else
1220 %error Invalid template parameter 4.
1221 %endif
1222 %endif
1223 %endif
1224 %endif ; VBOX_STRICT
1225
1226 %if %4 != 0
1227 ; Save the non-volatile SSE host register state.
1228 movdqa [rbp + frm_saved_xmm6 ], xmm6
1229 movdqa [rbp + frm_saved_xmm7 ], xmm7
1230 movdqa [rbp + frm_saved_xmm8 ], xmm8
1231 movdqa [rbp + frm_saved_xmm9 ], xmm9
1232 movdqa [rbp + frm_saved_xmm10], xmm10
1233 movdqa [rbp + frm_saved_xmm11], xmm11
1234 movdqa [rbp + frm_saved_xmm12], xmm12
1235 movdqa [rbp + frm_saved_xmm13], xmm13
1236 movdqa [rbp + frm_saved_xmm14], xmm14
1237 movdqa [rbp + frm_saved_xmm15], xmm15
1238 stmxcsr [rbp + frm_saved_mxcsr]
1239
1240 ; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx.
1241 lea rcx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.XState]
1242 %if %4 = 1 ; manual
1243 movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h]
1244 movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h]
1245 movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h]
1246 movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h]
1247 movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h]
1248 movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h]
1249 movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h]
1250 movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h]
1251 movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h]
1252 movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h]
1253 movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h]
1254 movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h]
1255 movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h]
1256 movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h]
1257 movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h]
1258 movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h]
1259 ldmxcsr [rcx + X86FXSTATE.MXCSR]
1260 %elif %4 = 2 ; use xrstor/xsave
1261 mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
1262 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1263 xor edx, edx
1264 xrstor [rcx]
1265 %else
1266 %error invalid template parameter 4
1267 %endif
1268 %endif
1269
1270 %if %2 != 0
1271 ; Save the host XCR0 and load the guest one if necessary.
1272 xor ecx, ecx
1273 xgetbv ; save the host XCR0 on the stack
1274 mov [rbp + frm_uHostXcr0 + 8], rdx
1275 mov [rbp + frm_uHostXcr0 ], rax
1276
1277 mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr] ; load the guest XCR0
1278 mov edx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr + 4]
1279 xor ecx, ecx ; paranoia
1280 xsetbv
1281 %endif
1282
1283 ; Save host fs, gs, sysenter msr etc.
1284 mov rax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.svm + HMR0CPUSVM.HCPhysVmcbHost]
1285 mov qword [rbp + frm_HCPhysVmcbHost], rax ; save for the vmload after vmrun
1286 lea rsi, [rsi + VMCPU.cpum.GstCtx]
1287 mov qword [rbp + frm_pGstCtx], rsi
1288 vmsave
1289
1290 %if %3 & HM_WSF_IBPB_ENTRY
1291 ; Fight spectre (trashes rax, rdx and rcx).
1292 mov ecx, MSR_IA32_PRED_CMD
1293 mov eax, MSR_IA32_PRED_CMD_F_IBPB
1294 xor edx, edx
1295 wrmsr
1296 %endif
1297
1298 ; Setup rax for VMLOAD.
1299 mov rax, r8 ; HCPhysVmcb (64 bits physical address; take low dword only)
1300
1301 ; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
1302 mov rbx, qword [rsi + CPUMCTX.ebx]
1303 mov rcx, qword [rsi + CPUMCTX.ecx]
1304 mov rdx, qword [rsi + CPUMCTX.edx]
1305 mov rdi, qword [rsi + CPUMCTX.edi]
1306 mov rbp, qword [rsi + CPUMCTX.ebp]
1307 mov r8, qword [rsi + CPUMCTX.r8]
1308 mov r9, qword [rsi + CPUMCTX.r9]
1309 mov r10, qword [rsi + CPUMCTX.r10]
1310 mov r11, qword [rsi + CPUMCTX.r11]
1311 mov r12, qword [rsi + CPUMCTX.r12]
1312 mov r13, qword [rsi + CPUMCTX.r13]
1313 mov r14, qword [rsi + CPUMCTX.r14]
1314 mov r15, qword [rsi + CPUMCTX.r15]
1315 mov rsi, qword [rsi + CPUMCTX.esi]
1316
1317 ; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1318 clgi
1319 sti
1320
1321 ; Load guest FS, GS, Sysenter MSRs etc.
1322 vmload
1323
1324 ; Run the VM.
1325 vmrun
1326
1327 ; Save guest fs, gs, sysenter msr etc.
1328 vmsave
1329
1330 ; Load host fs, gs, sysenter msr etc.
1331 mov rax, [rsp + cbFrame + frm_HCPhysVmcbHost] ; load HCPhysVmcbHost (rbp is not operational yet, thus rsp)
1332 vmload
1333
1334 ; Set the global interrupt flag again, but execute cli to make sure IF=0.
1335 cli
1336 stgi
1337
1338 ; Pop pVCpu (saved above) and save the guest GPRs (sans RSP and RAX).
1339 mov rax, [rsp + cbFrame + frm_pGstCtx] ; (rbp still not operational)
1340
1341 mov qword [rax + CPUMCTX.edx], rdx
1342 mov qword [rax + CPUMCTX.ecx], rcx
1343 mov rcx, rax
1344 rdtsc
1345 mov qword [rcx + CPUMCTX.ebp], rbp
1346 lea rbp, [rsp + cbFrame]
1347 shl rdx, 20h
1348 or rax, rdx ; TSC value in RAX
1349 mov qword [rcx + CPUMCTX.r8], r8
1350 mov r8, SPECTRE_FILLER ; SPECTRE filler in R8
1351 mov qword [rcx + CPUMCTX.r9], r9
1352 mov r9, r8
1353 mov qword [rcx + CPUMCTX.r10], r10
1354 mov r10, r8
1355 mov qword [rcx + GVMCPU.hmr0 + HMR0PERVCPU.uTscExit - VMCPU.cpum.GstCtx], rax
1356 mov qword [rcx + CPUMCTX.r11], r11
1357 mov r11, r8
1358 mov qword [rcx + CPUMCTX.edi], rdi
1359 %ifdef ASM_CALL64_MSC
1360 mov rdi, [rbp + frm_saved_rdi]
1361 %else
1362 mov rdi, r8
1363 %endif
1364 mov qword [rcx + CPUMCTX.esi], rsi
1365 %ifdef ASM_CALL64_MSC
1366 mov rsi, [rbp + frm_saved_rsi]
1367 %else
1368 mov rsi, r8
1369 %endif
1370 mov qword [rcx + CPUMCTX.ebx], rbx
1371 mov rbx, [rbp + frm_saved_rbx]
1372 mov qword [rcx + CPUMCTX.r12], r12
1373 mov r12, [rbp + frm_saved_r12]
1374 mov qword [rcx + CPUMCTX.r13], r13
1375 mov r13, [rbp + frm_saved_r13]
1376 mov qword [rcx + CPUMCTX.r14], r14
1377 mov r14, [rbp + frm_saved_r14]
1378 mov qword [rcx + CPUMCTX.r15], r15
1379 mov r15, [rbp + frm_saved_r15]
1380
1381 %if %4 != 0
1382 ; Set r8 = &pVCpu->cpum.GstCtx; for use below when saving and restoring SSE state.
1383 mov r8, rcx
1384 %endif
1385
1386 %if %3 & HM_WSF_IBPB_EXIT
1387 ; Fight spectre (trashes rax, rdx and rcx).
1388 mov ecx, MSR_IA32_PRED_CMD
1389 mov eax, MSR_IA32_PRED_CMD_F_IBPB
1390 xor edx, edx
1391 wrmsr
1392 %endif
1393
1394 %if %2 != 0
1395 ; Restore the host xcr0.
1396 xor ecx, ecx
1397 mov rdx, [rbp + frm_uHostXcr0 + 8]
1398 mov rax, [rbp + frm_uHostXcr0]
1399 xsetbv
1400 %endif
1401
1402 %if %4 != 0
1403 ; Save the guest SSE state related to non-volatile and volatile SSE registers.
1404 lea rcx, [r8 + CPUMCTX.XState]
1405 %if %4 = 1 ; manual
1406 stmxcsr [rcx + X86FXSTATE.MXCSR]
1407 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1408 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1409 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1410 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1411 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1412 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1413 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1414 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1415 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1416 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1417 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1418 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1419 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1420 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1421 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1422 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1423 %elif %4 = 2 ; use xrstor/xsave
1424 mov eax, [r8 + CPUMCTX.fXStateMask]
1425 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1426 xor edx, edx
1427 xsave [rcx]
1428 %else
1429 %error invalid template parameter 4
1430 %endif
1431
1432 ; Restore the host non-volatile SSE register state.
1433 ldmxcsr [rbp + frm_saved_mxcsr]
1434 movdqa xmm6, [rbp + frm_saved_xmm6 ]
1435 movdqa xmm7, [rbp + frm_saved_xmm7 ]
1436 movdqa xmm8, [rbp + frm_saved_xmm8 ]
1437 movdqa xmm9, [rbp + frm_saved_xmm9 ]
1438 movdqa xmm10, [rbp + frm_saved_xmm10]
1439 movdqa xmm11, [rbp + frm_saved_xmm11]
1440 movdqa xmm12, [rbp + frm_saved_xmm12]
1441 movdqa xmm13, [rbp + frm_saved_xmm13]
1442 movdqa xmm14, [rbp + frm_saved_xmm14]
1443 movdqa xmm15, [rbp + frm_saved_xmm15]
1444 %endif ; %4 != 0
1445
1446 ; Epilogue (assumes we restored volatile registers above when saving the guest GPRs).
1447 mov eax, VINF_SUCCESS
1448 add rsp, cbFrame - 8h
1449 popf
1450 leave
1451 ret
1452
1453 %ifdef VBOX_STRICT
1454 ; Precondition checks failed.
1455.failure_return:
1456 POP_CALLEE_PRESERVED_REGISTERS
1457 %if cbFrame != cbBaseFrame
1458 %error Bad frame size value: cbFrame
1459 %endif
1460 add rsp, cbFrame - 8h
1461 popf
1462 leave
1463 ret
1464 %endif
1465
1466%undef frm_uHostXcr0
1467%undef frm_fNoRestoreXcr0
1468%undef frm_pVCpu
1469%undef frm_HCPhysVmcbHost
1470%undef cbFrame
1471ENDPROC RT_CONCAT(hmR0SvmVmRun,%1)
1472
1473%endmacro ; hmR0SvmVmRunTemplate
1474
1475;
1476; Instantiate the hmR0SvmVmRun various variations.
1477;
1478hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit, 0, 0, 0
1479hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit, 1, 0, 0
1480hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit, 0, HM_WSF_IBPB_ENTRY, 0
1481hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit, 1, HM_WSF_IBPB_ENTRY, 0
1482hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit, 0, HM_WSF_IBPB_EXIT, 0
1483hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit, 1, HM_WSF_IBPB_EXIT, 0
1484hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 0
1485hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 0
1486%ifdef VBOX_WITH_KERNEL_USING_XMM
1487hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 0, 0, 1
1488hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 1, 0, 1
1489hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 0, HM_WSF_IBPB_ENTRY, 1
1490hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 1, HM_WSF_IBPB_ENTRY, 1
1491hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 0, HM_WSF_IBPB_EXIT, 1
1492hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 1, HM_WSF_IBPB_EXIT, 1
1493hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 1
1494hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 1
1495
1496hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 0, 0, 2
1497hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 1, 0, 2
1498hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 0, HM_WSF_IBPB_ENTRY, 2
1499hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 1, HM_WSF_IBPB_ENTRY, 2
1500hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 0, HM_WSF_IBPB_EXIT, 2
1501hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 1, HM_WSF_IBPB_EXIT, 2
1502hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 2
1503hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 2
1504%endif
1505
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette