VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 101521

Last change on this file since 101521 was 101521, checked in by vboxsync, 13 months ago

HMR0A.asm: The code handled Linux's read-only GDT in addition to the already existing support for PaX's KERNEXEC based variant. However, in contrast to upstream Linux's implementation, there's no writable alias mapping of the GDT available under PaX KERNEXEC. The CR0.WP toggling method must therefore be used in this case. Contribution from Mathias Krause. Thanks! bugref:8864
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 59.8 KB
Line 
1; $Id: HMR0A.asm 101521 2023-10-20 14:29:27Z vboxsync $
2;; @file
3; HM - Ring-0 VMX, SVM world-switch and helper routines.
4;
5
6;
7; Copyright (C) 2006-2023 Oracle and/or its affiliates.
8;
9; This file is part of VirtualBox base platform packages, as
10; available from https://www.virtualbox.org.
11;
12; This program is free software; you can redistribute it and/or
13; modify it under the terms of the GNU General Public License
14; as published by the Free Software Foundation, in version 3 of the
15; License.
16;
17; This program is distributed in the hope that it will be useful, but
18; WITHOUT ANY WARRANTY; without even the implied warranty of
19; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20; General Public License for more details.
21;
22; You should have received a copy of the GNU General Public License
23; along with this program; if not, see <https://www.gnu.org/licenses>.
24;
25; SPDX-License-Identifier: GPL-3.0-only
26;
27
28;*********************************************************************************************************************************
29;* Header Files *
30;*********************************************************************************************************************************
31;%define RT_ASM_WITH_SEH64 - trouble with SEH, alignment and (probably) 2nd pass optimizations.
32%define RT_ASM_WITH_SEH64_ALT ; Use asmdefs.mac hackery for manually emitting unwind info.
33%include "VBox/asmdefs.mac"
34%include "VBox/err.mac"
35%include "VBox/vmm/hm_vmx.mac"
36%include "VBox/vmm/cpum.mac"
37%include "VBox/vmm/gvm.mac"
38%include "iprt/x86.mac"
39%include "HMInternal.mac"
40
41%ifndef RT_ARCH_AMD64
42 %error AMD64 only.
43%endif
44
45
46;*********************************************************************************************************************************
47;* Defined Constants And Macros *
48;*********************************************************************************************************************************
49;; The offset of the XMM registers in X86FXSTATE.
50; Use define because I'm too lazy to convert the struct.
51%define XMM_OFF_IN_X86FXSTATE 160
52
53;; Spectre filler for 64-bit mode.
54; Choosen to be an invalid address (also with 5 level paging).
55%define SPECTRE_FILLER 0x02204204207fffff
56
57;;
58; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation.
59;
60; @note This is normally done by hmR0VmxExportHostSegmentRegs and VMXRestoreHostState,
61; so much of this is untested code.
62; @{
63%define VMX_SKIP_GDTR
64%define VMX_SKIP_TR
65%define VBOX_SKIP_RESTORE_SEG
66%ifdef RT_OS_DARWIN
67 ; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
68 ; risk loading a stale LDT value or something invalid.
69 %define HM_64_BIT_USE_NULL_SEL
70 ; Darwin (Mavericks) uses IDTR limit to store the CPU number so we need to always restore it.
71 ; See @bugref{6875}.
72 %undef VMX_SKIP_IDTR
73%else
74 %define VMX_SKIP_IDTR
75%endif
76;; @}
77
78;; @def CALLEE_PRESERVED_REGISTER_COUNT
79; Number of registers pushed by PUSH_CALLEE_PRESERVED_REGISTERS
80%ifdef ASM_CALL64_GCC
81 %define CALLEE_PRESERVED_REGISTER_COUNT 5
82%else
83 %define CALLEE_PRESERVED_REGISTER_COUNT 7
84%endif
85
86;; @def PUSH_CALLEE_PRESERVED_REGISTERS
87; Macro for pushing all GPRs we must preserve for the caller.
88%macro PUSH_CALLEE_PRESERVED_REGISTERS 0
89 push r15
90 SEH64_PUSH_GREG r15
91 %assign cbFrame cbFrame + 8
92 %assign frm_saved_r15 -cbFrame
93
94 push r14
95 SEH64_PUSH_GREG r14
96 %assign cbFrame cbFrame + 8
97 %assign frm_saved_r14 -cbFrame
98
99 push r13
100 SEH64_PUSH_GREG r13
101 %assign cbFrame cbFrame + 8
102 %assign frm_saved_r13 -cbFrame
103
104 push r12
105 SEH64_PUSH_GREG r12
106 %assign cbFrame cbFrame + 8
107 %assign frm_saved_r12 -cbFrame
108
109 push rbx
110 SEH64_PUSH_GREG rbx
111 %assign cbFrame cbFrame + 8
112 %assign frm_saved_rbx -cbFrame
113
114 %ifdef ASM_CALL64_MSC
115 push rsi
116 SEH64_PUSH_GREG rsi
117 %assign cbFrame cbFrame + 8
118 %assign frm_saved_rsi -cbFrame
119
120 push rdi
121 SEH64_PUSH_GREG rdi
122 %assign cbFrame cbFrame + 8
123 %assign frm_saved_rdi -cbFrame
124 %endif
125%endmacro
126
127;; @def POP_CALLEE_PRESERVED_REGISTERS
128; Counterpart to PUSH_CALLEE_PRESERVED_REGISTERS for use in the epilogue.
129%macro POP_CALLEE_PRESERVED_REGISTERS 0
130 %ifdef ASM_CALL64_MSC
131 pop rdi
132 %assign cbFrame cbFrame - 8
133 %undef frm_saved_rdi
134
135 pop rsi
136 %assign cbFrame cbFrame - 8
137 %undef frm_saved_rsi
138 %endif
139 pop rbx
140 %assign cbFrame cbFrame - 8
141 %undef frm_saved_rbx
142
143 pop r12
144 %assign cbFrame cbFrame - 8
145 %undef frm_saved_r12
146
147 pop r13
148 %assign cbFrame cbFrame - 8
149 %undef frm_saved_r13
150
151 pop r14
152 %assign cbFrame cbFrame - 8
153 %undef frm_saved_r14
154
155 pop r15
156 %assign cbFrame cbFrame - 8
157 %undef frm_saved_r15
158%endmacro
159
160
161;; @def PUSH_RELEVANT_SEGMENT_REGISTERS
162; Macro saving all segment registers on the stack.
163; @param 1 Full width register name.
164; @param 2 16-bit register name for \a 1.
165; @cobbers rax, rdx, rcx
166%macro PUSH_RELEVANT_SEGMENT_REGISTERS 2
167 %ifndef VBOX_SKIP_RESTORE_SEG
168 %error untested code. probably does not work any more!
169 %ifndef HM_64_BIT_USE_NULL_SEL
170 mov %2, es
171 push %1
172 mov %2, ds
173 push %1
174 %endif
175
176 ; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode,
177 ; Solaris OTOH doesn't and we must save it.
178 mov ecx, MSR_K8_FS_BASE
179 rdmsr
180 push rdx
181 push rax
182 %ifndef HM_64_BIT_USE_NULL_SEL
183 push fs
184 %endif
185
186 ; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel.
187 ; The same happens on exit.
188 mov ecx, MSR_K8_GS_BASE
189 rdmsr
190 push rdx
191 push rax
192 %ifndef HM_64_BIT_USE_NULL_SEL
193 push gs
194 %endif
195 %endif ; !VBOX_SKIP_RESTORE_SEG
196%endmacro ; PUSH_RELEVANT_SEGMENT_REGISTERS
197
198;; @def POP_RELEVANT_SEGMENT_REGISTERS
199; Macro restoring all segment registers on the stack.
200; @param 1 Full width register name.
201; @param 2 16-bit register name for \a 1.
202; @cobbers rax, rdx, rcx
203%macro POP_RELEVANT_SEGMENT_REGISTERS 2
204 %ifndef VBOX_SKIP_RESTORE_SEG
205 %error untested code. probably does not work any more!
206 ; Note: do not step through this code with a debugger!
207 %ifndef HM_64_BIT_USE_NULL_SEL
208 xor eax, eax
209 mov ds, ax
210 mov es, ax
211 mov fs, ax
212 mov gs, ax
213 %endif
214
215 %ifndef HM_64_BIT_USE_NULL_SEL
216 pop gs
217 %endif
218 pop rax
219 pop rdx
220 mov ecx, MSR_K8_GS_BASE
221 wrmsr
222
223 %ifndef HM_64_BIT_USE_NULL_SEL
224 pop fs
225 %endif
226 pop rax
227 pop rdx
228 mov ecx, MSR_K8_FS_BASE
229 wrmsr
230 ; Now it's safe to step again
231
232 %ifndef HM_64_BIT_USE_NULL_SEL
233 pop %1
234 mov ds, %2
235 pop %1
236 mov es, %2
237 %endif
238 %endif ; !VBOX_SKIP_RESTORE_SEG
239%endmacro ; POP_RELEVANT_SEGMENT_REGISTERS
240
241
242;*********************************************************************************************************************************
243;* External Symbols *
244;*********************************************************************************************************************************
245%ifdef VBOX_WITH_KERNEL_USING_XMM
246extern NAME(CPUMIsGuestFPUStateActive)
247%endif
248
249
250BEGINCODE
251
252
253;;
254; Used on platforms with poor inline assembly support to retrieve all the
255; info from the CPU and put it in the @a pRestoreHost structure.
256;
257; @returns VBox status code
258; @param pRestoreHost msc: rcx gcc: rdi Pointer to the RestoreHost struct.
259; @param fHaveFsGsBase msc: dl gcc: sil Whether we can use rdfsbase or not.
260;
261ALIGNCODE(64)
262BEGINPROC hmR0VmxExportHostSegmentRegsAsmHlp
263%ifdef ASM_CALL64_MSC
264 %define pRestoreHost rcx
265%elifdef ASM_CALL64_GCC
266 %define pRestoreHost rdi
267%else
268 %error Unknown calling convension.
269%endif
270 SEH64_END_PROLOGUE
271
272 ; Start with the FS and GS base so we can trash DL/SIL.
273%ifdef ASM_CALL64_MSC
274 or dl, dl
275%else
276 or sil, sil
277%endif
278 jz .use_rdmsr_for_fs_and_gs_base
279 rdfsbase rax
280 mov [pRestoreHost + VMXRESTOREHOST.uHostFSBase], rax
281 rdgsbase rax
282 mov [pRestoreHost + VMXRESTOREHOST.uHostGSBase], rax
283.done_fs_and_gs_base:
284
285 ; TR, GDTR and IDTR
286 str [pRestoreHost + VMXRESTOREHOST.uHostSelTR]
287 sgdt [pRestoreHost + VMXRESTOREHOST.HostGdtr]
288 sidt [pRestoreHost + VMXRESTOREHOST.HostIdtr]
289
290 ; Segment registers.
291 xor eax, eax
292 mov eax, cs
293 mov [pRestoreHost + VMXRESTOREHOST.uHostSelCS], ax
294
295 mov eax, ss
296 mov [pRestoreHost + VMXRESTOREHOST.uHostSelSS], ax
297
298 mov eax, gs
299 mov [pRestoreHost + VMXRESTOREHOST.uHostSelGS], ax
300
301 mov eax, fs
302 mov [pRestoreHost + VMXRESTOREHOST.uHostSelFS], ax
303
304 mov eax, es
305 mov [pRestoreHost + VMXRESTOREHOST.uHostSelES], ax
306
307 mov eax, ds
308 mov [pRestoreHost + VMXRESTOREHOST.uHostSelDS], ax
309
310 ret
311
312ALIGNCODE(16)
313.use_rdmsr_for_fs_and_gs_base:
314%ifdef ASM_CALL64_MSC
315 mov r8, pRestoreHost
316%endif
317
318 mov ecx, MSR_K8_FS_BASE
319 rdmsr
320 shl rdx, 32
321 or rdx, rax
322 mov [r8 + VMXRESTOREHOST.uHostFSBase], rdx
323
324 mov ecx, MSR_K8_GS_BASE
325 rdmsr
326 shl rdx, 32
327 or rdx, rax
328 mov [r8 + VMXRESTOREHOST.uHostGSBase], rdx
329
330%ifdef ASM_CALL64_MSC
331 mov pRestoreHost, r8
332%endif
333 jmp .done_fs_and_gs_base
334%undef pRestoreHost
335ENDPROC hmR0VmxExportHostSegmentRegsAsmHlp
336
337
338;;
339; Restores host-state fields.
340;
341; @returns VBox status code
342; @param f32RestoreHost msc: ecx gcc: edi RestoreHost flags.
343; @param pRestoreHost msc: rdx gcc: rsi Pointer to the RestoreHost struct.
344;
345ALIGNCODE(64)
346BEGINPROC VMXRestoreHostState
347%ifndef ASM_CALL64_GCC
348 ; Use GCC's input registers since we'll be needing both rcx and rdx further
349 ; down with the wrmsr instruction. Use the R10 and R11 register for saving
350 ; RDI and RSI since MSC preserve the two latter registers.
351 mov r10, rdi
352 mov r11, rsi
353 mov rdi, rcx
354 mov rsi, rdx
355%endif
356 SEH64_END_PROLOGUE
357
358.restore_gdtr:
359 test edi, VMX_RESTORE_HOST_GDTR
360 jz .restore_idtr
361 lgdt [rsi + VMXRESTOREHOST.HostGdtr]
362
363.restore_idtr:
364 test edi, VMX_RESTORE_HOST_IDTR
365 jz .restore_ds
366 lidt [rsi + VMXRESTOREHOST.HostIdtr]
367
368.restore_ds:
369 test edi, VMX_RESTORE_HOST_SEL_DS
370 jz .restore_es
371 mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
372 mov ds, eax
373
374.restore_es:
375 test edi, VMX_RESTORE_HOST_SEL_ES
376 jz .restore_tr
377 mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
378 mov es, eax
379
380.restore_tr:
381 test edi, VMX_RESTORE_HOST_SEL_TR
382 jz .restore_fs
383 ; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
384 mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
385 mov ax, dx
386 and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
387 test edi, VMX_RESTORE_HOST_GDT_READ_ONLY | VMX_RESTORE_HOST_GDT_NEED_WRITABLE
388 jnz .gdt_readonly_or_need_writable
389 add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
390 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
391 ltr dx
392
393.restore_fs:
394 ;
395 ; When restoring the selector values for FS and GS, we'll temporarily trash
396 ; the base address (at least the high 32-bit bits, but quite possibly the
397 ; whole base address), the wrmsr will restore it correctly. (VT-x actually
398 ; restores the base correctly when leaving guest mode, but not the selector
399 ; value, so there is little problem with interrupts being enabled prior to
400 ; this restore job.)
401 ; We'll disable ints once for both FS and GS as that's probably faster.
402 ;
403 test edi, VMX_RESTORE_HOST_SEL_FS | VMX_RESTORE_HOST_SEL_GS
404 jz .restore_success
405 pushfq
406 cli ; (see above)
407
408 test edi, VMX_RESTORE_HOST_CAN_USE_WRFSBASE_AND_WRGSBASE
409 jz .restore_fs_using_wrmsr
410
411.restore_fs_using_wrfsbase:
412 test edi, VMX_RESTORE_HOST_SEL_FS
413 jz .restore_gs_using_wrgsbase
414 mov rax, qword [rsi + VMXRESTOREHOST.uHostFSBase]
415 mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
416 mov fs, ecx
417 wrfsbase rax
418
419.restore_gs_using_wrgsbase:
420 test edi, VMX_RESTORE_HOST_SEL_GS
421 jz .restore_flags
422 mov rax, qword [rsi + VMXRESTOREHOST.uHostGSBase]
423 mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
424 mov gs, ecx
425 wrgsbase rax
426
427.restore_flags:
428 popfq
429
430.restore_success:
431 mov eax, VINF_SUCCESS
432%ifndef ASM_CALL64_GCC
433 ; Restore RDI and RSI on MSC.
434 mov rdi, r10
435 mov rsi, r11
436%endif
437 ret
438
439ALIGNCODE(8)
440.gdt_readonly_or_need_writable:
441 test edi, VMX_RESTORE_HOST_GDT_READ_ONLY
442 jz .gdt_readonly_need_writable
443.gdt_readonly:
444 mov rcx, cr0
445 mov r9, rcx
446 add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
447 and rcx, ~X86_CR0_WP
448 mov cr0, rcx
449 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
450 ltr dx
451 mov cr0, r9
452 jmp .restore_fs
453
454ALIGNCODE(8)
455.gdt_readonly_need_writable:
456 add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw
457 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
458 lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
459 ltr dx
460 lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT
461 jmp .restore_fs
462
463ALIGNCODE(8)
464.restore_fs_using_wrmsr:
465 test edi, VMX_RESTORE_HOST_SEL_FS
466 jz .restore_gs_using_wrmsr
467 mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
468 mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
469 mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
470 mov fs, ecx
471 mov ecx, MSR_K8_FS_BASE
472 wrmsr
473
474.restore_gs_using_wrmsr:
475 test edi, VMX_RESTORE_HOST_SEL_GS
476 jz .restore_flags
477 mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
478 mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
479 mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
480 mov gs, ecx
481 mov ecx, MSR_K8_GS_BASE
482 wrmsr
483 jmp .restore_flags
484ENDPROC VMXRestoreHostState
485
486
487;;
488; Clears the MDS buffers using VERW.
489ALIGNCODE(16)
490BEGINPROC hmR0MdsClear
491 SEH64_END_PROLOGUE
492 sub xSP, xCB
493 mov [xSP], ds
494 verw [xSP]
495 add xSP, xCB
496 ret
497ENDPROC hmR0MdsClear
498
499
500;;
501; Dispatches an NMI to the host.
502;
503ALIGNCODE(16)
504BEGINPROC VMXDispatchHostNmi
505 ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
506 SEH64_END_PROLOGUE
507 int 2
508 ret
509ENDPROC VMXDispatchHostNmi
510
511
512;;
513; Common restore logic for success and error paths. We duplicate this because we
514; don't want to waste writing the VINF_SUCCESS return value to the stack in the
515; regular code path.
516;
517; @param 1 Zero if regular return, non-zero if error return. Controls label emission.
518; @param 2 fLoadSaveGuestXcr0 value
519; @param 3 The (HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY) + HM_WSF_IBPB_EXIT value.
520; The entry values are either all set or not at all, as we're too lazy to flesh out all the variants.
521; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
522;
523; @note Important that this does not modify cbFrame or rsp.
524%macro RESTORE_STATE_VMX 4
525 ; Restore base and limit of the IDTR & GDTR.
526 %ifndef VMX_SKIP_IDTR
527 lidt [rsp + cbFrame + frm_saved_idtr]
528 %endif
529 %ifndef VMX_SKIP_GDTR
530 lgdt [rsp + cbFrame + frm_saved_gdtr]
531 %endif
532
533 ; Save the guest state and restore the non-volatile registers. We use rcx=pGstCtx (&pVCpu->cpum.GstCtx) here.
534 mov [rsp + cbFrame + frm_guest_rcx], rcx
535 mov rcx, [rsp + cbFrame + frm_pGstCtx]
536
537 mov qword [rcx + CPUMCTX.eax], rax
538 mov qword [rcx + CPUMCTX.edx], rdx
539 rdtsc
540 mov qword [rcx + CPUMCTX.ebp], rbp
541 lea rbp, [rsp + cbFrame] ; re-establish the frame pointer as early as possible.
542 shl rdx, 20h
543 or rax, rdx ; TSC value in RAX
544 mov rdx, [rbp + frm_guest_rcx]
545 mov qword [rcx + CPUMCTX.ecx], rdx
546 mov rdx, SPECTRE_FILLER ; FILLER in RDX
547 mov qword [rcx + GVMCPU.hmr0 + HMR0PERVCPU.uTscExit - VMCPU.cpum.GstCtx], rax
548 mov qword [rcx + CPUMCTX.r8], r8
549 mov r8, rdx
550 mov qword [rcx + CPUMCTX.r9], r9
551 mov r9, rdx
552 mov qword [rcx + CPUMCTX.r10], r10
553 mov r10, rdx
554 mov qword [rcx + CPUMCTX.r11], r11
555 mov r11, rdx
556 mov qword [rcx + CPUMCTX.esi], rsi
557 %ifdef ASM_CALL64_MSC
558 mov rsi, [rbp + frm_saved_rsi]
559 %else
560 mov rsi, rdx
561 %endif
562 mov qword [rcx + CPUMCTX.edi], rdi
563 %ifdef ASM_CALL64_MSC
564 mov rdi, [rbp + frm_saved_rdi]
565 %else
566 mov rdi, rdx
567 %endif
568 mov qword [rcx + CPUMCTX.ebx], rbx
569 mov rbx, [rbp + frm_saved_rbx]
570 mov qword [rcx + CPUMCTX.r12], r12
571 mov r12, [rbp + frm_saved_r12]
572 mov qword [rcx + CPUMCTX.r13], r13
573 mov r13, [rbp + frm_saved_r13]
574 mov qword [rcx + CPUMCTX.r14], r14
575 mov r14, [rbp + frm_saved_r14]
576 mov qword [rcx + CPUMCTX.r15], r15
577 mov r15, [rbp + frm_saved_r15]
578
579 mov rax, cr2
580 mov qword [rcx + CPUMCTX.cr2], rax
581 mov rax, rdx
582
583 %if %4 != 0
584 ; Save the context pointer in r8 for the SSE save/restore.
585 mov r8, rcx
586 %endif
587
588 %if %3 & HM_WSF_IBPB_EXIT
589 ; Fight spectre (trashes rax, rdx and rcx).
590 %if %1 = 0 ; Skip this in failure branch (=> guru)
591 mov ecx, MSR_IA32_PRED_CMD
592 mov eax, MSR_IA32_PRED_CMD_F_IBPB
593 xor edx, edx
594 wrmsr
595 %endif
596 %endif
597
598 %ifndef VMX_SKIP_TR
599 ; Restore TSS selector; must mark it as not busy before using ltr!
600 ; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p).
601 %ifndef VMX_SKIP_GDTR
602 lgdt [rbp + frm_saved_gdtr]
603 %endif
604 movzx eax, word [rbp + frm_saved_tr]
605 mov ecx, eax
606 and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
607 add rax, [rbp + frm_saved_gdtr + 2] ; eax <- GDTR.address + descriptor offset
608 and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
609 ltr cx
610 %endif
611 movzx edx, word [rbp + frm_saved_ldtr]
612 test edx, edx
613 jz %%skip_ldt_write
614 lldt dx
615%%skip_ldt_write:
616
617 %if %1 != 0
618.return_after_vmwrite_error:
619 %endif
620 ; Restore segment registers.
621 ;POP_RELEVANT_SEGMENT_REGISTERS rax, ax - currently broken.
622
623 %if %2 != 0
624 ; Restore the host XCR0.
625 xor ecx, ecx
626 mov eax, [rbp + frm_uHostXcr0]
627 mov edx, [rbp + frm_uHostXcr0 + 4]
628 xsetbv
629 %endif
630%endmacro ; RESTORE_STATE_VMX
631
632
633;;
634; hmR0VmxStartVm template
635;
636; @param 1 The suffix of the variation.
637; @param 2 fLoadSaveGuestXcr0 value
638; @param 3 The HM_WSF_IBPB_ENTRY + HM_WSF_IBPB_EXIT value.
639; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
640; Drivers shouldn't use AVX registers without saving+loading:
641; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
642; However the compiler docs have different idea:
643; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
644; We'll go with the former for now.
645;
646%macro hmR0VmxStartVmTemplate 4
647
648;;
649; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
650;
651; @returns VBox status code
652; @param pVmcsInfo msc:rcx, gcc:rdi Pointer to the VMCS info (for cached host RIP and RSP).
653; @param pVCpu msc:rdx, gcc:rsi The cross context virtual CPU structure of the calling EMT.
654; @param fResume msc:r8l, gcc:dl Whether to use vmlauch/vmresume.
655;
656ALIGNCODE(64)
657BEGINPROC RT_CONCAT(hmR0VmxStartVm,%1)
658 %ifdef VBOX_WITH_KERNEL_USING_XMM
659 %if %4 = 0
660 ;
661 ; The non-saving variant will currently check the two SSE preconditions and pick
662 ; the right variant to continue with. Later we can see if we can't manage to
663 ; move these decisions into hmR0VmxUpdateStartVmFunction().
664 ;
665 %ifdef ASM_CALL64_MSC
666 test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
667 %else
668 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
669 %endif
670 jz .save_xmm_no_need
671 %ifdef ASM_CALL64_MSC
672 cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
673 %else
674 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
675 %endif
676 je RT_CONCAT3(hmR0VmxStartVm,%1,_SseManual)
677 jmp RT_CONCAT3(hmR0VmxStartVm,%1,_SseXSave)
678.save_xmm_no_need:
679 %endif
680 %endif
681 push xBP
682 SEH64_PUSH_xBP
683 mov xBP, xSP
684 SEH64_SET_FRAME_xBP 0
685 pushf
686 cli
687
688 %define frm_fRFlags -008h
689 %define frm_pGstCtx -010h ; Where we stash guest CPU context for use after the vmrun.
690 %define frm_uHostXcr0 -020h ; 128-bit
691 %define frm_saved_gdtr -02ah ; 16+64: Only used when VMX_SKIP_GDTR isn't defined
692 %define frm_saved_tr -02ch ; 16-bit: Only used when VMX_SKIP_TR isn't defined
693 %define frm_MDS_seg -030h ; 16-bit: Temporary storage for the MDS flushing.
694 %define frm_saved_idtr -03ah ; 16+64: Only used when VMX_SKIP_IDTR isn't defined
695 %define frm_saved_ldtr -03ch ; 16-bit: always saved.
696 %define frm_rcError -040h ; 32-bit: Error status code (not used in the success path)
697 %define frm_guest_rcx -048h ; Temporary storage slot for guest RCX.
698 %if %4 = 0
699 %assign cbFrame 048h
700 %else
701 %define frm_saved_xmm6 -050h
702 %define frm_saved_xmm7 -060h
703 %define frm_saved_xmm8 -070h
704 %define frm_saved_xmm9 -080h
705 %define frm_saved_xmm10 -090h
706 %define frm_saved_xmm11 -0a0h
707 %define frm_saved_xmm12 -0b0h
708 %define frm_saved_xmm13 -0c0h
709 %define frm_saved_xmm14 -0d0h
710 %define frm_saved_xmm15 -0e0h
711 %define frm_saved_mxcsr -0f0h
712 %assign cbFrame 0f0h
713 %endif
714 %assign cbBaseFrame cbFrame
715 sub rsp, cbFrame - 8h
716 SEH64_ALLOCATE_STACK cbFrame
717
718 ; Save all general purpose host registers.
719 PUSH_CALLEE_PRESERVED_REGISTERS
720 ;PUSH_RELEVANT_SEGMENT_REGISTERS xAX, ax - currently broken
721 SEH64_END_PROLOGUE
722
723 ;
724 ; Unify the input parameter registers: r9=pVmcsInfo, rsi=pVCpu, bl=fResume, rdi=&pVCpu->cpum.GstCtx;
725 ;
726 %ifdef ASM_CALL64_GCC
727 mov r9, rdi ; pVmcsInfo
728 mov ebx, edx ; fResume
729 %else
730 mov r9, rcx ; pVmcsInfo
731 mov rsi, rdx ; pVCpu
732 mov ebx, r8d ; fResume
733 %endif
734 lea rdi, [rsi + VMCPU.cpum.GstCtx]
735 mov [rbp + frm_pGstCtx], rdi
736
737 %ifdef VBOX_STRICT
738 ;
739 ; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
740 ;
741 cmp byte [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fLoadSaveGuestXcr0], %2
742 mov eax, VERR_VMX_STARTVM_PRECOND_0
743 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
744
745 mov eax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fWorldSwitcher]
746 and eax, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT
747 cmp eax, %3
748 mov eax, VERR_VMX_STARTVM_PRECOND_1
749 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
750
751 %ifdef VBOX_WITH_KERNEL_USING_XMM
752 mov eax, VERR_VMX_STARTVM_PRECOND_2
753 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
754 %if %4 = 0
755 jnz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
756 %else
757 jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
758
759 mov eax, VERR_VMX_STARTVM_PRECOND_3
760 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
761 %if %4 = 1
762 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
763 %elif %4 = 2
764 je NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
765 %else
766 %error Invalid template parameter 4.
767 %endif
768 %endif
769 %endif
770 %endif ; VBOX_STRICT
771
772 %if %4 != 0
773 ; Save the non-volatile SSE host register state.
774 movdqa [rbp + frm_saved_xmm6 ], xmm6
775 movdqa [rbp + frm_saved_xmm7 ], xmm7
776 movdqa [rbp + frm_saved_xmm8 ], xmm8
777 movdqa [rbp + frm_saved_xmm9 ], xmm9
778 movdqa [rbp + frm_saved_xmm10], xmm10
779 movdqa [rbp + frm_saved_xmm11], xmm11
780 movdqa [rbp + frm_saved_xmm12], xmm12
781 movdqa [rbp + frm_saved_xmm13], xmm13
782 movdqa [rbp + frm_saved_xmm14], xmm14
783 movdqa [rbp + frm_saved_xmm15], xmm15
784 stmxcsr [rbp + frm_saved_mxcsr]
785
786 ; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx.
787 lea rcx, [rdi + CPUMCTX.XState]
788 %if %4 = 1 ; manual
789 movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h]
790 movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h]
791 movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h]
792 movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h]
793 movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h]
794 movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h]
795 movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h]
796 movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h]
797 movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h]
798 movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h]
799 movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h]
800 movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h]
801 movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h]
802 movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h]
803 movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h]
804 movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h]
805 ldmxcsr [rcx + X86FXSTATE.MXCSR]
806 %elif %4 = 2 ; use xrstor/xsave
807 mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
808 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
809 xor edx, edx
810 xrstor [rcx]
811 %else
812 %error invalid template parameter 4
813 %endif
814 %endif
815
816 %if %2 != 0
817 ; Save the host XCR0 and load the guest one if necessary.
818 ; Note! Trashes rax, rdx and rcx.
819 xor ecx, ecx
820 xgetbv ; save the host one on the stack
821 mov [rbp + frm_uHostXcr0], eax
822 mov [rbp + frm_uHostXcr0 + 4], edx
823
824 mov eax, [rdi + CPUMCTX.aXcr] ; load the guest one
825 mov edx, [rdi + CPUMCTX.aXcr + 4]
826 xor ecx, ecx ; paranoia; indicate that we must restore XCR0 (popped into ecx, thus 0)
827 xsetbv
828 %endif
829
830 ; Save host LDTR.
831 sldt word [rbp + frm_saved_ldtr]
832
833 %ifndef VMX_SKIP_TR
834 ; The host TR limit is reset to 0x67; save & restore it manually.
835 str word [rbp + frm_saved_tr]
836 %endif
837
838 %ifndef VMX_SKIP_GDTR
839 ; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
840 sgdt [rbp + frm_saved_gdtr]
841 %endif
842 %ifndef VMX_SKIP_IDTR
843 sidt [rbp + frm_saved_idtr]
844 %endif
845
846 ; Load CR2 if necessary (expensive as writing CR2 is a synchronizing instruction - (bird: still expensive on 10980xe)).
847 mov rcx, qword [rdi + CPUMCTX.cr2]
848 mov rdx, cr2
849 cmp rcx, rdx
850 je .skip_cr2_write
851 mov cr2, rcx
852.skip_cr2_write:
853
854 ; Set the vmlaunch/vmresume "return" host RIP and RSP values if they've changed (unlikly).
855 ; The vmwrite isn't quite for free (on an 10980xe at least), thus we check if anything changed
856 ; before writing here.
857 lea rcx, [NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) wrt rip]
858 cmp rcx, [r9 + VMXVMCSINFO.uHostRip]
859 jne .write_host_rip
860.wrote_host_rip:
861 cmp rsp, [r9 + VMXVMCSINFO.uHostRsp]
862 jne .write_host_rsp
863.wrote_host_rsp:
864
865 ;
866 ; Fight spectre and similar. Trashes rax, rcx, and rdx.
867 ;
868 %if %3 & (HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY) ; The eax:edx value is the same for the first two.
869 AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
870 mov eax, MSR_IA32_PRED_CMD_F_IBPB
871 xor edx, edx
872 %endif
873 %if %3 & HM_WSF_IBPB_ENTRY ; Indirect branch barrier.
874 mov ecx, MSR_IA32_PRED_CMD
875 wrmsr
876 %endif
877 %if %3 & HM_WSF_L1D_ENTRY ; Level 1 data cache flush.
878 mov ecx, MSR_IA32_FLUSH_CMD
879 wrmsr
880 %elif %3 & HM_WSF_MDS_ENTRY ; MDS flushing is included in L1D_FLUSH
881 mov word [rbp + frm_MDS_seg], ds
882 verw word [rbp + frm_MDS_seg]
883 %endif
884
885 ; Resume or start VM?
886 cmp bl, 0 ; fResume
887
888 ; Load guest general purpose registers.
889 mov rax, qword [rdi + CPUMCTX.eax]
890 mov rbx, qword [rdi + CPUMCTX.ebx]
891 mov rcx, qword [rdi + CPUMCTX.ecx]
892 mov rdx, qword [rdi + CPUMCTX.edx]
893 mov rbp, qword [rdi + CPUMCTX.ebp]
894 mov rsi, qword [rdi + CPUMCTX.esi]
895 mov r8, qword [rdi + CPUMCTX.r8]
896 mov r9, qword [rdi + CPUMCTX.r9]
897 mov r10, qword [rdi + CPUMCTX.r10]
898 mov r11, qword [rdi + CPUMCTX.r11]
899 mov r12, qword [rdi + CPUMCTX.r12]
900 mov r13, qword [rdi + CPUMCTX.r13]
901 mov r14, qword [rdi + CPUMCTX.r14]
902 mov r15, qword [rdi + CPUMCTX.r15]
903 mov rdi, qword [rdi + CPUMCTX.edi]
904
905 je .vmlaunch64_launch
906
907 vmresume
908 jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr)
909 jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed)
910 jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmresume detected a failure
911
912.vmlaunch64_launch:
913 vmlaunch
914 jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr)
915 jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed)
916 jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmlaunch detected a failure
917
918
919; Put these two outside the normal code path as they should rarely change.
920ALIGNCODE(8)
921.write_host_rip:
922 %ifdef VBOX_WITH_STATISTICS
923 inc qword [rsi + VMCPU.hm + HMCPU.StatVmxWriteHostRip]
924 %endif
925 mov [r9 + VMXVMCSINFO.uHostRip], rcx
926 mov eax, VMX_VMCS_HOST_RIP ;; @todo It is only strictly necessary to write VMX_VMCS_HOST_RIP when
927 vmwrite rax, rcx ;; the VMXVMCSINFO::pfnStartVM function changes (eventually
928 %ifdef VBOX_STRICT ;; take the Windows/SSE stuff into account then)...
929 jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed)
930 %endif
931 jmp .wrote_host_rip
932
933ALIGNCODE(8)
934.write_host_rsp:
935 %ifdef VBOX_WITH_STATISTICS
936 inc qword [rsi + VMCPU.hm + HMCPU.StatVmxWriteHostRsp]
937 %endif
938 mov [r9 + VMXVMCSINFO.uHostRsp], rsp
939 mov eax, VMX_VMCS_HOST_RSP
940 vmwrite rax, rsp
941 %ifdef VBOX_STRICT
942 jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed)
943 %endif
944 jmp .wrote_host_rsp
945
946ALIGNCODE(64)
947GLOBALNAME RT_CONCAT(hmR0VmxStartVmHostRIP,%1)
948 RESTORE_STATE_VMX 0, %2, %3, %4
949 mov eax, VINF_SUCCESS
950
951.vmstart64_end:
952 %if %4 != 0
953 mov r11d, eax ; save the return code.
954
955 ; Save the guest SSE state related to non-volatile and volatile SSE registers.
956 lea rcx, [r8 + CPUMCTX.XState]
957 %if %4 = 1 ; manual
958 stmxcsr [rcx + X86FXSTATE.MXCSR]
959 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
960 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
961 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
962 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
963 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
964 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
965 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
966 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
967 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
968 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
969 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
970 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
971 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
972 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
973 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
974 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
975 %elif %4 = 2 ; use xrstor/xsave
976 mov eax, [r8 + CPUMCTX.fXStateMask]
977 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
978 xor edx, edx
979 xsave [rcx]
980 %else
981 %error invalid template parameter 4
982 %endif
983
984 ; Restore the host non-volatile SSE register state.
985 ldmxcsr [rbp + frm_saved_mxcsr]
986 movdqa xmm6, [rbp + frm_saved_xmm6 ]
987 movdqa xmm7, [rbp + frm_saved_xmm7 ]
988 movdqa xmm8, [rbp + frm_saved_xmm8 ]
989 movdqa xmm9, [rbp + frm_saved_xmm9 ]
990 movdqa xmm10, [rbp + frm_saved_xmm10]
991 movdqa xmm11, [rbp + frm_saved_xmm11]
992 movdqa xmm12, [rbp + frm_saved_xmm12]
993 movdqa xmm13, [rbp + frm_saved_xmm13]
994 movdqa xmm14, [rbp + frm_saved_xmm14]
995 movdqa xmm15, [rbp + frm_saved_xmm15]
996
997 mov eax, r11d
998 %endif ; %4 != 0
999
1000 lea rsp, [rbp + frm_fRFlags]
1001 popf
1002 leave
1003 ret
1004
1005 ;
1006 ; Error returns.
1007 ;
1008 %ifdef VBOX_STRICT
1009.vmwrite_failed:
1010 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_FIELD
1011 jz .return_after_vmwrite_error
1012 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR
1013 jmp .return_after_vmwrite_error
1014 %endif
1015.vmxstart64_invalid_vmcs_ptr:
1016 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1017 jmp .vmstart64_error_return
1018.vmxstart64_start_failed:
1019 mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_UNABLE_TO_START_VM
1020.vmstart64_error_return:
1021 RESTORE_STATE_VMX 1, %2, %3, %4
1022 mov eax, [rbp + frm_rcError]
1023 jmp .vmstart64_end
1024
1025 %ifdef VBOX_STRICT
1026 ; Precondition checks failed.
1027.precond_failure_return:
1028 POP_CALLEE_PRESERVED_REGISTERS
1029 %if cbFrame != cbBaseFrame
1030 %error Bad frame size value: cbFrame, expected cbBaseFrame
1031 %endif
1032 jmp .vmstart64_end
1033 %endif
1034
1035 %undef frm_fRFlags
1036 %undef frm_pGstCtx
1037 %undef frm_uHostXcr0
1038 %undef frm_saved_gdtr
1039 %undef frm_saved_tr
1040 %undef frm_fNoRestoreXcr0
1041 %undef frm_saved_idtr
1042 %undef frm_saved_ldtr
1043 %undef frm_rcError
1044 %undef frm_guest_rax
1045 %undef cbFrame
1046ENDPROC RT_CONCAT(hmR0VmxStartVm,%1)
1047 %ifdef ASM_FORMAT_ELF
1048size NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) NAME(RT_CONCAT(hmR0VmxStartVm,%1) %+ _EndProc) - NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1))
1049 %endif
1050
1051
1052%endmacro ; hmR0VmxStartVmTemplate
1053
1054%macro hmR0VmxStartVmSseTemplate 2
1055hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, 0 | 0 | 0 | 0 , %1
1056hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, 0 | 0 | 0 | 0 , %1
1057hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | 0 | 0 , %1
1058hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | 0 | 0 , %1
1059hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1060hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1061hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1062hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | 0 , %1
1063hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, 0 | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1064hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, 0 | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1065hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1066hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | 0 , %1
1067hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1068hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1069hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1070hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1
1071hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, 0 | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1072hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, 0 | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1073hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1074hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | 0 | HM_WSF_IBPB_EXIT, %1
1075hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1076hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1077hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1078hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1
1079hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, 0 | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1080hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, 0 | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1081hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1082hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1083hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1084hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1085hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1086hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1
1087%endmacro
1088
1089hmR0VmxStartVmSseTemplate 0,,
1090%ifdef VBOX_WITH_KERNEL_USING_XMM
1091hmR0VmxStartVmSseTemplate 1,_SseManual
1092hmR0VmxStartVmSseTemplate 2,_SseXSave
1093%endif
1094
1095
1096;;
1097; hmR0SvmVmRun template
1098;
1099; @param 1 The suffix of the variation.
1100; @param 2 fLoadSaveGuestXcr0 value
1101; @param 3 The HM_WSF_IBPB_ENTRY + HM_WSF_IBPB_EXIT value.
1102; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
1103; Drivers shouldn't use AVX registers without saving+loading:
1104; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
1105; However the compiler docs have different idea:
1106; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
1107; We'll go with the former for now.
1108;
1109%macro hmR0SvmVmRunTemplate 4
1110
1111;;
1112; Prepares for and executes VMRUN (32-bit and 64-bit guests).
1113;
1114; @returns VBox status code
1115; @param pVM msc:rcx,gcc:rdi The cross context VM structure (unused).
1116; @param pVCpu msc:rdx,gcc:rsi The cross context virtual CPU structure of the calling EMT.
1117; @param HCPhysVmcb msc:r8, gcc:rdx Physical address of guest VMCB.
1118;
1119ALIGNCODE(64) ; This + immediate optimizations causes serious trouble for yasm and the SEH frames: prologue -28 bytes, must be <256
1120 ; So the SEH64_XXX stuff is currently not operational.
1121BEGINPROC RT_CONCAT(hmR0SvmVmRun,%1)
1122 %ifdef VBOX_WITH_KERNEL_USING_XMM
1123 %if %4 = 0
1124 ;
1125 ; The non-saving variant will currently check the two SSE preconditions and pick
1126 ; the right variant to continue with. Later we can see if we can't manage to
1127 ; move these decisions into hmR0SvmUpdateVmRunFunction().
1128 ;
1129 %ifdef ASM_CALL64_MSC
1130 test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1131 %else
1132 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1133 %endif
1134 jz .save_xmm_no_need
1135 %ifdef ASM_CALL64_MSC
1136 cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1137 %else
1138 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1139 %endif
1140 je RT_CONCAT3(hmR0SvmVmRun,%1,_SseManual)
1141 jmp RT_CONCAT3(hmR0SvmVmRun,%1,_SseXSave)
1142.save_xmm_no_need:
1143 %endif
1144 %endif
1145 push rbp
1146 SEH64_PUSH_xBP
1147 mov rbp, rsp
1148 SEH64_SET_FRAME_xBP 0
1149 pushf
1150 %assign cbFrame 30h
1151 %if %4 != 0
1152 %assign cbFrame cbFrame + 16 * 11 ; Reserve space for 10x 128-bit XMM registers and MXCSR (32-bit)
1153 %endif
1154 %assign cbBaseFrame cbFrame
1155 sub rsp, cbFrame - 8h ; We subtract 8 bytes for the above pushf
1156 SEH64_ALLOCATE_STACK cbFrame ; And we have CALLEE_PRESERVED_REGISTER_COUNT following it.
1157
1158 %define frm_fRFlags -008h
1159 %define frm_uHostXcr0 -018h ; 128-bit
1160 ;%define frm_fNoRestoreXcr0 -020h ; Non-zero if we should skip XCR0 restoring.
1161 %define frm_pGstCtx -028h ; Where we stash guest CPU context for use after the vmrun.
1162 %define frm_HCPhysVmcbHost -030h ; Where we stash HCPhysVmcbHost for the vmload after vmrun.
1163 %if %4 != 0
1164 %define frm_saved_xmm6 -040h
1165 %define frm_saved_xmm7 -050h
1166 %define frm_saved_xmm8 -060h
1167 %define frm_saved_xmm9 -070h
1168 %define frm_saved_xmm10 -080h
1169 %define frm_saved_xmm11 -090h
1170 %define frm_saved_xmm12 -0a0h
1171 %define frm_saved_xmm13 -0b0h
1172 %define frm_saved_xmm14 -0c0h
1173 %define frm_saved_xmm15 -0d0h
1174 %define frm_saved_mxcsr -0e0h
1175 %endif
1176
1177 ; Manual save and restore:
1178 ; - General purpose registers except RIP, RSP, RAX
1179 ;
1180 ; Trashed:
1181 ; - CR2 (we don't care)
1182 ; - LDTR (reset to 0)
1183 ; - DRx (presumably not changed at all)
1184 ; - DR7 (reset to 0x400)
1185
1186 ; Save all general purpose host registers.
1187 PUSH_CALLEE_PRESERVED_REGISTERS
1188 SEH64_END_PROLOGUE
1189 %if cbFrame != (cbBaseFrame + 8 * CALLEE_PRESERVED_REGISTER_COUNT)
1190 %error Bad cbFrame value
1191 %endif
1192
1193 ; Shuffle parameter registers so that r8=HCPhysVmcb and rsi=pVCpu. (rdx & rcx will soon be trashed.)
1194 %ifdef ASM_CALL64_GCC
1195 mov r8, rdx ; Put HCPhysVmcb in r8 like on MSC as rdx is trashed below.
1196 %else
1197 mov rsi, rdx ; Put pVCpu in rsi like on GCC as rdx is trashed below.
1198 ;mov rdi, rcx ; Put pVM in rdi like on GCC as rcx is trashed below.
1199 %endif
1200
1201 %ifdef VBOX_STRICT
1202 ;
1203 ; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
1204 ;
1205 cmp byte [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fLoadSaveGuestXcr0], %2
1206 mov eax, VERR_SVM_VMRUN_PRECOND_0
1207 jne .failure_return
1208
1209 mov eax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fWorldSwitcher]
1210 and eax, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT
1211 cmp eax, %3
1212 mov eax, VERR_SVM_VMRUN_PRECOND_1
1213 jne .failure_return
1214
1215 %ifdef VBOX_WITH_KERNEL_USING_XMM
1216 mov eax, VERR_SVM_VMRUN_PRECOND_2
1217 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1218 %if %4 = 0
1219 jnz .failure_return
1220 %else
1221 jz .failure_return
1222
1223 mov eax, VERR_SVM_VMRUN_PRECOND_3
1224 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1225 %if %4 = 1
1226 jne .failure_return
1227 %elif %4 = 2
1228 je .failure_return
1229 %else
1230 %error Invalid template parameter 4.
1231 %endif
1232 %endif
1233 %endif
1234 %endif ; VBOX_STRICT
1235
1236 %if %4 != 0
1237 ; Save the non-volatile SSE host register state.
1238 movdqa [rbp + frm_saved_xmm6 ], xmm6
1239 movdqa [rbp + frm_saved_xmm7 ], xmm7
1240 movdqa [rbp + frm_saved_xmm8 ], xmm8
1241 movdqa [rbp + frm_saved_xmm9 ], xmm9
1242 movdqa [rbp + frm_saved_xmm10], xmm10
1243 movdqa [rbp + frm_saved_xmm11], xmm11
1244 movdqa [rbp + frm_saved_xmm12], xmm12
1245 movdqa [rbp + frm_saved_xmm13], xmm13
1246 movdqa [rbp + frm_saved_xmm14], xmm14
1247 movdqa [rbp + frm_saved_xmm15], xmm15
1248 stmxcsr [rbp + frm_saved_mxcsr]
1249
1250 ; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx.
1251 lea rcx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.XState]
1252 %if %4 = 1 ; manual
1253 movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h]
1254 movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h]
1255 movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h]
1256 movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h]
1257 movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h]
1258 movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h]
1259 movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h]
1260 movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h]
1261 movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h]
1262 movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h]
1263 movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h]
1264 movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h]
1265 movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h]
1266 movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h]
1267 movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h]
1268 movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h]
1269 ldmxcsr [rcx + X86FXSTATE.MXCSR]
1270 %elif %4 = 2 ; use xrstor/xsave
1271 mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
1272 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1273 xor edx, edx
1274 xrstor [rcx]
1275 %else
1276 %error invalid template parameter 4
1277 %endif
1278 %endif
1279
1280 %if %2 != 0
1281 ; Save the host XCR0 and load the guest one if necessary.
1282 xor ecx, ecx
1283 xgetbv ; save the host XCR0 on the stack
1284 mov [rbp + frm_uHostXcr0 + 8], rdx
1285 mov [rbp + frm_uHostXcr0 ], rax
1286
1287 mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr] ; load the guest XCR0
1288 mov edx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr + 4]
1289 xor ecx, ecx ; paranoia
1290 xsetbv
1291 %endif
1292
1293 ; Save host fs, gs, sysenter msr etc.
1294 mov rax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.svm + HMR0CPUSVM.HCPhysVmcbHost]
1295 mov qword [rbp + frm_HCPhysVmcbHost], rax ; save for the vmload after vmrun
1296 lea rsi, [rsi + VMCPU.cpum.GstCtx]
1297 mov qword [rbp + frm_pGstCtx], rsi
1298 vmsave
1299
1300 %if %3 & HM_WSF_IBPB_ENTRY
1301 ; Fight spectre (trashes rax, rdx and rcx).
1302 mov ecx, MSR_IA32_PRED_CMD
1303 mov eax, MSR_IA32_PRED_CMD_F_IBPB
1304 xor edx, edx
1305 wrmsr
1306 %endif
1307
1308 ; Setup rax for VMLOAD.
1309 mov rax, r8 ; HCPhysVmcb (64 bits physical address; take low dword only)
1310
1311 ; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
1312 mov rbx, qword [rsi + CPUMCTX.ebx]
1313 mov rcx, qword [rsi + CPUMCTX.ecx]
1314 mov rdx, qword [rsi + CPUMCTX.edx]
1315 mov rdi, qword [rsi + CPUMCTX.edi]
1316 mov rbp, qword [rsi + CPUMCTX.ebp]
1317 mov r8, qword [rsi + CPUMCTX.r8]
1318 mov r9, qword [rsi + CPUMCTX.r9]
1319 mov r10, qword [rsi + CPUMCTX.r10]
1320 mov r11, qword [rsi + CPUMCTX.r11]
1321 mov r12, qword [rsi + CPUMCTX.r12]
1322 mov r13, qword [rsi + CPUMCTX.r13]
1323 mov r14, qword [rsi + CPUMCTX.r14]
1324 mov r15, qword [rsi + CPUMCTX.r15]
1325 mov rsi, qword [rsi + CPUMCTX.esi]
1326
1327 ; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1328 clgi
1329 sti
1330
1331 ; Load guest FS, GS, Sysenter MSRs etc.
1332 vmload
1333
1334 ; Run the VM.
1335 vmrun
1336
1337 ; Save guest fs, gs, sysenter msr etc.
1338 vmsave
1339
1340 ; Load host fs, gs, sysenter msr etc.
1341 mov rax, [rsp + cbFrame + frm_HCPhysVmcbHost] ; load HCPhysVmcbHost (rbp is not operational yet, thus rsp)
1342 vmload
1343
1344 ; Set the global interrupt flag again, but execute cli to make sure IF=0.
1345 cli
1346 stgi
1347
1348 ; Pop pVCpu (saved above) and save the guest GPRs (sans RSP and RAX).
1349 mov rax, [rsp + cbFrame + frm_pGstCtx] ; (rbp still not operational)
1350
1351 mov qword [rax + CPUMCTX.edx], rdx
1352 mov qword [rax + CPUMCTX.ecx], rcx
1353 mov rcx, rax
1354 rdtsc
1355 mov qword [rcx + CPUMCTX.ebp], rbp
1356 lea rbp, [rsp + cbFrame]
1357 shl rdx, 20h
1358 or rax, rdx ; TSC value in RAX
1359 mov qword [rcx + CPUMCTX.r8], r8
1360 mov r8, SPECTRE_FILLER ; SPECTRE filler in R8
1361 mov qword [rcx + CPUMCTX.r9], r9
1362 mov r9, r8
1363 mov qword [rcx + CPUMCTX.r10], r10
1364 mov r10, r8
1365 mov qword [rcx + GVMCPU.hmr0 + HMR0PERVCPU.uTscExit - VMCPU.cpum.GstCtx], rax
1366 mov qword [rcx + CPUMCTX.r11], r11
1367 mov r11, r8
1368 mov qword [rcx + CPUMCTX.edi], rdi
1369 %ifdef ASM_CALL64_MSC
1370 mov rdi, [rbp + frm_saved_rdi]
1371 %else
1372 mov rdi, r8
1373 %endif
1374 mov qword [rcx + CPUMCTX.esi], rsi
1375 %ifdef ASM_CALL64_MSC
1376 mov rsi, [rbp + frm_saved_rsi]
1377 %else
1378 mov rsi, r8
1379 %endif
1380 mov qword [rcx + CPUMCTX.ebx], rbx
1381 mov rbx, [rbp + frm_saved_rbx]
1382 mov qword [rcx + CPUMCTX.r12], r12
1383 mov r12, [rbp + frm_saved_r12]
1384 mov qword [rcx + CPUMCTX.r13], r13
1385 mov r13, [rbp + frm_saved_r13]
1386 mov qword [rcx + CPUMCTX.r14], r14
1387 mov r14, [rbp + frm_saved_r14]
1388 mov qword [rcx + CPUMCTX.r15], r15
1389 mov r15, [rbp + frm_saved_r15]
1390
1391 %if %4 != 0
1392 ; Set r8 = &pVCpu->cpum.GstCtx; for use below when saving and restoring SSE state.
1393 mov r8, rcx
1394 %endif
1395
1396 %if %3 & HM_WSF_IBPB_EXIT
1397 ; Fight spectre (trashes rax, rdx and rcx).
1398 mov ecx, MSR_IA32_PRED_CMD
1399 mov eax, MSR_IA32_PRED_CMD_F_IBPB
1400 xor edx, edx
1401 wrmsr
1402 %endif
1403
1404 %if %2 != 0
1405 ; Restore the host xcr0.
1406 xor ecx, ecx
1407 mov rdx, [rbp + frm_uHostXcr0 + 8]
1408 mov rax, [rbp + frm_uHostXcr0]
1409 xsetbv
1410 %endif
1411
1412 %if %4 != 0
1413 ; Save the guest SSE state related to non-volatile and volatile SSE registers.
1414 lea rcx, [r8 + CPUMCTX.XState]
1415 %if %4 = 1 ; manual
1416 stmxcsr [rcx + X86FXSTATE.MXCSR]
1417 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1418 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1419 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1420 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1421 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1422 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1423 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1424 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1425 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1426 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1427 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1428 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1429 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1430 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1431 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1432 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1433 %elif %4 = 2 ; use xrstor/xsave
1434 mov eax, [r8 + CPUMCTX.fXStateMask]
1435 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1436 xor edx, edx
1437 xsave [rcx]
1438 %else
1439 %error invalid template parameter 4
1440 %endif
1441
1442 ; Restore the host non-volatile SSE register state.
1443 ldmxcsr [rbp + frm_saved_mxcsr]
1444 movdqa xmm6, [rbp + frm_saved_xmm6 ]
1445 movdqa xmm7, [rbp + frm_saved_xmm7 ]
1446 movdqa xmm8, [rbp + frm_saved_xmm8 ]
1447 movdqa xmm9, [rbp + frm_saved_xmm9 ]
1448 movdqa xmm10, [rbp + frm_saved_xmm10]
1449 movdqa xmm11, [rbp + frm_saved_xmm11]
1450 movdqa xmm12, [rbp + frm_saved_xmm12]
1451 movdqa xmm13, [rbp + frm_saved_xmm13]
1452 movdqa xmm14, [rbp + frm_saved_xmm14]
1453 movdqa xmm15, [rbp + frm_saved_xmm15]
1454 %endif ; %4 != 0
1455
1456 ; Epilogue (assumes we restored volatile registers above when saving the guest GPRs).
1457 mov eax, VINF_SUCCESS
1458 add rsp, cbFrame - 8h
1459 popf
1460 leave
1461 ret
1462
1463 %ifdef VBOX_STRICT
1464 ; Precondition checks failed.
1465.failure_return:
1466 POP_CALLEE_PRESERVED_REGISTERS
1467 %if cbFrame != cbBaseFrame
1468 %error Bad frame size value: cbFrame
1469 %endif
1470 add rsp, cbFrame - 8h
1471 popf
1472 leave
1473 ret
1474 %endif
1475
1476%undef frm_uHostXcr0
1477%undef frm_fNoRestoreXcr0
1478%undef frm_pVCpu
1479%undef frm_HCPhysVmcbHost
1480%undef cbFrame
1481ENDPROC RT_CONCAT(hmR0SvmVmRun,%1)
1482
1483%endmacro ; hmR0SvmVmRunTemplate
1484
1485;
1486; Instantiate the hmR0SvmVmRun various variations.
1487;
1488hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit, 0, 0, 0
1489hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit, 1, 0, 0
1490hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit, 0, HM_WSF_IBPB_ENTRY, 0
1491hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit, 1, HM_WSF_IBPB_ENTRY, 0
1492hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit, 0, HM_WSF_IBPB_EXIT, 0
1493hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit, 1, HM_WSF_IBPB_EXIT, 0
1494hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 0
1495hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 0
1496%ifdef VBOX_WITH_KERNEL_USING_XMM
1497hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 0, 0, 1
1498hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 1, 0, 1
1499hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 0, HM_WSF_IBPB_ENTRY, 1
1500hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 1, HM_WSF_IBPB_ENTRY, 1
1501hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 0, HM_WSF_IBPB_EXIT, 1
1502hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 1, HM_WSF_IBPB_EXIT, 1
1503hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 1
1504hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 1
1505
1506hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 0, 0, 2
1507hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 1, 0, 2
1508hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 0, HM_WSF_IBPB_ENTRY, 2
1509hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 1, HM_WSF_IBPB_ENTRY, 2
1510hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 0, HM_WSF_IBPB_EXIT, 2
1511hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 1, HM_WSF_IBPB_EXIT, 2
1512hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 2
1513hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 2
1514%endif
1515
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette