NEMR3Native-darwin.cpp@ 98045

Last change on this file since 98045 was 97723, checked in by vboxsync, 2 years ago
VMM/NEM: No else after return.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 179.3 KB

Line
1	/* $Id: NEMR3Native-darwin.cpp 97723 2022-12-01 11:06:38Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 macOS backend using Hypervisor.framework.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	*/
9
10	/*
11	* Copyright (C) 2020-2022 Oracle and/or its affiliates.
12	*
13	* This file is part of VirtualBox base platform packages, as
14	* available from https://www.virtualbox.org.
15	*
16	* This program is free software; you can redistribute it and/or
17	* modify it under the terms of the GNU General Public License
18	* as published by the Free Software Foundation, in version 3 of the
19	* License.
20	*
21	* This program is distributed in the hope that it will be useful, but
22	* WITHOUT ANY WARRANTY; without even the implied warranty of
23	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	* General Public License for more details.
25	*
26	* You should have received a copy of the GNU General Public License
27	* along with this program; if not, see <https://www.gnu.org/licenses>.
28	*
29	* SPDX-License-Identifier: GPL-3.0-only
30	*/
31
32
33	/*********************************************************************************************************************************
34	* Header Files *
35	*********************************************************************************************************************************/
36	#define LOG_GROUP LOG_GROUP_NEM
37	#define VMCPU_INCL_CPUM_GST_CTX
38	#define CPUM_WITH_NONCONST_HOST_FEATURES /* required for initializing parts of the g_CpumHostFeatures structure here. */
39	#include <VBox/vmm/nem.h>
40	#include <VBox/vmm/iem.h>
41	#include <VBox/vmm/em.h>
42	#include <VBox/vmm/apic.h>
43	#include <VBox/vmm/pdm.h>
44	#include <VBox/vmm/hm.h>
45	#include <VBox/vmm/hm_vmx.h>
46	#include <VBox/vmm/dbgftrace.h>
47	#include <VBox/vmm/gcm.h>
48	#include "VMXInternal.h"
49	#include "NEMInternal.h"
50	#include <VBox/vmm/vmcc.h>
51	#include "dtrace/VBoxVMM.h"
52
53	#include <iprt/asm.h>
54	#include <iprt/ldr.h>
55	#include <iprt/mem.h>
56	#include <iprt/path.h>
57	#include <iprt/string.h>
58	#include <iprt/system.h>
59	#include <iprt/utf16.h>
60
61	#include <mach/mach_time.h>
62	#include <mach/kern_return.h>
63
64
65	/*********************************************************************************************************************************
66	* Defined Constants And Macros *
67	*********************************************************************************************************************************/
68	/* No nested hwvirt (for now). */
69	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
70	# undef VBOX_WITH_NESTED_HWVIRT_VMX
71	#endif
72
73
74	/** @name HV return codes.
75	* @{ */
76	/** Operation was successful. */
77	#define HV_SUCCESS 0
78	/** An error occurred during operation. */
79	#define HV_ERROR 0xfae94001
80	/** The operation could not be completed right now, try again. */
81	#define HV_BUSY 0xfae94002
82	/** One of the parameters passed wis invalid. */
83	#define HV_BAD_ARGUMENT 0xfae94003
84	/** Not enough resources left to fulfill the operation. */
85	#define HV_NO_RESOURCES 0xfae94005
86	/** The device could not be found. */
87	#define HV_NO_DEVICE 0xfae94006
88	/** The operation is not supportd on this platform with this configuration. */
89	#define HV_UNSUPPORTED 0xfae94007
90	/** @} */
91
92
93	/** @name HV memory protection flags.
94	* @{ */
95	/** Memory is readable. */
96	#define HV_MEMORY_READ RT_BIT_64(0)
97	/** Memory is writeable. */
98	#define HV_MEMORY_WRITE RT_BIT_64(1)
99	/** Memory is executable. */
100	#define HV_MEMORY_EXEC RT_BIT_64(2)
101	/** @} */
102
103
104	/** @name HV shadow VMCS protection flags.
105	* @{ */
106	/** Shadow VMCS field is not accessible. */
107	#define HV_SHADOW_VMCS_NONE 0
108	/** Shadow VMCS fild is readable. */
109	#define HV_SHADOW_VMCS_READ RT_BIT_64(0)
110	/** Shadow VMCS field is writeable. */
111	#define HV_SHADOW_VMCS_WRITE RT_BIT_64(1)
112	/** @} */
113
114
115	/** Default VM creation flags. */
116	#define HV_VM_DEFAULT 0
117	/** Default guest address space creation flags. */
118	#define HV_VM_SPACE_DEFAULT 0
119	/** Default vCPU creation flags. */
120	#define HV_VCPU_DEFAULT 0
121
122	#define HV_DEADLINE_FOREVER UINT64_MAX
123
124
125	/*********************************************************************************************************************************
126	* Structures and Typedefs *
127	*********************************************************************************************************************************/
128
129	/** HV return code type. */
130	typedef uint32_t hv_return_t;
131	/** HV capability bitmask. */
132	typedef uint64_t hv_capability_t;
133	/** Option bitmask type when creating a VM. */
134	typedef uint64_t hv_vm_options_t;
135	/** Option bitmask when creating a vCPU. */
136	typedef uint64_t hv_vcpu_options_t;
137	/** HV memory protection flags type. */
138	typedef uint64_t hv_memory_flags_t;
139	/** Shadow VMCS protection flags. */
140	typedef uint64_t hv_shadow_flags_t;
141	/** Guest physical address type. */
142	typedef uint64_t hv_gpaddr_t;
143
144
145	/**
146	* VMX Capability enumeration.
147	*/
148	typedef enum
149	{
150	HV_VMX_CAP_PINBASED = 0,
151	HV_VMX_CAP_PROCBASED,
152	HV_VMX_CAP_PROCBASED2,
153	HV_VMX_CAP_ENTRY,
154	HV_VMX_CAP_EXIT,
155	HV_VMX_CAP_BASIC, /* Since 11.0 */
156	HV_VMX_CAP_TRUE_PINBASED, /* Since 11.0 */
157	HV_VMX_CAP_TRUE_PROCBASED, /* Since 11.0 */
158	HV_VMX_CAP_TRUE_ENTRY, /* Since 11.0 */
159	HV_VMX_CAP_TRUE_EXIT, /* Since 11.0 */
160	HV_VMX_CAP_MISC, /* Since 11.0 */
161	HV_VMX_CAP_CR0_FIXED0, /* Since 11.0 */
162	HV_VMX_CAP_CR0_FIXED1, /* Since 11.0 */
163	HV_VMX_CAP_CR4_FIXED0, /* Since 11.0 */
164	HV_VMX_CAP_CR4_FIXED1, /* Since 11.0 */
165	HV_VMX_CAP_VMCS_ENUM, /* Since 11.0 */
166	HV_VMX_CAP_EPT_VPID_CAP, /* Since 11.0 */
167	HV_VMX_CAP_PREEMPTION_TIMER = 32
168	} hv_vmx_capability_t;
169
170
171	/**
172	* MSR information.
173	*/
174	typedef enum
175	{
176	HV_VMX_INFO_MSR_IA32_ARCH_CAPABILITIES = 0,
177	HV_VMX_INFO_MSR_IA32_PERF_CAPABILITIES,
178	HV_VMX_VALID_MSR_IA32_PERFEVNTSEL,
179	HV_VMX_VALID_MSR_IA32_FIXED_CTR_CTRL,
180	HV_VMX_VALID_MSR_IA32_PERF_GLOBAL_CTRL,
181	HV_VMX_VALID_MSR_IA32_PERF_GLOBAL_STATUS,
182	HV_VMX_VALID_MSR_IA32_DEBUGCTL,
183	HV_VMX_VALID_MSR_IA32_SPEC_CTRL,
184	HV_VMX_NEED_MSR_IA32_SPEC_CTRL
185	} hv_vmx_msr_info_t;
186
187
188	/**
189	* HV x86 register enumeration.
190	*/
191	typedef enum
192	{
193	HV_X86_RIP = 0,
194	HV_X86_RFLAGS,
195	HV_X86_RAX,
196	HV_X86_RCX,
197	HV_X86_RDX,
198	HV_X86_RBX,
199	HV_X86_RSI,
200	HV_X86_RDI,
201	HV_X86_RSP,
202	HV_X86_RBP,
203	HV_X86_R8,
204	HV_X86_R9,
205	HV_X86_R10,
206	HV_X86_R11,
207	HV_X86_R12,
208	HV_X86_R13,
209	HV_X86_R14,
210	HV_X86_R15,
211	HV_X86_CS,
212	HV_X86_SS,
213	HV_X86_DS,
214	HV_X86_ES,
215	HV_X86_FS,
216	HV_X86_GS,
217	HV_X86_IDT_BASE,
218	HV_X86_IDT_LIMIT,
219	HV_X86_GDT_BASE,
220	HV_X86_GDT_LIMIT,
221	HV_X86_LDTR,
222	HV_X86_LDT_BASE,
223	HV_X86_LDT_LIMIT,
224	HV_X86_LDT_AR,
225	HV_X86_TR,
226	HV_X86_TSS_BASE,
227	HV_X86_TSS_LIMIT,
228	HV_X86_TSS_AR,
229	HV_X86_CR0,
230	HV_X86_CR1,
231	HV_X86_CR2,
232	HV_X86_CR3,
233	HV_X86_CR4,
234	HV_X86_DR0,
235	HV_X86_DR1,
236	HV_X86_DR2,
237	HV_X86_DR3,
238	HV_X86_DR4,
239	HV_X86_DR5,
240	HV_X86_DR6,
241	HV_X86_DR7,
242	HV_X86_TPR,
243	HV_X86_XCR0,
244	HV_X86_REGISTERS_MAX
245	} hv_x86_reg_t;
246
247
248	/** MSR permission flags type. */
249	typedef uint32_t hv_msr_flags_t;
250	/** MSR can't be accessed. */
251	#define HV_MSR_NONE 0
252	/** MSR is readable by the guest. */
253	#define HV_MSR_READ RT_BIT(0)
254	/** MSR is writeable by the guest. */
255	#define HV_MSR_WRITE RT_BIT(1)
256
257
258	typedef hv_return_t FN_HV_CAPABILITY(hv_capability_t capability, uint64_t *valu);
259	typedef hv_return_t FN_HV_VM_CREATE(hv_vm_options_t flags);
260	typedef hv_return_t FN_HV_VM_DESTROY(void);
261	typedef hv_return_t FN_HV_VM_SPACE_CREATE(hv_vm_space_t *asid);
262	typedef hv_return_t FN_HV_VM_SPACE_DESTROY(hv_vm_space_t asid);
263	typedef hv_return_t FN_HV_VM_MAP(const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
264	typedef hv_return_t FN_HV_VM_UNMAP(hv_gpaddr_t gpa, size_t size);
265	typedef hv_return_t FN_HV_VM_PROTECT(hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
266	typedef hv_return_t FN_HV_VM_MAP_SPACE(hv_vm_space_t asid, const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
267	typedef hv_return_t FN_HV_VM_UNMAP_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size);
268	typedef hv_return_t FN_HV_VM_PROTECT_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
269	typedef hv_return_t FN_HV_VM_SYNC_TSC(uint64_t tsc);
270
271	typedef hv_return_t FN_HV_VCPU_CREATE(hv_vcpuid_t *vcpu, hv_vcpu_options_t flags);
272	typedef hv_return_t FN_HV_VCPU_DESTROY(hv_vcpuid_t vcpu);
273	typedef hv_return_t FN_HV_VCPU_SET_SPACE(hv_vcpuid_t vcpu, hv_vm_space_t asid);
274	typedef hv_return_t FN_HV_VCPU_READ_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t *value);
275	typedef hv_return_t FN_HV_VCPU_WRITE_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t value);
276	typedef hv_return_t FN_HV_VCPU_READ_FPSTATE(hv_vcpuid_t vcpu, void *buffer, size_t size);
277	typedef hv_return_t FN_HV_VCPU_WRITE_FPSTATE(hv_vcpuid_t vcpu, const void *buffer, size_t size);
278	typedef hv_return_t FN_HV_VCPU_ENABLE_NATIVE_MSR(hv_vcpuid_t vcpu, uint32_t msr, bool enable);
279	typedef hv_return_t FN_HV_VCPU_READ_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t *value);
280	typedef hv_return_t FN_HV_VCPU_WRITE_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t value);
281	typedef hv_return_t FN_HV_VCPU_FLUSH(hv_vcpuid_t vcpu);
282	typedef hv_return_t FN_HV_VCPU_INVALIDATE_TLB(hv_vcpuid_t vcpu);
283	typedef hv_return_t FN_HV_VCPU_RUN(hv_vcpuid_t vcpu);
284	typedef hv_return_t FN_HV_VCPU_RUN_UNTIL(hv_vcpuid_t vcpu, uint64_t deadline);
285	typedef hv_return_t FN_HV_VCPU_INTERRUPT(hv_vcpuid_t *vcpus, unsigned int vcpu_count);
286	typedef hv_return_t FN_HV_VCPU_GET_EXEC_TIME(hv_vcpuid_t vcpus, uint64_t time);
287
288	typedef hv_return_t FN_HV_VMX_VCPU_READ_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
289	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
290
291	typedef hv_return_t FN_HV_VMX_VCPU_READ_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
292	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
293	typedef hv_return_t FN_HV_VMX_VCPU_SET_SHADOW_ACCESS(hv_vcpuid_t vcpu, uint32_t field, hv_shadow_flags_t flags);
294
295	typedef hv_return_t FN_HV_VMX_READ_CAPABILITY(hv_vmx_capability_t field, uint64_t *value);
296	typedef hv_return_t FN_HV_VMX_VCPU_SET_APIC_ADDRESS(hv_vcpuid_t vcpu, hv_gpaddr_t gpa);
297
298	/* Since 11.0 */
299	typedef hv_return_t FN_HV_VMX_GET_MSR_INFO(hv_vmx_msr_info_t field, uint64_t *value);
300	typedef hv_return_t FN_HV_VMX_VCPU_GET_CAP_WRITE_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t allowed_0, uint64_t allowed_1);
301	typedef hv_return_t FN_HV_VCPU_ENABLE_MANAGED_MSR(hv_vcpuid_t vcpu, uint32_t msr, bool enable);
302	typedef hv_return_t FN_HV_VCPU_SET_MSR_ACCESS(hv_vcpuid_t vcpu, uint32_t msr, hv_msr_flags_t flags);
303
304
305	/*********************************************************************************************************************************
306	* Global Variables *
307	*********************************************************************************************************************************/
308	static void nemR3DarwinVmcsDump(PVMCPU pVCpu);
309
310	/** NEM_DARWIN_PAGE_STATE_XXX names. */
311	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
312	/** MSRs. */
313	static SUPHWVIRTMSRS g_HmMsrs;
314	/** VMX: Set if swapping EFER is supported. */
315	static bool g_fHmVmxSupportsVmcsEfer = false;
316	/** @name APIs imported from Hypervisor.framework.
317	* @{ */
318	static FN_HV_CAPABILITY g_pfnHvCapability = NULL; / Since 10.15 */
319	static FN_HV_VM_CREATE g_pfnHvVmCreate = NULL; / Since 10.10 */
320	static FN_HV_VM_DESTROY g_pfnHvVmDestroy = NULL; / Since 10.10 */
321	static FN_HV_VM_SPACE_CREATE g_pfnHvVmSpaceCreate = NULL; / Since 10.15 */
322	static FN_HV_VM_SPACE_DESTROY g_pfnHvVmSpaceDestroy = NULL; / Since 10.15 */
323	static FN_HV_VM_MAP g_pfnHvVmMap = NULL; / Since 10.10 */
324	static FN_HV_VM_UNMAP g_pfnHvVmUnmap = NULL; / Since 10.10 */
325	static FN_HV_VM_PROTECT g_pfnHvVmProtect = NULL; / Since 10.10 */
326	static FN_HV_VM_MAP_SPACE g_pfnHvVmMapSpace = NULL; / Since 10.15 */
327	static FN_HV_VM_UNMAP_SPACE g_pfnHvVmUnmapSpace = NULL; / Since 10.15 */
328	static FN_HV_VM_PROTECT_SPACE g_pfnHvVmProtectSpace = NULL; / Since 10.15 */
329	static FN_HV_VM_SYNC_TSC g_pfnHvVmSyncTsc = NULL; / Since 10.10 */
330
331	static FN_HV_VCPU_CREATE g_pfnHvVCpuCreate = NULL; / Since 10.10 */
332	static FN_HV_VCPU_DESTROY g_pfnHvVCpuDestroy = NULL; / Since 10.10 */
333	static FN_HV_VCPU_SET_SPACE g_pfnHvVCpuSetSpace = NULL; / Since 10.15 */
334	static FN_HV_VCPU_READ_REGISTER g_pfnHvVCpuReadRegister = NULL; / Since 10.10 */
335	static FN_HV_VCPU_WRITE_REGISTER g_pfnHvVCpuWriteRegister = NULL; / Since 10.10 */
336	static FN_HV_VCPU_READ_FPSTATE g_pfnHvVCpuReadFpState = NULL; / Since 10.10 */
337	static FN_HV_VCPU_WRITE_FPSTATE g_pfnHvVCpuWriteFpState = NULL; / Since 10.10 */
338	static FN_HV_VCPU_ENABLE_NATIVE_MSR g_pfnHvVCpuEnableNativeMsr = NULL; / Since 10.10 */
339	static FN_HV_VCPU_READ_MSR g_pfnHvVCpuReadMsr = NULL; / Since 10.10 */
340	static FN_HV_VCPU_WRITE_MSR g_pfnHvVCpuWriteMsr = NULL; / Since 10.10 */
341	static FN_HV_VCPU_FLUSH g_pfnHvVCpuFlush = NULL; / Since 10.10 */
342	static FN_HV_VCPU_INVALIDATE_TLB g_pfnHvVCpuInvalidateTlb = NULL; / Since 10.10 */
343	static FN_HV_VCPU_RUN g_pfnHvVCpuRun = NULL; / Since 10.10 */
344	static FN_HV_VCPU_RUN_UNTIL g_pfnHvVCpuRunUntil = NULL; / Since 10.15 */
345	static FN_HV_VCPU_INTERRUPT g_pfnHvVCpuInterrupt = NULL; / Since 10.10 */
346	static FN_HV_VCPU_GET_EXEC_TIME g_pfnHvVCpuGetExecTime = NULL; / Since 10.10 */
347
348	static FN_HV_VMX_READ_CAPABILITY g_pfnHvVmxReadCapability = NULL; / Since 10.10 */
349	static FN_HV_VMX_VCPU_READ_VMCS g_pfnHvVmxVCpuReadVmcs = NULL; / Since 10.10 */
350	static FN_HV_VMX_VCPU_WRITE_VMCS g_pfnHvVmxVCpuWriteVmcs = NULL; / Since 10.10 */
351	static FN_HV_VMX_VCPU_READ_SHADOW_VMCS g_pfnHvVmxVCpuReadShadowVmcs = NULL; / Since 10.15 */
352	static FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS g_pfnHvVmxVCpuWriteShadowVmcs = NULL; / Since 10.15 */
353	static FN_HV_VMX_VCPU_SET_SHADOW_ACCESS g_pfnHvVmxVCpuSetShadowAccess = NULL; / Since 10.15 */
354	static FN_HV_VMX_VCPU_SET_APIC_ADDRESS g_pfnHvVmxVCpuSetApicAddress = NULL; / Since 10.10 */
355
356	static FN_HV_VMX_GET_MSR_INFO g_pfnHvVmxGetMsrInfo = NULL; / Since 11.0 */
357	static FN_HV_VMX_VCPU_GET_CAP_WRITE_VMCS g_pfnHvVmxVCpuGetCapWriteVmcs = NULL; / Since 11.0 */
358	static FN_HV_VCPU_ENABLE_MANAGED_MSR g_pfnHvVCpuEnableManagedMsr = NULL; / Since 11.0 */
359	static FN_HV_VCPU_SET_MSR_ACCESS g_pfnHvVCpuSetMsrAccess = NULL; / Since 11.0 */
360	/** @} */
361
362
363	/**
364	* Import instructions.
365	*/
366	static const struct
367	{
368	bool fOptional; /*< Set if import is optional. /
369	void ppfn; /< The function pointer variable. */
370	const char pszName; /< The function name. /
371	} g_aImports[] =
372	{
373	#define NEM_DARWIN_IMPORT(a_fOptional, a_Pfn, a_Name) { (a_fOptional), (void **)&(a_Pfn), #a_Name }
374	NEM_DARWIN_IMPORT(true, g_pfnHvCapability, hv_capability),
375	NEM_DARWIN_IMPORT(false, g_pfnHvVmCreate, hv_vm_create),
376	NEM_DARWIN_IMPORT(false, g_pfnHvVmDestroy, hv_vm_destroy),
377	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceCreate, hv_vm_space_create),
378	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceDestroy, hv_vm_space_destroy),
379	NEM_DARWIN_IMPORT(false, g_pfnHvVmMap, hv_vm_map),
380	NEM_DARWIN_IMPORT(false, g_pfnHvVmUnmap, hv_vm_unmap),
381	NEM_DARWIN_IMPORT(false, g_pfnHvVmProtect, hv_vm_protect),
382	NEM_DARWIN_IMPORT(true, g_pfnHvVmMapSpace, hv_vm_map_space),
383	NEM_DARWIN_IMPORT(true, g_pfnHvVmUnmapSpace, hv_vm_unmap_space),
384	NEM_DARWIN_IMPORT(true, g_pfnHvVmProtectSpace, hv_vm_protect_space),
385	NEM_DARWIN_IMPORT(false, g_pfnHvVmSyncTsc, hv_vm_sync_tsc),
386
387	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuCreate, hv_vcpu_create),
388	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuDestroy, hv_vcpu_destroy),
389	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuSetSpace, hv_vcpu_set_space),
390	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadRegister, hv_vcpu_read_register),
391	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteRegister, hv_vcpu_write_register),
392	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadFpState, hv_vcpu_read_fpstate),
393	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteFpState, hv_vcpu_write_fpstate),
394	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuEnableNativeMsr, hv_vcpu_enable_native_msr),
395	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadMsr, hv_vcpu_read_msr),
396	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteMsr, hv_vcpu_write_msr),
397	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuFlush, hv_vcpu_flush),
398	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInvalidateTlb, hv_vcpu_invalidate_tlb),
399	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuRun, hv_vcpu_run),
400	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuRunUntil, hv_vcpu_run_until),
401	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInterrupt, hv_vcpu_interrupt),
402	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuGetExecTime, hv_vcpu_get_exec_time),
403	NEM_DARWIN_IMPORT(false, g_pfnHvVmxReadCapability, hv_vmx_read_capability),
404	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuReadVmcs, hv_vmx_vcpu_read_vmcs),
405	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuWriteVmcs, hv_vmx_vcpu_write_vmcs),
406	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuReadShadowVmcs, hv_vmx_vcpu_read_shadow_vmcs),
407	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuWriteShadowVmcs, hv_vmx_vcpu_write_shadow_vmcs),
408	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuSetShadowAccess, hv_vmx_vcpu_set_shadow_access),
409	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuSetApicAddress, hv_vmx_vcpu_set_apic_address),
410	NEM_DARWIN_IMPORT(true, g_pfnHvVmxGetMsrInfo, hv_vmx_get_msr_info),
411	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuGetCapWriteVmcs, hv_vmx_vcpu_get_cap_write_vmcs),
412	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuEnableManagedMsr, hv_vcpu_enable_managed_msr),
413	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuSetMsrAccess, hv_vcpu_set_msr_access)
414	#undef NEM_DARWIN_IMPORT
415	};
416
417
418	/*
419	* Let the preprocessor alias the APIs to import variables for better autocompletion.
420	*/
421	#ifndef IN_SLICKEDIT
422	# define hv_capability g_pfnHvCapability
423	# define hv_vm_create g_pfnHvVmCreate
424	# define hv_vm_destroy g_pfnHvVmDestroy
425	# define hv_vm_space_create g_pfnHvVmSpaceCreate
426	# define hv_vm_space_destroy g_pfnHvVmSpaceDestroy
427	# define hv_vm_map g_pfnHvVmMap
428	# define hv_vm_unmap g_pfnHvVmUnmap
429	# define hv_vm_protect g_pfnHvVmProtect
430	# define hv_vm_map_space g_pfnHvVmMapSpace
431	# define hv_vm_unmap_space g_pfnHvVmUnmapSpace
432	# define hv_vm_protect_space g_pfnHvVmProtectSpace
433	# define hv_vm_sync_tsc g_pfnHvVmSyncTsc
434
435	# define hv_vcpu_create g_pfnHvVCpuCreate
436	# define hv_vcpu_destroy g_pfnHvVCpuDestroy
437	# define hv_vcpu_set_space g_pfnHvVCpuSetSpace
438	# define hv_vcpu_read_register g_pfnHvVCpuReadRegister
439	# define hv_vcpu_write_register g_pfnHvVCpuWriteRegister
440	# define hv_vcpu_read_fpstate g_pfnHvVCpuReadFpState
441	# define hv_vcpu_write_fpstate g_pfnHvVCpuWriteFpState
442	# define hv_vcpu_enable_native_msr g_pfnHvVCpuEnableNativeMsr
443	# define hv_vcpu_read_msr g_pfnHvVCpuReadMsr
444	# define hv_vcpu_write_msr g_pfnHvVCpuWriteMsr
445	# define hv_vcpu_flush g_pfnHvVCpuFlush
446	# define hv_vcpu_invalidate_tlb g_pfnHvVCpuInvalidateTlb
447	# define hv_vcpu_run g_pfnHvVCpuRun
448	# define hv_vcpu_run_until g_pfnHvVCpuRunUntil
449	# define hv_vcpu_interrupt g_pfnHvVCpuInterrupt
450	# define hv_vcpu_get_exec_time g_pfnHvVCpuGetExecTime
451
452	# define hv_vmx_read_capability g_pfnHvVmxReadCapability
453	# define hv_vmx_vcpu_read_vmcs g_pfnHvVmxVCpuReadVmcs
454	# define hv_vmx_vcpu_write_vmcs g_pfnHvVmxVCpuWriteVmcs
455	# define hv_vmx_vcpu_read_shadow_vmcs g_pfnHvVmxVCpuReadShadowVmcs
456	# define hv_vmx_vcpu_write_shadow_vmcs g_pfnHvVmxVCpuWriteShadowVmcs
457	# define hv_vmx_vcpu_set_shadow_access g_pfnHvVmxVCpuSetShadowAccess
458	# define hv_vmx_vcpu_set_apic_address g_pfnHvVmxVCpuSetApicAddress
459
460	# define hv_vmx_get_msr_info g_pfnHvVmxGetMsrInfo
461	# define hv_vmx_vcpu_get_cap_write_vmcs g_pfnHvVmxVCpuGetCapWriteVmcs
462	# define hv_vcpu_enable_managed_msr g_pfnHvVCpuEnableManagedMsr
463	# define hv_vcpu_set_msr_access g_pfnHvVCpuSetMsrAccess
464	#endif
465
466	static const struct
467	{
468	uint32_t u32VmcsFieldId; /*< The VMCS field identifier. /
469	const char pszVmcsField; /< The VMCS field name. /
470	bool f64Bit;
471	} g_aVmcsFieldsCap[] =
472	{
473	#define NEM_DARWIN_VMCS64_FIELD_CAP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, true }
474	#define NEM_DARWIN_VMCS32_FIELD_CAP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, false }
475
476	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PIN_EXEC),
477	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PROC_EXEC),
478	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_EXCEPTION_BITMAP),
479	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_EXIT),
480	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_ENTRY),
481	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PROC_EXEC2),
482	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PLE_GAP),
483	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PLE_WINDOW),
484	NEM_DARWIN_VMCS64_FIELD_CAP(VMX_VMCS64_CTRL_TSC_OFFSET_FULL),
485	NEM_DARWIN_VMCS64_FIELD_CAP(VMX_VMCS64_GUEST_DEBUGCTL_FULL)
486	#undef NEM_DARWIN_VMCS64_FIELD_CAP
487	#undef NEM_DARWIN_VMCS32_FIELD_CAP
488	};
489
490
491	/*********************************************************************************************************************************
492	* Internal Functions *
493	*********************************************************************************************************************************/
494	DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo);
495
496
497	/**
498	* Converts a HV return code to a VBox status code.
499	*
500	* @returns VBox status code.
501	* @param hrc The HV return code to convert.
502	*/
503	DECLINLINE(int) nemR3DarwinHvSts2Rc(hv_return_t hrc)
504	{
505	if (hrc == HV_SUCCESS)
506	return VINF_SUCCESS;
507
508	switch (hrc)
509	{
510	case HV_ERROR: return VERR_INVALID_STATE;
511	case HV_BUSY: return VERR_RESOURCE_BUSY;
512	case HV_BAD_ARGUMENT: return VERR_INVALID_PARAMETER;
513	case HV_NO_RESOURCES: return VERR_OUT_OF_RESOURCES;
514	case HV_NO_DEVICE: return VERR_NOT_FOUND;
515	case HV_UNSUPPORTED: return VERR_NOT_SUPPORTED;
516	}
517
518	return VERR_IPE_UNEXPECTED_STATUS;
519	}
520
521
522	/**
523	* Unmaps the given guest physical address range (page aligned).
524	*
525	* @returns VBox status code.
526	* @param pVM The cross context VM structure.
527	* @param GCPhys The guest physical address to start unmapping at.
528	* @param cb The size of the range to unmap in bytes.
529	* @param pu2State Where to store the new state of the unmappd page, optional.
530	*/
531	DECLINLINE(int) nemR3DarwinUnmap(PVM pVM, RTGCPHYS GCPhys, size_t cb, uint8_t *pu2State)
532	{
533	if (*pu2State <= NEM_DARWIN_PAGE_STATE_UNMAPPED)
534	{
535	Log5(("nemR3DarwinUnmap: %RGp == unmapped\n", GCPhys));
536	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
537	return VINF_SUCCESS;
538	}
539
540	LogFlowFunc(("Unmapping %RGp LB %zu\n", GCPhys, cb));
541	hv_return_t hrc;
542	if (pVM->nem.s.fCreatedAsid)
543	hrc = hv_vm_unmap_space(pVM->nem.s.uVmAsid, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, cb);
544	else
545	hrc = hv_vm_unmap(GCPhys, cb);
546	if (RT_LIKELY(hrc == HV_SUCCESS))
547	{
548	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
549	if (pu2State)
550	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
551	Log5(("nemR3DarwinUnmap: %RGp => unmapped\n", GCPhys));
552	return VINF_SUCCESS;
553	}
554
555	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
556	LogRel(("nemR3DarwinUnmap(%RGp): failed! hrc=%#x\n",
557	GCPhys, hrc));
558	return VERR_NEM_IPE_6;
559	}
560
561
562	/**
563	* Maps a given guest physical address range backed by the given memory with the given
564	* protection flags.
565	*
566	* @returns VBox status code.
567	* @param pVM The cross context VM structure.
568	* @param GCPhys The guest physical address to start mapping.
569	* @param pvRam The R3 pointer of the memory to back the range with.
570	* @param cb The size of the range, page aligned.
571	* @param fPageProt The page protection flags to use for this range, combination of NEM_PAGE_PROT_XXX
572	* @param pu2State Where to store the state for the new page, optional.
573	*/
574	DECLINLINE(int) nemR3DarwinMap(PVM pVM, RTGCPHYS GCPhys, const void pvRam, size_t cb, uint32_t fPageProt, uint8_t pu2State)
575	{
576	LogFlowFunc(("Mapping %RGp LB %zu fProt=%#x\n", GCPhys, cb, fPageProt));
577
578	Assert(fPageProt != NEM_PAGE_PROT_NONE);
579
580	hv_memory_flags_t fHvMemProt = 0;
581	if (fPageProt & NEM_PAGE_PROT_READ)
582	fHvMemProt \|= HV_MEMORY_READ;
583	if (fPageProt & NEM_PAGE_PROT_WRITE)
584	fHvMemProt \|= HV_MEMORY_WRITE;
585	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
586	fHvMemProt \|= HV_MEMORY_EXEC;
587
588	hv_return_t hrc;
589	if (pVM->nem.s.fCreatedAsid)
590	hrc = hv_vm_map_space(pVM->nem.s.uVmAsid, pvRam, GCPhys, cb, fHvMemProt);
591	else
592	hrc = hv_vm_map(pvRam, GCPhys, cb, fHvMemProt);
593	if (hrc == HV_SUCCESS)
594	{
595	if (pu2State)
596	*pu2State = (fPageProt & NEM_PAGE_PROT_WRITE)
597	? NEM_DARWIN_PAGE_STATE_WRITABLE
598	: NEM_DARWIN_PAGE_STATE_READABLE;
599	return VINF_SUCCESS;
600	}
601
602	return nemR3DarwinHvSts2Rc(hrc);
603	}
604
605	#if 0 /* unused */
606	DECLINLINE(int) nemR3DarwinProtectPage(PVM pVM, RTGCPHYS GCPhys, size_t cb, uint32_t fPageProt)
607	{
608	hv_memory_flags_t fHvMemProt = 0;
609	if (fPageProt & NEM_PAGE_PROT_READ)
610	fHvMemProt \|= HV_MEMORY_READ;
611	if (fPageProt & NEM_PAGE_PROT_WRITE)
612	fHvMemProt \|= HV_MEMORY_WRITE;
613	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
614	fHvMemProt \|= HV_MEMORY_EXEC;
615
616	hv_return_t hrc;
617	if (pVM->nem.s.fCreatedAsid)
618	hrc = hv_vm_protect_space(pVM->nem.s.uVmAsid, GCPhys, cb, fHvMemProt);
619	else
620	hrc = hv_vm_protect(GCPhys, cb, fHvMemProt);
621
622	return nemR3DarwinHvSts2Rc(hrc);
623	}
624	#endif
625
626	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
627	{
628	PGMPAGEMAPLOCK Lock;
629	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
630	if (RT_SUCCESS(rc))
631	PGMPhysReleasePageMappingLock(pVM, &Lock);
632	return rc;
633	}
634
635
636	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
637	{
638	PGMPAGEMAPLOCK Lock;
639	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
640	if (RT_SUCCESS(rc))
641	PGMPhysReleasePageMappingLock(pVM, &Lock);
642	return rc;
643	}
644
645
646	#ifdef LOG_ENABLED
647	/**
648	* Logs the current CPU state.
649	*/
650	static void nemR3DarwinLogState(PVMCC pVM, PVMCPUCC pVCpu)
651	{
652	if (LogIs3Enabled())
653	{
654	#if 0
655	char szRegs[4096];
656	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
657	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
658	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
659	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
660	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
661	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
662	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
663	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
664	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
665	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
666	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
667	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
668	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
669	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
670	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
671	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
672	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
673	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
674	" efer=%016VR{efer}\n"
675	" pat=%016VR{pat}\n"
676	" sf_mask=%016VR{sf_mask}\n"
677	"krnl_gs_base=%016VR{krnl_gs_base}\n"
678	" lstar=%016VR{lstar}\n"
679	" star=%016VR{star} cstar=%016VR{cstar}\n"
680	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
681	);
682
683	char szInstr[256];
684	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
685	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
686	szInstr, sizeof(szInstr), NULL);
687	Log3(("%s%s\n", szRegs, szInstr));
688	#else
689	RT_NOREF(pVM, pVCpu);
690	#endif
691	}
692	}
693	#endif /* LOG_ENABLED */
694
695
696	DECLINLINE(int) nemR3DarwinReadVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t *pData)
697	{
698	uint64_t u64Data;
699	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
700	if (RT_LIKELY(hrc == HV_SUCCESS))
701	{
702	*pData = (uint16_t)u64Data;
703	return VINF_SUCCESS;
704	}
705
706	return nemR3DarwinHvSts2Rc(hrc);
707	}
708
709
710	DECLINLINE(int) nemR3DarwinReadVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t *pData)
711	{
712	uint64_t u64Data;
713	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
714	if (RT_LIKELY(hrc == HV_SUCCESS))
715	{
716	*pData = (uint32_t)u64Data;
717	return VINF_SUCCESS;
718	}
719
720	return nemR3DarwinHvSts2Rc(hrc);
721	}
722
723
724	DECLINLINE(int) nemR3DarwinReadVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t *pData)
725	{
726	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, pData);
727	if (RT_LIKELY(hrc == HV_SUCCESS))
728	return VINF_SUCCESS;
729
730	return nemR3DarwinHvSts2Rc(hrc);
731	}
732
733
734	DECLINLINE(int) nemR3DarwinWriteVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t u16Val)
735	{
736	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u16Val);
737	if (RT_LIKELY(hrc == HV_SUCCESS))
738	return VINF_SUCCESS;
739
740	return nemR3DarwinHvSts2Rc(hrc);
741	}
742
743
744	DECLINLINE(int) nemR3DarwinWriteVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t u32Val)
745	{
746	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u32Val);
747	if (RT_LIKELY(hrc == HV_SUCCESS))
748	return VINF_SUCCESS;
749
750	return nemR3DarwinHvSts2Rc(hrc);
751	}
752
753
754	DECLINLINE(int) nemR3DarwinWriteVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t u64Val)
755	{
756	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u64Val);
757	if (RT_LIKELY(hrc == HV_SUCCESS))
758	return VINF_SUCCESS;
759
760	return nemR3DarwinHvSts2Rc(hrc);
761	}
762
763	DECLINLINE(int) nemR3DarwinMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Val)
764	{
765	hv_return_t hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, idMsr, pu64Val);
766	if (RT_LIKELY(hrc == HV_SUCCESS))
767	return VINF_SUCCESS;
768
769	return nemR3DarwinHvSts2Rc(hrc);
770	}
771
772	#if 0 /unused/
773	DECLINLINE(int) nemR3DarwinMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Val)
774	{
775	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, idMsr, u64Val);
776	if (RT_LIKELY(hrc == HV_SUCCESS))
777	return VINF_SUCCESS;
778
779	return nemR3DarwinHvSts2Rc(hrc);
780	}
781	#endif
782
783	static int nemR3DarwinCopyStateFromHv(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
784	{
785	#define READ_GREG(a_GReg, a_Value) \
786	do \
787	{ \
788	hrc = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, (a_GReg), &(a_Value)); \
789	if (RT_LIKELY(hrc == HV_SUCCESS)) \
790	{ /* likely */ } \
791	else \
792	return VERR_INTERNAL_ERROR; \
793	} while(0)
794	#define READ_VMCS_FIELD(a_Field, a_Value) \
795	do \
796	{ \
797	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &(a_Value)); \
798	if (RT_LIKELY(hrc == HV_SUCCESS)) \
799	{ /* likely */ } \
800	else \
801	return VERR_INTERNAL_ERROR; \
802	} while(0)
803	#define READ_VMCS16_FIELD(a_Field, a_Value) \
804	do \
805	{ \
806	uint64_t u64Data; \
807	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
808	if (RT_LIKELY(hrc == HV_SUCCESS)) \
809	{ (a_Value) = (uint16_t)u64Data; } \
810	else \
811	return VERR_INTERNAL_ERROR; \
812	} while(0)
813	#define READ_VMCS32_FIELD(a_Field, a_Value) \
814	do \
815	{ \
816	uint64_t u64Data; \
817	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
818	if (RT_LIKELY(hrc == HV_SUCCESS)) \
819	{ (a_Value) = (uint32_t)u64Data; } \
820	else \
821	return VERR_INTERNAL_ERROR; \
822	} while(0)
823	#define READ_MSR(a_Msr, a_Value) \
824	do \
825	{ \
826	hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, (a_Msr), &(a_Value)); \
827	if (RT_LIKELY(hrc == HV_SUCCESS)) \
828	{ /* likely */ } \
829	else \
830	AssertFailedReturn(VERR_INTERNAL_ERROR); \
831	} while(0)
832
833	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateImport, x);
834
835	RT_NOREF(pVM);
836	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
837
838	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
839	vmxHCImportGuestIntrState(pVCpu, &pVCpu->nem.s.VmcsInfo);
840
841	/* GPRs */
842	hv_return_t hrc;
843	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
844	{
845	if (fWhat & CPUMCTX_EXTRN_RAX)
846	READ_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
847	if (fWhat & CPUMCTX_EXTRN_RCX)
848	READ_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
849	if (fWhat & CPUMCTX_EXTRN_RDX)
850	READ_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
851	if (fWhat & CPUMCTX_EXTRN_RBX)
852	READ_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
853	if (fWhat & CPUMCTX_EXTRN_RSP)
854	READ_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
855	if (fWhat & CPUMCTX_EXTRN_RBP)
856	READ_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
857	if (fWhat & CPUMCTX_EXTRN_RSI)
858	READ_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
859	if (fWhat & CPUMCTX_EXTRN_RDI)
860	READ_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
861	if (fWhat & CPUMCTX_EXTRN_R8_R15)
862	{
863	READ_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
864	READ_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
865	READ_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
866	READ_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
867	READ_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
868	READ_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
869	READ_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
870	READ_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
871	}
872	}
873
874	/* RIP & Flags */
875	if (fWhat & CPUMCTX_EXTRN_RIP)
876	READ_GREG(HV_X86_RIP, pVCpu->cpum.GstCtx.rip);
877	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
878	{
879	uint64_t fRFlagsTmp = 0;
880	READ_GREG(HV_X86_RFLAGS, fRFlagsTmp);
881	pVCpu->cpum.GstCtx.rflags.u = fRFlagsTmp;
882	}
883
884	/* Segments */
885	#define READ_SEG(a_SReg, a_enmName) \
886	do { \
887	READ_VMCS16_FIELD(VMX_VMCS16_GUEST_ ## a_enmName ## _SEL, (a_SReg).Sel); \
888	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _LIMIT, (a_SReg).u32Limit); \
889	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _ACCESS_RIGHTS, (a_SReg).Attr.u); \
890	READ_VMCS_FIELD(VMX_VMCS_GUEST_ ## a_enmName ## _BASE, (a_SReg).u64Base); \
891	(a_SReg).ValidSel = (a_SReg).Sel; \
892	} while (0)
893	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
894	{
895	if (fWhat & CPUMCTX_EXTRN_ES)
896	READ_SEG(pVCpu->cpum.GstCtx.es, ES);
897	if (fWhat & CPUMCTX_EXTRN_CS)
898	READ_SEG(pVCpu->cpum.GstCtx.cs, CS);
899	if (fWhat & CPUMCTX_EXTRN_SS)
900	READ_SEG(pVCpu->cpum.GstCtx.ss, SS);
901	if (fWhat & CPUMCTX_EXTRN_DS)
902	READ_SEG(pVCpu->cpum.GstCtx.ds, DS);
903	if (fWhat & CPUMCTX_EXTRN_FS)
904	READ_SEG(pVCpu->cpum.GstCtx.fs, FS);
905	if (fWhat & CPUMCTX_EXTRN_GS)
906	READ_SEG(pVCpu->cpum.GstCtx.gs, GS);
907	}
908
909	/* Descriptor tables and the task segment. */
910	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
911	{
912	if (fWhat & CPUMCTX_EXTRN_LDTR)
913	READ_SEG(pVCpu->cpum.GstCtx.ldtr, LDTR);
914
915	if (fWhat & CPUMCTX_EXTRN_TR)
916	{
917	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
918	avoid to trigger sanity assertions around the code, always fix this. */
919	READ_SEG(pVCpu->cpum.GstCtx.tr, TR);
920	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
921	{
922	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
923	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
924	break;
925	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
926	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
927	break;
928	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
929	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
930	break;
931	}
932	}
933	if (fWhat & CPUMCTX_EXTRN_IDTR)
934	{
935	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_IDTR_LIMIT, pVCpu->cpum.GstCtx.idtr.cbIdt);
936	READ_VMCS_FIELD(VMX_VMCS_GUEST_IDTR_BASE, pVCpu->cpum.GstCtx.idtr.pIdt);
937	}
938	if (fWhat & CPUMCTX_EXTRN_GDTR)
939	{
940	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_GDTR_LIMIT, pVCpu->cpum.GstCtx.gdtr.cbGdt);
941	READ_VMCS_FIELD(VMX_VMCS_GUEST_GDTR_BASE, pVCpu->cpum.GstCtx.gdtr.pGdt);
942	}
943	}
944
945	/* Control registers. */
946	bool fMaybeChangedMode = false;
947	bool fUpdateCr3 = false;
948	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
949	{
950	uint64_t u64CrTmp = 0;
951
952	if (fWhat & CPUMCTX_EXTRN_CR0)
953	{
954	READ_GREG(HV_X86_CR0, u64CrTmp);
955	if (pVCpu->cpum.GstCtx.cr0 != u64CrTmp)
956	{
957	CPUMSetGuestCR0(pVCpu, u64CrTmp);
958	fMaybeChangedMode = true;
959	}
960	}
961	if (fWhat & CPUMCTX_EXTRN_CR2)
962	READ_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
963	if (fWhat & CPUMCTX_EXTRN_CR3)
964	{
965	READ_GREG(HV_X86_CR3, u64CrTmp);
966	if (pVCpu->cpum.GstCtx.cr3 != u64CrTmp)
967	{
968	CPUMSetGuestCR3(pVCpu, u64CrTmp);
969	fUpdateCr3 = true;
970	}
971
972	/*
973	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
974	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
975	*/
976	if (CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx))
977	{
978	X86PDPE aPaePdpes[4];
979	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE0_FULL, aPaePdpes[0].u);
980	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE1_FULL, aPaePdpes[1].u);
981	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE2_FULL, aPaePdpes[2].u);
982	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE3_FULL, aPaePdpes[3].u);
983	if (memcmp(&aPaePdpes[0], &pVCpu->cpum.GstCtx.aPaePdpes[0], sizeof(aPaePdpes)))
984	{
985	memcpy(&pVCpu->cpum.GstCtx.aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
986	fUpdateCr3 = true;
987	}
988	}
989	}
990	if (fWhat & CPUMCTX_EXTRN_CR4)
991	{
992	READ_GREG(HV_X86_CR4, u64CrTmp);
993	u64CrTmp &= ~VMX_V_CR4_FIXED0;
994
995	if (pVCpu->cpum.GstCtx.cr4 != u64CrTmp)
996	{
997	CPUMSetGuestCR4(pVCpu, u64CrTmp);
998	fMaybeChangedMode = true;
999	}
1000	}
1001	}
1002
1003	#if 0 /* Always done. */
1004	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1005	{
1006	uint64_t u64Cr8 = 0;
1007
1008	READ_GREG(HV_X86_TPR, u64Cr8);
1009	APICSetTpr(pVCpu, u64Cr8 << 4);
1010	}
1011	#endif
1012
1013	if (fWhat & CPUMCTX_EXTRN_XCRx)
1014	READ_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
1015
1016	/* Debug registers. */
1017	if (fWhat & CPUMCTX_EXTRN_DR7)
1018	{
1019	uint64_t u64Dr7;
1020	READ_GREG(HV_X86_DR7, u64Dr7);
1021	if (pVCpu->cpum.GstCtx.dr[7] != u64Dr7)
1022	CPUMSetGuestDR7(pVCpu, u64Dr7);
1023	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
1024	}
1025	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1026	{
1027	uint64_t u64DrTmp;
1028
1029	READ_GREG(HV_X86_DR0, u64DrTmp);
1030	if (pVCpu->cpum.GstCtx.dr[0] != u64DrTmp)
1031	CPUMSetGuestDR0(pVCpu, u64DrTmp);
1032	READ_GREG(HV_X86_DR1, u64DrTmp);
1033	if (pVCpu->cpum.GstCtx.dr[1] != u64DrTmp)
1034	CPUMSetGuestDR1(pVCpu, u64DrTmp);
1035	READ_GREG(HV_X86_DR2, u64DrTmp);
1036	if (pVCpu->cpum.GstCtx.dr[2] != u64DrTmp)
1037	CPUMSetGuestDR2(pVCpu, u64DrTmp);
1038	READ_GREG(HV_X86_DR3, u64DrTmp);
1039	if (pVCpu->cpum.GstCtx.dr[3] != u64DrTmp)
1040	CPUMSetGuestDR3(pVCpu, u64DrTmp);
1041	}
1042	if (fWhat & CPUMCTX_EXTRN_DR6)
1043	{
1044	uint64_t u64Dr6;
1045	READ_GREG(HV_X86_DR6, u64Dr6);
1046	if (pVCpu->cpum.GstCtx.dr[6] != u64Dr6)
1047	CPUMSetGuestDR6(pVCpu, u64Dr6);
1048	}
1049
1050	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
1051	{
1052	hrc = hv_vcpu_read_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1053	if (hrc == HV_SUCCESS)
1054	{ /* likely */ }
1055	else
1056	{
1057	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1058	return nemR3DarwinHvSts2Rc(hrc);
1059	}
1060	}
1061
1062	/* MSRs */
1063	if (fWhat & CPUMCTX_EXTRN_EFER)
1064	{
1065	uint64_t u64Efer;
1066
1067	READ_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, u64Efer);
1068	if (u64Efer != pVCpu->cpum.GstCtx.msrEFER)
1069	{
1070	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, u64Efer));
1071	if ((u64Efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1072	PGMNotifyNxeChanged(pVCpu, RT_BOOL(u64Efer & MSR_K6_EFER_NXE));
1073	pVCpu->cpum.GstCtx.msrEFER = u64Efer;
1074	fMaybeChangedMode = true;
1075	}
1076	}
1077
1078	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1079	READ_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1080	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1081	{
1082	uint64_t u64Tmp;
1083	READ_MSR(MSR_IA32_SYSENTER_EIP, u64Tmp);
1084	pVCpu->cpum.GstCtx.SysEnter.eip = u64Tmp;
1085	READ_MSR(MSR_IA32_SYSENTER_ESP, u64Tmp);
1086	pVCpu->cpum.GstCtx.SysEnter.esp = u64Tmp;
1087	READ_MSR(MSR_IA32_SYSENTER_CS, u64Tmp);
1088	pVCpu->cpum.GstCtx.SysEnter.cs = u64Tmp;
1089	}
1090	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1091	{
1092	READ_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1093	READ_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1094	READ_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1095	READ_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1096	}
1097	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1098	{
1099	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1100	READ_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1101	}
1102	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1103	{
1104	/* Last Branch Record. */
1105	if (pVM->nem.s.fLbr)
1106	{
1107	PVMXVMCSINFOSHARED const pVmcsInfoShared = &pVCpu->nem.s.vmx.VmcsInfo;
1108	uint32_t const idFromIpMsrStart = pVM->nem.s.idLbrFromIpMsrFirst;
1109	uint32_t const idToIpMsrStart = pVM->nem.s.idLbrToIpMsrFirst;
1110	uint32_t const idInfoMsrStart = pVM->nem.s.idLbrInfoMsrFirst;
1111	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
1112	Assert(cLbrStack <= 32);
1113	for (uint32_t i = 0; i < cLbrStack; i++)
1114	{
1115	READ_MSR(idFromIpMsrStart + i, pVmcsInfoShared->au64LbrFromIpMsr[i]);
1116
1117	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
1118	if (idToIpMsrStart != 0)
1119	READ_MSR(idToIpMsrStart + i, pVmcsInfoShared->au64LbrToIpMsr[i]);
1120	if (idInfoMsrStart != 0)
1121	READ_MSR(idInfoMsrStart + i, pVmcsInfoShared->au64LbrInfoMsr[i]);
1122	}
1123
1124	READ_MSR(pVM->nem.s.idLbrTosMsr, pVmcsInfoShared->u64LbrTosMsr);
1125
1126	if (pVM->nem.s.idLerFromIpMsr)
1127	READ_MSR(pVM->nem.s.idLerFromIpMsr, pVmcsInfoShared->u64LerFromIpMsr);
1128	if (pVM->nem.s.idLerToIpMsr)
1129	READ_MSR(pVM->nem.s.idLerToIpMsr, pVmcsInfoShared->u64LerToIpMsr);
1130	}
1131	}
1132
1133	/* Almost done, just update extrn flags and maybe change PGM mode. */
1134	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1135	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1136	pVCpu->cpum.GstCtx.fExtrn = 0;
1137
1138	#ifdef LOG_ENABLED
1139	nemR3DarwinLogState(pVM, pVCpu);
1140	#endif
1141
1142	/* Typical. */
1143	if (!fMaybeChangedMode && !fUpdateCr3)
1144	{
1145	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1146	return VINF_SUCCESS;
1147	}
1148
1149	/*
1150	* Slow.
1151	*/
1152	if (fMaybeChangedMode)
1153	{
1154	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
1155	false /* fForce */);
1156	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1157	}
1158
1159	if (fUpdateCr3)
1160	{
1161	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
1162	if (rc == VINF_SUCCESS)
1163	{ /* likely */ }
1164	else
1165	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1166	}
1167
1168	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1169
1170	return VINF_SUCCESS;
1171	#undef READ_GREG
1172	#undef READ_VMCS_FIELD
1173	#undef READ_VMCS32_FIELD
1174	#undef READ_SEG
1175	#undef READ_MSR
1176	}
1177
1178
1179	/**
1180	* State to pass between vmxHCExitEptViolation
1181	* and nemR3DarwinHandleMemoryAccessPageCheckerCallback.
1182	*/
1183	typedef struct NEMHCDARWINHMACPCCSTATE
1184	{
1185	/** Input: Write access. */
1186	bool fWriteAccess;
1187	/** Output: Set if we did something. */
1188	bool fDidSomething;
1189	/** Output: Set it we should resume. */
1190	bool fCanResume;
1191	} NEMHCDARWINHMACPCCSTATE;
1192
1193	/**
1194	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1195	* Worker for vmxHCExitEptViolation; pvUser points to a
1196	* NEMHCDARWINHMACPCCSTATE structure. }
1197	*/
1198	static DECLCALLBACK(int)
1199	nemR3DarwinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1200	{
1201	RT_NOREF(pVCpu);
1202
1203	NEMHCDARWINHMACPCCSTATE pState = (NEMHCDARWINHMACPCCSTATE )pvUser;
1204	pState->fDidSomething = false;
1205	pState->fCanResume = false;
1206
1207	uint8_t u2State = pInfo->u2NemState;
1208
1209	/*
1210	* Consolidate current page state with actual page protection and access type.
1211	* We don't really consider downgrades here, as they shouldn't happen.
1212	*/
1213	switch (u2State)
1214	{
1215	case NEM_DARWIN_PAGE_STATE_UNMAPPED:
1216	case NEM_DARWIN_PAGE_STATE_NOT_SET:
1217	{
1218	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1219	{
1220	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1221	return VINF_SUCCESS;
1222	}
1223
1224	/* Don't bother remapping it if it's a write request to a non-writable page. */
1225	if ( pState->fWriteAccess
1226	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1227	{
1228	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1229	return VINF_SUCCESS;
1230	}
1231
1232	int rc = VINF_SUCCESS;
1233	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1234	{
1235	void *pvPage;
1236	rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhys, &pvPage);
1237	if (RT_SUCCESS(rc))
1238	rc = nemR3DarwinMap(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, pvPage, X86_PAGE_SIZE, pInfo->fNemProt, &u2State);
1239	}
1240	else if (pInfo->fNemProt & NEM_PAGE_PROT_READ)
1241	{
1242	const void *pvPage;
1243	rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhys, &pvPage);
1244	if (RT_SUCCESS(rc))
1245	rc = nemR3DarwinMap(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, pvPage, X86_PAGE_SIZE, pInfo->fNemProt, &u2State);
1246	}
1247	else /* Only EXECUTE doesn't work. */
1248	AssertReleaseFailed();
1249
1250	pInfo->u2NemState = u2State;
1251	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1252	GCPhys, g_apszPageStates[u2State], rc));
1253	pState->fDidSomething = true;
1254	pState->fCanResume = true;
1255	return rc;
1256	}
1257	case NEM_DARWIN_PAGE_STATE_READABLE:
1258	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1259	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1260	{
1261	pState->fCanResume = true;
1262	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1263	return VINF_SUCCESS;
1264	}
1265	break;
1266
1267	case NEM_DARWIN_PAGE_STATE_WRITABLE:
1268	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1269	{
1270	pState->fCanResume = true;
1271	if (pInfo->u2OldNemState == NEM_DARWIN_PAGE_STATE_WRITABLE)
1272	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: Spurious EPT fault\n", GCPhys));
1273	return VINF_SUCCESS;
1274	}
1275	break;
1276
1277	default:
1278	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1279	}
1280
1281	/* Unmap and restart the instruction. */
1282	int rc = nemR3DarwinUnmap(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE, &u2State);
1283	if (RT_SUCCESS(rc))
1284	{
1285	pInfo->u2NemState = u2State;
1286	pState->fDidSomething = true;
1287	pState->fCanResume = true;
1288	Log5(("NEM GPA unmapped/exit: %RGp (was %s)\n", GCPhys, g_apszPageStates[u2State]));
1289	return VINF_SUCCESS;
1290	}
1291
1292	LogRel(("nemR3DarwinHandleMemoryAccessPageCheckerCallback/unmap: GCPhys=%RGp %s rc=%Rrc\n",
1293	GCPhys, g_apszPageStates[u2State], rc));
1294	return VERR_NEM_UNMAP_PAGES_FAILED;
1295	}
1296
1297
1298	DECL_FORCE_INLINE(bool) nemR3DarwinIsUnrestrictedGuest(PCVMCC pVM)
1299	{
1300	RT_NOREF(pVM);
1301	return true;
1302	}
1303
1304
1305	DECL_FORCE_INLINE(bool) nemR3DarwinIsNestedPaging(PCVMCC pVM)
1306	{
1307	RT_NOREF(pVM);
1308	return true;
1309	}
1310
1311
1312	DECL_FORCE_INLINE(bool) nemR3DarwinIsPreemptTimerUsed(PCVMCC pVM)
1313	{
1314	RT_NOREF(pVM);
1315	return false;
1316	}
1317
1318
1319	#if 0 /* unused */
1320	DECL_FORCE_INLINE(bool) nemR3DarwinIsVmxLbr(PCVMCC pVM)
1321	{
1322	RT_NOREF(pVM);
1323	return false;
1324	}
1325	#endif
1326
1327
1328	/*
1329	* Instantiate the code we share with ring-0.
1330	*/
1331	#define IN_NEM_DARWIN
1332	//#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
1333	//#define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
1334	//#define HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
1335	#define VCPU_2_VMXSTATE(a_pVCpu) (a_pVCpu)->nem.s
1336	#define VCPU_2_VMXSTATS(a_pVCpu) (*(a_pVCpu)->nem.s.pVmxStats)
1337
1338	#define VM_IS_VMX_UNRESTRICTED_GUEST(a_pVM) nemR3DarwinIsUnrestrictedGuest((a_pVM))
1339	#define VM_IS_VMX_NESTED_PAGING(a_pVM) nemR3DarwinIsNestedPaging((a_pVM))
1340	#define VM_IS_VMX_PREEMPT_TIMER_USED(a_pVM) nemR3DarwinIsPreemptTimerUsed((a_pVM))
1341	#define VM_IS_VMX_LBR(a_pVM) nemR3DarwinIsVmxLbr((a_pVM))
1342
1343	#define VMX_VMCS_WRITE_16(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs16((a_pVCpu), (a_FieldEnc), (a_Val))
1344	#define VMX_VMCS_WRITE_32(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs32((a_pVCpu), (a_FieldEnc), (a_Val))
1345	#define VMX_VMCS_WRITE_64(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1346	#define VMX_VMCS_WRITE_NW(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1347
1348	#define VMX_VMCS_READ_16(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs16((a_pVCpu), (a_FieldEnc), (a_pVal))
1349	#define VMX_VMCS_READ_32(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs32((a_pVCpu), (a_FieldEnc), (a_pVal))
1350	#define VMX_VMCS_READ_64(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1351	#define VMX_VMCS_READ_NW(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1352
1353	#include "../VMMAll/VMXAllTemplate.cpp.h"
1354
1355	#undef VMX_VMCS_WRITE_16
1356	#undef VMX_VMCS_WRITE_32
1357	#undef VMX_VMCS_WRITE_64
1358	#undef VMX_VMCS_WRITE_NW
1359
1360	#undef VMX_VMCS_READ_16
1361	#undef VMX_VMCS_READ_32
1362	#undef VMX_VMCS_READ_64
1363	#undef VMX_VMCS_READ_NW
1364
1365	#undef VM_IS_VMX_PREEMPT_TIMER_USED
1366	#undef VM_IS_VMX_NESTED_PAGING
1367	#undef VM_IS_VMX_UNRESTRICTED_GUEST
1368	#undef VCPU_2_VMXSTATS
1369	#undef VCPU_2_VMXSTATE
1370
1371
1372	/**
1373	* Exports the guest GP registers to HV for execution.
1374	*
1375	* @returns VBox status code.
1376	* @param pVCpu The cross context virtual CPU structure of the
1377	* calling EMT.
1378	*/
1379	static int nemR3DarwinExportGuestGprs(PVMCPUCC pVCpu)
1380	{
1381	#define WRITE_GREG(a_GReg, a_Value) \
1382	do \
1383	{ \
1384	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1385	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1386	{ /* likely */ } \
1387	else \
1388	return VERR_INTERNAL_ERROR; \
1389	} while(0)
1390
1391	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->nem.s.fCtxChanged);
1392	if (fCtxChanged & HM_CHANGED_GUEST_GPRS_MASK)
1393	{
1394	if (fCtxChanged & HM_CHANGED_GUEST_RAX)
1395	WRITE_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
1396	if (fCtxChanged & HM_CHANGED_GUEST_RCX)
1397	WRITE_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
1398	if (fCtxChanged & HM_CHANGED_GUEST_RDX)
1399	WRITE_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
1400	if (fCtxChanged & HM_CHANGED_GUEST_RBX)
1401	WRITE_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
1402	if (fCtxChanged & HM_CHANGED_GUEST_RSP)
1403	WRITE_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
1404	if (fCtxChanged & HM_CHANGED_GUEST_RBP)
1405	WRITE_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
1406	if (fCtxChanged & HM_CHANGED_GUEST_RSI)
1407	WRITE_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
1408	if (fCtxChanged & HM_CHANGED_GUEST_RDI)
1409	WRITE_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
1410	if (fCtxChanged & HM_CHANGED_GUEST_R8_R15)
1411	{
1412	WRITE_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
1413	WRITE_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
1414	WRITE_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
1415	WRITE_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
1416	WRITE_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
1417	WRITE_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
1418	WRITE_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
1419	WRITE_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
1420	}
1421
1422	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_GPRS_MASK);
1423	}
1424
1425	if (fCtxChanged & HM_CHANGED_GUEST_CR2)
1426	{
1427	WRITE_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
1428	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_CR2);
1429	}
1430
1431	return VINF_SUCCESS;
1432	#undef WRITE_GREG
1433	}
1434
1435
1436	/**
1437	* Exports the guest debug registers into the guest-state applying any hypervisor
1438	* debug related states (hardware breakpoints from the debugger, etc.).
1439	*
1440	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
1441	*
1442	* @returns VBox status code.
1443	* @param pVCpu The cross context virtual CPU structure.
1444	* @param pVmxTransient The VMX-transient structure.
1445	*/
1446	static int nemR3DarwinExportDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1447	{
1448	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1449
1450	#ifdef VBOX_STRICT
1451	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
1452	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
1453	{
1454	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
1455	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
1456	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
1457	}
1458	#endif
1459
1460	bool fSteppingDB = false;
1461	bool fInterceptMovDRx = false;
1462	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
1463	if (pVCpu->nem.s.fSingleInstruction)
1464	{
1465	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
1466	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
1467	{
1468	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
1469	Assert(fSteppingDB == false);
1470	}
1471	else
1472	{
1473	pVCpu->cpum.GstCtx.eflags.u \|= X86_EFL_TF;
1474	pVCpu->nem.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
1475	pVCpu->nem.s.fClearTrapFlag = true;
1476	fSteppingDB = true;
1477	}
1478	}
1479
1480	uint64_t u64GuestDr7;
1481	if ( fSteppingDB
1482	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1483	{
1484	/*
1485	* Use the combined guest and host DRx values found in the hypervisor register set
1486	* because the hypervisor debugger has breakpoints active or someone is single stepping
1487	* on the host side without a monitor trap flag.
1488	*
1489	* Note! DBGF expects a clean DR6 state before executing guest code.
1490	*/
1491	if (!CPUMIsHyperDebugStateActive(pVCpu))
1492	{
1493	/*
1494	* Make sure the hypervisor values are up to date.
1495	*/
1496	CPUMRecalcHyperDRx(pVCpu, UINT8_MAX /* no loading, please */);
1497
1498	CPUMR3NemActivateHyperDebugState(pVCpu);
1499
1500	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1501	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1502	}
1503
1504	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
1505	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
1506	pVCpu->nem.s.fUsingHyperDR7 = true;
1507	fInterceptMovDRx = true;
1508	}
1509	else
1510	{
1511	/*
1512	* If the guest has enabled debug registers, we need to load them prior to
1513	* executing guest code so they'll trigger at the right time.
1514	*/
1515	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
1516	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
1517	{
1518	if (!CPUMIsGuestDebugStateActive(pVCpu))
1519	{
1520	CPUMR3NemActivateGuestDebugState(pVCpu);
1521
1522	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1523	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1524	}
1525	Assert(!fInterceptMovDRx);
1526	}
1527	else if (!CPUMIsGuestDebugStateActive(pVCpu))
1528	{
1529	/*
1530	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
1531	* must intercept #DB in order to maintain a correct DR6 guest value, and
1532	* because we need to intercept it to prevent nested #DBs from hanging the
1533	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
1534	*/
1535	fInterceptMovDRx = true;
1536	}
1537
1538	/* Update DR7 with the actual guest value. */
1539	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
1540	pVCpu->nem.s.fUsingHyperDR7 = false;
1541	}
1542
1543	if (fInterceptMovDRx)
1544	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
1545	else
1546	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
1547
1548	/*
1549	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
1550	* monitor-trap flag and update our cache.
1551	*/
1552	if (uProcCtls != pVmcsInfo->u32ProcCtls)
1553	{
1554	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
1555	AssertRC(rc);
1556	pVmcsInfo->u32ProcCtls = uProcCtls;
1557	}
1558
1559	/*
1560	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
1561	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
1562	*
1563	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
1564	*/
1565	if (fSteppingDB)
1566	{
1567	Assert(pVCpu->nem.s.fSingleInstruction);
1568	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
1569
1570	uint32_t fIntrState = 0;
1571	int rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
1572	AssertRC(rc);
1573
1574	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
1575	{
1576	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
1577	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
1578	AssertRC(rc);
1579	}
1580	}
1581
1582	/*
1583	* Store status of the shared guest/host debug state at the time of VM-entry.
1584	*/
1585	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
1586	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
1587
1588	return VINF_SUCCESS;
1589	}
1590
1591
1592	/**
1593	* Converts the given CPUM externalized bitmask to the appropriate HM changed bitmask.
1594	*
1595	* @returns Bitmask of HM changed flags.
1596	* @param fCpumExtrn The CPUM extern bitmask.
1597	*/
1598	static uint64_t nemR3DarwinCpumExtrnToHmChanged(uint64_t fCpumExtrn)
1599	{
1600	uint64_t fHmChanged = 0;
1601
1602	/* Invert to gt a mask of things which are kept in CPUM. */
1603	uint64_t fCpumIntern = ~fCpumExtrn;
1604
1605	if (fCpumIntern & CPUMCTX_EXTRN_GPRS_MASK)
1606	{
1607	if (fCpumIntern & CPUMCTX_EXTRN_RAX)
1608	fHmChanged \|= HM_CHANGED_GUEST_RAX;
1609	if (fCpumIntern & CPUMCTX_EXTRN_RCX)
1610	fHmChanged \|= HM_CHANGED_GUEST_RCX;
1611	if (fCpumIntern & CPUMCTX_EXTRN_RDX)
1612	fHmChanged \|= HM_CHANGED_GUEST_RDX;
1613	if (fCpumIntern & CPUMCTX_EXTRN_RBX)
1614	fHmChanged \|= HM_CHANGED_GUEST_RBX;
1615	if (fCpumIntern & CPUMCTX_EXTRN_RSP)
1616	fHmChanged \|= HM_CHANGED_GUEST_RSP;
1617	if (fCpumIntern & CPUMCTX_EXTRN_RBP)
1618	fHmChanged \|= HM_CHANGED_GUEST_RBP;
1619	if (fCpumIntern & CPUMCTX_EXTRN_RSI)
1620	fHmChanged \|= HM_CHANGED_GUEST_RSI;
1621	if (fCpumIntern & CPUMCTX_EXTRN_RDI)
1622	fHmChanged \|= HM_CHANGED_GUEST_RDI;
1623	if (fCpumIntern & CPUMCTX_EXTRN_R8_R15)
1624	fHmChanged \|= HM_CHANGED_GUEST_R8_R15;
1625	}
1626
1627	/* RIP & Flags */
1628	if (fCpumIntern & CPUMCTX_EXTRN_RIP)
1629	fHmChanged \|= HM_CHANGED_GUEST_RIP;
1630	if (fCpumIntern & CPUMCTX_EXTRN_RFLAGS)
1631	fHmChanged \|= HM_CHANGED_GUEST_RFLAGS;
1632
1633	/* Segments */
1634	if (fCpumIntern & CPUMCTX_EXTRN_SREG_MASK)
1635	{
1636	if (fCpumIntern & CPUMCTX_EXTRN_ES)
1637	fHmChanged \|= HM_CHANGED_GUEST_ES;
1638	if (fCpumIntern & CPUMCTX_EXTRN_CS)
1639	fHmChanged \|= HM_CHANGED_GUEST_CS;
1640	if (fCpumIntern & CPUMCTX_EXTRN_SS)
1641	fHmChanged \|= HM_CHANGED_GUEST_SS;
1642	if (fCpumIntern & CPUMCTX_EXTRN_DS)
1643	fHmChanged \|= HM_CHANGED_GUEST_DS;
1644	if (fCpumIntern & CPUMCTX_EXTRN_FS)
1645	fHmChanged \|= HM_CHANGED_GUEST_FS;
1646	if (fCpumIntern & CPUMCTX_EXTRN_GS)
1647	fHmChanged \|= HM_CHANGED_GUEST_GS;
1648	}
1649
1650	/* Descriptor tables & task segment. */
1651	if (fCpumIntern & CPUMCTX_EXTRN_TABLE_MASK)
1652	{
1653	if (fCpumIntern & CPUMCTX_EXTRN_LDTR)
1654	fHmChanged \|= HM_CHANGED_GUEST_LDTR;
1655	if (fCpumIntern & CPUMCTX_EXTRN_TR)
1656	fHmChanged \|= HM_CHANGED_GUEST_TR;
1657	if (fCpumIntern & CPUMCTX_EXTRN_IDTR)
1658	fHmChanged \|= HM_CHANGED_GUEST_IDTR;
1659	if (fCpumIntern & CPUMCTX_EXTRN_GDTR)
1660	fHmChanged \|= HM_CHANGED_GUEST_GDTR;
1661	}
1662
1663	/* Control registers. */
1664	if (fCpumIntern & CPUMCTX_EXTRN_CR_MASK)
1665	{
1666	if (fCpumIntern & CPUMCTX_EXTRN_CR0)
1667	fHmChanged \|= HM_CHANGED_GUEST_CR0;
1668	if (fCpumIntern & CPUMCTX_EXTRN_CR2)
1669	fHmChanged \|= HM_CHANGED_GUEST_CR2;
1670	if (fCpumIntern & CPUMCTX_EXTRN_CR3)
1671	fHmChanged \|= HM_CHANGED_GUEST_CR3;
1672	if (fCpumIntern & CPUMCTX_EXTRN_CR4)
1673	fHmChanged \|= HM_CHANGED_GUEST_CR4;
1674	}
1675	if (fCpumIntern & CPUMCTX_EXTRN_APIC_TPR)
1676	fHmChanged \|= HM_CHANGED_GUEST_APIC_TPR;
1677
1678	/* Debug registers. */
1679	if (fCpumIntern & CPUMCTX_EXTRN_DR0_DR3)
1680	fHmChanged \|= HM_CHANGED_GUEST_DR0_DR3;
1681	if (fCpumIntern & CPUMCTX_EXTRN_DR6)
1682	fHmChanged \|= HM_CHANGED_GUEST_DR6;
1683	if (fCpumIntern & CPUMCTX_EXTRN_DR7)
1684	fHmChanged \|= HM_CHANGED_GUEST_DR7;
1685
1686	/* Floating point state. */
1687	if (fCpumIntern & CPUMCTX_EXTRN_X87)
1688	fHmChanged \|= HM_CHANGED_GUEST_X87;
1689	if (fCpumIntern & CPUMCTX_EXTRN_SSE_AVX)
1690	fHmChanged \|= HM_CHANGED_GUEST_SSE_AVX;
1691	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_XSAVE)
1692	fHmChanged \|= HM_CHANGED_GUEST_OTHER_XSAVE;
1693	if (fCpumIntern & CPUMCTX_EXTRN_XCRx)
1694	fHmChanged \|= HM_CHANGED_GUEST_XCRx;
1695
1696	/* MSRs */
1697	if (fCpumIntern & CPUMCTX_EXTRN_EFER)
1698	fHmChanged \|= HM_CHANGED_GUEST_EFER_MSR;
1699	if (fCpumIntern & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1700	fHmChanged \|= HM_CHANGED_GUEST_KERNEL_GS_BASE;
1701	if (fCpumIntern & CPUMCTX_EXTRN_SYSENTER_MSRS)
1702	fHmChanged \|= HM_CHANGED_GUEST_SYSENTER_MSR_MASK;
1703	if (fCpumIntern & CPUMCTX_EXTRN_SYSCALL_MSRS)
1704	fHmChanged \|= HM_CHANGED_GUEST_SYSCALL_MSRS;
1705	if (fCpumIntern & CPUMCTX_EXTRN_TSC_AUX)
1706	fHmChanged \|= HM_CHANGED_GUEST_TSC_AUX;
1707	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_MSRS)
1708	fHmChanged \|= HM_CHANGED_GUEST_OTHER_MSRS;
1709
1710	return fHmChanged;
1711	}
1712
1713
1714	/**
1715	* Exports the guest state to HV for execution.
1716	*
1717	* @returns VBox status code.
1718	* @param pVM The cross context VM structure.
1719	* @param pVCpu The cross context virtual CPU structure of the
1720	* calling EMT.
1721	* @param pVmxTransient The transient VMX structure.
1722	*/
1723	static int nemR3DarwinExportGuestState(PVMCC pVM, PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1724	{
1725	#define WRITE_GREG(a_GReg, a_Value) \
1726	do \
1727	{ \
1728	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1729	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1730	{ /* likely */ } \
1731	else \
1732	return VERR_INTERNAL_ERROR; \
1733	} while(0)
1734	#define WRITE_VMCS_FIELD(a_Field, a_Value) \
1735	do \
1736	{ \
1737	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), (a_Value)); \
1738	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1739	{ /* likely */ } \
1740	else \
1741	return VERR_INTERNAL_ERROR; \
1742	} while(0)
1743	#define WRITE_MSR(a_Msr, a_Value) \
1744	do \
1745	{ \
1746	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, (a_Msr), (a_Value)); \
1747	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1748	{ /* likely */ } \
1749	else \
1750	AssertFailedReturn(VERR_INTERNAL_ERROR); \
1751	} while(0)
1752
1753	RT_NOREF(pVM);
1754
1755	#ifdef LOG_ENABLED
1756	nemR3DarwinLogState(pVM, pVCpu);
1757	#endif
1758
1759	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateExport, x);
1760
1761	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL;
1762	if (!fWhat)
1763	return VINF_SUCCESS;
1764
1765	pVCpu->nem.s.fCtxChanged \|= nemR3DarwinCpumExtrnToHmChanged(pVCpu->cpum.GstCtx.fExtrn);
1766
1767	int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
1768	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1769
1770	rc = nemR3DarwinExportGuestGprs(pVCpu);
1771	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1772
1773	rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
1774	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1775
1776	VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
1777	if (rcStrict == VINF_SUCCESS)
1778	{ /* likely */ }
1779	else
1780	{
1781	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
1782	return VBOXSTRICTRC_VAL(rcStrict);
1783	}
1784
1785	rc = nemR3DarwinExportDebugState(pVCpu, pVmxTransient);
1786	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1787
1788	vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
1789	vmxHCExportGuestRip(pVCpu);
1790	//vmxHCExportGuestRsp(pVCpu);
1791	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
1792
1793	rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
1794	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1795
1796	if (fWhat & CPUMCTX_EXTRN_XCRx)
1797	{
1798	WRITE_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
1799	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_XCRx);
1800	}
1801
1802	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1803	{
1804	Assert(pVCpu->nem.s.fCtxChanged & HM_CHANGED_GUEST_APIC_TPR);
1805	vmxHCExportGuestApicTpr(pVCpu, pVmxTransient);
1806
1807	rc = APICGetTpr(pVCpu, &pVmxTransient->u8GuestTpr, NULL /pfPending/, NULL /pu8PendingIntr/);
1808	AssertRC(rc);
1809
1810	WRITE_GREG(HV_X86_TPR, pVmxTransient->u8GuestTpr);
1811	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1812	}
1813
1814	/* Debug registers. */
1815	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1816	{
1817	WRITE_GREG(HV_X86_DR0, CPUMGetHyperDR0(pVCpu));
1818	WRITE_GREG(HV_X86_DR1, CPUMGetHyperDR1(pVCpu));
1819	WRITE_GREG(HV_X86_DR2, CPUMGetHyperDR2(pVCpu));
1820	WRITE_GREG(HV_X86_DR3, CPUMGetHyperDR3(pVCpu));
1821	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR0_DR3);
1822	}
1823	if (fWhat & CPUMCTX_EXTRN_DR6)
1824	{
1825	WRITE_GREG(HV_X86_DR6, CPUMGetHyperDR6(pVCpu));
1826	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR6);
1827	}
1828	if (fWhat & CPUMCTX_EXTRN_DR7)
1829	{
1830	WRITE_GREG(HV_X86_DR7, CPUMGetHyperDR7(pVCpu));
1831	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR7);
1832	}
1833
1834	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1835	{
1836	hv_return_t hrc = hv_vcpu_write_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1837	if (hrc == HV_SUCCESS)
1838	{ /* likely */ }
1839	else
1840	return nemR3DarwinHvSts2Rc(hrc);
1841
1842	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~(HM_CHANGED_GUEST_X87 \| HM_CHANGED_GUEST_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE));
1843	}
1844
1845	/* MSRs */
1846	if (fWhat & CPUMCTX_EXTRN_EFER)
1847	{
1848	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, pVCpu->cpum.GstCtx.msrEFER);
1849	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
1850	}
1851	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1852	{
1853	WRITE_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1854	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_KERNEL_GS_BASE);
1855	}
1856	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1857	{
1858	WRITE_MSR(MSR_IA32_SYSENTER_CS, pVCpu->cpum.GstCtx.SysEnter.cs);
1859	WRITE_MSR(MSR_IA32_SYSENTER_EIP, pVCpu->cpum.GstCtx.SysEnter.eip);
1860	WRITE_MSR(MSR_IA32_SYSENTER_ESP, pVCpu->cpum.GstCtx.SysEnter.esp);
1861	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
1862	}
1863	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1864	{
1865	WRITE_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1866	WRITE_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1867	WRITE_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1868	WRITE_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1869	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSCALL_MSRS);
1870	}
1871	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1872	{
1873	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1874
1875	WRITE_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1876	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_TSC_AUX);
1877	}
1878	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1879	{
1880	/* Last Branch Record. */
1881	if (pVM->nem.s.fLbr)
1882	{
1883	PVMXVMCSINFOSHARED const pVmcsInfoShared = &pVCpu->nem.s.vmx.VmcsInfo;
1884	uint32_t const idFromIpMsrStart = pVM->nem.s.idLbrFromIpMsrFirst;
1885	uint32_t const idToIpMsrStart = pVM->nem.s.idLbrToIpMsrFirst;
1886	uint32_t const idInfoMsrStart = pVM->nem.s.idLbrInfoMsrFirst;
1887	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
1888	Assert(cLbrStack <= 32);
1889	for (uint32_t i = 0; i < cLbrStack; i++)
1890	{
1891	WRITE_MSR(idFromIpMsrStart + i, pVmcsInfoShared->au64LbrFromIpMsr[i]);
1892
1893	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
1894	if (idToIpMsrStart != 0)
1895	WRITE_MSR(idToIpMsrStart + i, pVmcsInfoShared->au64LbrToIpMsr[i]);
1896	if (idInfoMsrStart != 0)
1897	WRITE_MSR(idInfoMsrStart + i, pVmcsInfoShared->au64LbrInfoMsr[i]);
1898	}
1899
1900	WRITE_MSR(pVM->nem.s.idLbrTosMsr, pVmcsInfoShared->u64LbrTosMsr);
1901	if (pVM->nem.s.idLerFromIpMsr)
1902	WRITE_MSR(pVM->nem.s.idLerFromIpMsr, pVmcsInfoShared->u64LerFromIpMsr);
1903	if (pVM->nem.s.idLerToIpMsr)
1904	WRITE_MSR(pVM->nem.s.idLerToIpMsr, pVmcsInfoShared->u64LerToIpMsr);
1905	}
1906
1907	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
1908	}
1909
1910	hv_vcpu_invalidate_tlb(pVCpu->nem.s.hVCpuId);
1911	hv_vcpu_flush(pVCpu->nem.s.hVCpuId);
1912
1913	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_NEM;
1914
1915	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
1916	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~( HM_CHANGED_GUEST_HWVIRT
1917	\| HM_CHANGED_VMX_GUEST_AUTO_MSRS
1918	\| HM_CHANGED_VMX_GUEST_LAZY_MSRS
1919	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
1920
1921	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateExport, x);
1922	return VINF_SUCCESS;
1923	#undef WRITE_GREG
1924	#undef WRITE_VMCS_FIELD
1925	}
1926
1927
1928	/**
1929	* Common worker for both nemR3DarwinHandleExit() and nemR3DarwinHandleExitDebug().
1930	*
1931	* @returns VBox strict status code.
1932	* @param pVM The cross context VM structure.
1933	* @param pVCpu The cross context virtual CPU structure of the
1934	* calling EMT.
1935	* @param pVmxTransient The transient VMX structure.
1936	*/
1937	DECLINLINE(int) nemR3DarwinHandleExitCommon(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1938	{
1939	uint32_t uExitReason;
1940	int rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
1941	AssertRC(rc);
1942	pVmxTransient->fVmcsFieldsRead = 0;
1943	pVmxTransient->fIsNestedGuest = false;
1944	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
1945	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
1946
1947	if (RT_UNLIKELY(pVmxTransient->fVMEntryFailed))
1948	AssertLogRelMsgFailedReturn(("Running guest failed for CPU #%u: %#x %u\n",
1949	pVCpu->idCpu, pVmxTransient->uExitReason, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
1950	VERR_NEM_IPE_0);
1951
1952	/** @todo Only copy the state on demand (the R0 VT-x code saves some stuff unconditionally and the VMX template assumes that
1953	* when handling exits). */
1954	/*
1955	* Note! What is being fetched here must match the default value for the
1956	* a_fDonePostExit parameter of vmxHCImportGuestState exactly!
1957	*/
1958	rc = nemR3DarwinCopyStateFromHv(pVM, pVCpu, CPUMCTX_EXTRN_ALL);
1959	AssertRCReturn(rc, rc);
1960
1961	STAM_COUNTER_INC(&pVCpu->nem.s.pVmxStats->aStatExitReason[pVmxTransient->uExitReason & MASK_EXITREASON_STAT]);
1962	STAM_REL_COUNTER_INC(&pVCpu->nem.s.pVmxStats->StatExitAll);
1963	return VINF_SUCCESS;
1964	}
1965
1966
1967	/**
1968	* Handles an exit from hv_vcpu_run().
1969	*
1970	* @returns VBox strict status code.
1971	* @param pVM The cross context VM structure.
1972	* @param pVCpu The cross context virtual CPU structure of the
1973	* calling EMT.
1974	* @param pVmxTransient The transient VMX structure.
1975	*/
1976	static VBOXSTRICTRC nemR3DarwinHandleExit(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1977	{
1978	int rc = nemR3DarwinHandleExitCommon(pVM, pVCpu, pVmxTransient);
1979	AssertRCReturn(rc, rc);
1980
1981	#ifndef HMVMX_USE_FUNCTION_TABLE
1982	return vmxHCHandleExit(pVCpu, pVmxTransient);
1983	#else
1984	return g_aVMExitHandlers[pVmxTransient->uExitReason].pfn(pVCpu, pVmxTransient);
1985	#endif
1986	}
1987
1988
1989	/**
1990	* Handles an exit from hv_vcpu_run() - debug runloop variant.
1991	*
1992	* @returns VBox strict status code.
1993	* @param pVM The cross context VM structure.
1994	* @param pVCpu The cross context virtual CPU structure of the
1995	* calling EMT.
1996	* @param pVmxTransient The transient VMX structure.
1997	* @param pDbgState The debug state structure.
1998	*/
1999	static VBOXSTRICTRC nemR3DarwinHandleExitDebug(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
2000	{
2001	int rc = nemR3DarwinHandleExitCommon(pVM, pVCpu, pVmxTransient);
2002	AssertRCReturn(rc, rc);
2003
2004	return vmxHCRunDebugHandleExit(pVCpu, pVmxTransient, pDbgState);
2005	}
2006
2007
2008	/**
2009	* Worker for nemR3NativeInit that loads the Hypervisor.framework shared library.
2010	*
2011	* @returns VBox status code.
2012	* @param fForced Whether the HMForced flag is set and we should
2013	* fail if we cannot initialize.
2014	* @param pErrInfo Where to always return error info.
2015	*/
2016	static int nemR3DarwinLoadHv(bool fForced, PRTERRINFO pErrInfo)
2017	{
2018	RTLDRMOD hMod = NIL_RTLDRMOD;
2019	static const char *s_pszHvPath = "/System/Library/Frameworks/Hypervisor.framework/Hypervisor";
2020
2021	int rc = RTLdrLoadEx(s_pszHvPath, &hMod, RTLDRLOAD_FLAGS_NO_UNLOAD \| RTLDRLOAD_FLAGS_NO_SUFFIX, pErrInfo);
2022	if (RT_SUCCESS(rc))
2023	{
2024	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
2025	{
2026	int rc2 = RTLdrGetSymbol(hMod, g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
2027	if (RT_SUCCESS(rc2))
2028	{
2029	if (g_aImports[i].fOptional)
2030	LogRel(("NEM: info: Found optional import Hypervisor!%s.\n",
2031	g_aImports[i].pszName));
2032	}
2033	else
2034	{
2035	*g_aImports[i].ppfn = NULL;
2036
2037	LogRel(("NEM: %s: Failed to import Hypervisor!%s: %Rrc\n",
2038	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
2039	g_aImports[i].pszName, rc2));
2040	if (!g_aImports[i].fOptional)
2041	{
2042	if (RTErrInfoIsSet(pErrInfo))
2043	RTErrInfoAddF(pErrInfo, rc2, ", Hypervisor!%s", g_aImports[i].pszName);
2044	else
2045	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: Hypervisor!%s", g_aImports[i].pszName);
2046	Assert(RT_FAILURE(rc));
2047	}
2048	}
2049	}
2050	if (RT_SUCCESS(rc))
2051	{
2052	Assert(!RTErrInfoIsSet(pErrInfo));
2053	}
2054
2055	RTLdrClose(hMod);
2056	}
2057	else
2058	{
2059	RTErrInfoAddF(pErrInfo, rc, "Failed to load Hypervisor.framwork: %s: %Rrc", s_pszHvPath, rc);
2060	rc = VERR_NEM_INIT_FAILED;
2061	}
2062
2063	return rc;
2064	}
2065
2066
2067	/**
2068	* Read and initialize the global capabilities supported by this CPU.
2069	*
2070	* @returns VBox status code.
2071	*/
2072	static int nemR3DarwinCapsInit(void)
2073	{
2074	RT_ZERO(g_HmMsrs);
2075
2076	hv_return_t hrc = hv_vmx_read_capability(HV_VMX_CAP_PINBASED, &g_HmMsrs.u.vmx.PinCtls.u);
2077	if (hrc == HV_SUCCESS)
2078	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED, &g_HmMsrs.u.vmx.ProcCtls.u);
2079	if (hrc == HV_SUCCESS)
2080	hrc = hv_vmx_read_capability(HV_VMX_CAP_ENTRY, &g_HmMsrs.u.vmx.EntryCtls.u);
2081	if (hrc == HV_SUCCESS)
2082	hrc = hv_vmx_read_capability(HV_VMX_CAP_EXIT, &g_HmMsrs.u.vmx.ExitCtls.u);
2083	if (hrc == HV_SUCCESS)
2084	{
2085	hrc = hv_vmx_read_capability(HV_VMX_CAP_BASIC, &g_HmMsrs.u.vmx.u64Basic);
2086	if (hrc == HV_SUCCESS)
2087	{
2088	if (hrc == HV_SUCCESS)
2089	hrc = hv_vmx_read_capability(HV_VMX_CAP_MISC, &g_HmMsrs.u.vmx.u64Misc);
2090	if (hrc == HV_SUCCESS)
2091	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED0, &g_HmMsrs.u.vmx.u64Cr0Fixed0);
2092	if (hrc == HV_SUCCESS)
2093	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED1, &g_HmMsrs.u.vmx.u64Cr0Fixed1);
2094	if (hrc == HV_SUCCESS)
2095	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED0, &g_HmMsrs.u.vmx.u64Cr4Fixed0);
2096	if (hrc == HV_SUCCESS)
2097	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED1, &g_HmMsrs.u.vmx.u64Cr4Fixed1);
2098	if (hrc == HV_SUCCESS)
2099	hrc = hv_vmx_read_capability(HV_VMX_CAP_VMCS_ENUM, &g_HmMsrs.u.vmx.u64VmcsEnum);
2100	if ( hrc == HV_SUCCESS
2101	&& RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_TRUE_CTLS))
2102	{
2103	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PINBASED, &g_HmMsrs.u.vmx.TruePinCtls.u);
2104	if (hrc == HV_SUCCESS)
2105	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PROCBASED, &g_HmMsrs.u.vmx.TrueProcCtls.u);
2106	if (hrc == HV_SUCCESS)
2107	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_ENTRY, &g_HmMsrs.u.vmx.TrueEntryCtls.u);
2108	if (hrc == HV_SUCCESS)
2109	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_EXIT, &g_HmMsrs.u.vmx.TrueExitCtls.u);
2110	}
2111	}
2112	else
2113	{
2114	/* Likely running on anything < 11.0 (BigSur) so provide some sensible defaults. */
2115	g_HmMsrs.u.vmx.u64Cr0Fixed0 = 0x80000021;
2116	g_HmMsrs.u.vmx.u64Cr0Fixed1 = 0xffffffff;
2117	g_HmMsrs.u.vmx.u64Cr4Fixed0 = 0x2000;
2118	g_HmMsrs.u.vmx.u64Cr4Fixed1 = 0x1767ff;
2119	hrc = HV_SUCCESS;
2120	}
2121	}
2122
2123	if ( hrc == HV_SUCCESS
2124	&& g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2125	{
2126	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED2, &g_HmMsrs.u.vmx.ProcCtls2.u);
2127
2128	if ( hrc == HV_SUCCESS
2129	&& g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & (VMX_PROC_CTLS2_EPT \| VMX_PROC_CTLS2_VPID))
2130	{
2131	hrc = hv_vmx_read_capability(HV_VMX_CAP_EPT_VPID_CAP, &g_HmMsrs.u.vmx.u64EptVpidCaps);
2132	if (hrc != HV_SUCCESS)
2133	hrc = HV_SUCCESS; /* Probably just outdated OS. */
2134	}
2135
2136	g_HmMsrs.u.vmx.u64VmFunc = 0; /* No way to read that on macOS. */
2137	}
2138
2139	if (hrc == HV_SUCCESS)
2140	{
2141	/*
2142	* Check for EFER swapping support.
2143	*/
2144	g_fHmVmxSupportsVmcsEfer = true; //(g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2145	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2146	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
2147	}
2148
2149	/*
2150	* Get MSR_IA32_ARCH_CAPABILITIES and expand it into the host feature structure.
2151	* This is only available with 11.0+ (BigSur) as the required API is only available there,
2152	* we could in theory initialize this when creating the EMTs using hv_vcpu_read_msr() but
2153	* the required vCPU handle is created after CPUM was initialized which is too late.
2154	* Given that the majority of users is on 11.0 and later we don't care for now.
2155	*/
2156	if ( hrc == HV_SUCCESS
2157	&& hv_vmx_get_msr_info)
2158	{
2159	g_CpumHostFeatures.s.fArchRdclNo = 0;
2160	g_CpumHostFeatures.s.fArchIbrsAll = 0;
2161	g_CpumHostFeatures.s.fArchRsbOverride = 0;
2162	g_CpumHostFeatures.s.fArchVmmNeedNotFlushL1d = 0;
2163	g_CpumHostFeatures.s.fArchMdsNo = 0;
2164	uint32_t const cStdRange = ASMCpuId_EAX(0);
2165	if ( RTX86IsValidStdRange(cStdRange)
2166	&& cStdRange >= 7)
2167	{
2168	uint32_t const fStdFeaturesEdx = ASMCpuId_EDX(1);
2169	uint32_t fStdExtFeaturesEdx;
2170	ASMCpuIdExSlow(7, 0, 0, 0, NULL, NULL, NULL, &fStdExtFeaturesEdx);
2171	if ( (fStdExtFeaturesEdx & X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP)
2172	&& (fStdFeaturesEdx & X86_CPUID_FEATURE_EDX_MSR))
2173	{
2174	uint64_t fArchVal;
2175	hrc = hv_vmx_get_msr_info(HV_VMX_INFO_MSR_IA32_ARCH_CAPABILITIES, &fArchVal);
2176	if (hrc == HV_SUCCESS)
2177	{
2178	g_CpumHostFeatures.s.fArchRdclNo = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RDCL_NO);
2179	g_CpumHostFeatures.s.fArchIbrsAll = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_IBRS_ALL);
2180	g_CpumHostFeatures.s.fArchRsbOverride = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RSBO);
2181	g_CpumHostFeatures.s.fArchVmmNeedNotFlushL1d = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_VMM_NEED_NOT_FLUSH_L1D);
2182	g_CpumHostFeatures.s.fArchMdsNo = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_MDS_NO);
2183	}
2184	}
2185	else
2186	g_CpumHostFeatures.s.fArchCap = 0;
2187	}
2188	}
2189
2190	return nemR3DarwinHvSts2Rc(hrc);
2191	}
2192
2193
2194	/**
2195	* Sets up the LBR MSR ranges based on the host CPU.
2196	*
2197	* @returns VBox status code.
2198	* @param pVM The cross context VM structure.
2199	*
2200	* @sa hmR0VmxSetupLbrMsrRange
2201	*/
2202	static int nemR3DarwinSetupLbrMsrRange(PVMCC pVM)
2203	{
2204	Assert(pVM->nem.s.fLbr);
2205	uint32_t idLbrFromIpMsrFirst;
2206	uint32_t idLbrFromIpMsrLast;
2207	uint32_t idLbrToIpMsrFirst;
2208	uint32_t idLbrToIpMsrLast;
2209	uint32_t idLbrInfoMsrFirst;
2210	uint32_t idLbrInfoMsrLast;
2211	uint32_t idLbrTosMsr;
2212	uint32_t idLbrSelectMsr;
2213	uint32_t idLerFromIpMsr;
2214	uint32_t idLerToIpMsr;
2215
2216	/*
2217	* Determine the LBR MSRs supported for this host CPU family and model.
2218	*
2219	* See Intel spec. 17.4.8 "LBR Stack".
2220	* See Intel "Model-Specific Registers" spec.
2221	*/
2222	uint32_t const uFamilyModel = (g_CpumHostFeatures.s.uFamily << 8)
2223	\| g_CpumHostFeatures.s.uModel;
2224	switch (uFamilyModel)
2225	{
2226	case 0x0f01: case 0x0f02:
2227	idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
2228	idLbrFromIpMsrLast = MSR_P4_LASTBRANCH_3;
2229	idLbrToIpMsrFirst = 0x0;
2230	idLbrToIpMsrLast = 0x0;
2231	idLbrInfoMsrFirst = 0x0;
2232	idLbrInfoMsrLast = 0x0;
2233	idLbrTosMsr = MSR_P4_LASTBRANCH_TOS;
2234	idLbrSelectMsr = 0x0;
2235	idLerFromIpMsr = 0x0;
2236	idLerToIpMsr = 0x0;
2237	break;
2238
2239	case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
2240	case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
2241	case 0x066a: case 0x066c: case 0x067d: case 0x067e:
2242	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2243	idLbrFromIpMsrLast = MSR_LASTBRANCH_31_FROM_IP;
2244	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2245	idLbrToIpMsrLast = MSR_LASTBRANCH_31_TO_IP;
2246	idLbrInfoMsrFirst = MSR_LASTBRANCH_0_INFO;
2247	idLbrInfoMsrLast = MSR_LASTBRANCH_31_INFO;
2248	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2249	idLbrSelectMsr = MSR_LASTBRANCH_SELECT;
2250	idLerFromIpMsr = MSR_LER_FROM_IP;
2251	idLerToIpMsr = MSR_LER_TO_IP;
2252	break;
2253
2254	case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
2255	case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
2256	case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
2257	case 0x062e: case 0x0625: case 0x062c: case 0x062f:
2258	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2259	idLbrFromIpMsrLast = MSR_LASTBRANCH_15_FROM_IP;
2260	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2261	idLbrToIpMsrLast = MSR_LASTBRANCH_15_TO_IP;
2262	idLbrInfoMsrFirst = MSR_LASTBRANCH_0_INFO;
2263	idLbrInfoMsrLast = MSR_LASTBRANCH_15_INFO;
2264	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2265	idLbrSelectMsr = MSR_LASTBRANCH_SELECT;
2266	idLerFromIpMsr = MSR_LER_FROM_IP;
2267	idLerToIpMsr = MSR_LER_TO_IP;
2268	break;
2269
2270	case 0x0617: case 0x061d: case 0x060f:
2271	idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
2272	idLbrFromIpMsrLast = MSR_CORE2_LASTBRANCH_3_FROM_IP;
2273	idLbrToIpMsrFirst = MSR_CORE2_LASTBRANCH_0_TO_IP;
2274	idLbrToIpMsrLast = MSR_CORE2_LASTBRANCH_3_TO_IP;
2275	idLbrInfoMsrFirst = 0x0;
2276	idLbrInfoMsrLast = 0x0;
2277	idLbrTosMsr = MSR_CORE2_LASTBRANCH_TOS;
2278	idLbrSelectMsr = 0x0;
2279	idLerFromIpMsr = 0x0;
2280	idLerToIpMsr = 0x0;
2281	break;
2282
2283	/* Atom and related microarchitectures we don't care about:
2284	case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
2285	case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
2286	case 0x0636: */
2287	/* All other CPUs: */
2288	default:
2289	{
2290	LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
2291	VMCC_GET_CPU_0(pVM)->nem.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
2292	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2293	}
2294	}
2295
2296	/*
2297	* Validate.
2298	*/
2299	uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
2300	PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
2301	AssertCompile( RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrFromIpMsr)
2302	== RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrToIpMsr));
2303	AssertCompile( RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrFromIpMsr)
2304	== RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrInfoMsr));
2305	if (cLbrStack > RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrFromIpMsr))
2306	{
2307	LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
2308	VMCC_GET_CPU_0(pVM)->nem.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
2309	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2310	}
2311	NOREF(pVCpu0);
2312
2313	/*
2314	* Update the LBR info. to the VM struct. for use later.
2315	*/
2316	pVM->nem.s.idLbrTosMsr = idLbrTosMsr;
2317	pVM->nem.s.idLbrSelectMsr = idLbrSelectMsr;
2318
2319	pVM->nem.s.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
2320	pVM->nem.s.idLbrFromIpMsrLast = idLbrFromIpMsrLast;
2321
2322	pVM->nem.s.idLbrToIpMsrFirst = idLbrToIpMsrFirst;
2323	pVM->nem.s.idLbrToIpMsrLast = idLbrToIpMsrLast;
2324
2325	pVM->nem.s.idLbrInfoMsrFirst = idLbrInfoMsrFirst;
2326	pVM->nem.s.idLbrInfoMsrLast = idLbrInfoMsrLast;
2327
2328	pVM->nem.s.idLerFromIpMsr = idLerFromIpMsr;
2329	pVM->nem.s.idLerToIpMsr = idLerToIpMsr;
2330	return VINF_SUCCESS;
2331	}
2332
2333
2334	/**
2335	* Sets up pin-based VM-execution controls in the VMCS.
2336	*
2337	* @returns VBox status code.
2338	* @param pVCpu The cross context virtual CPU structure.
2339	* @param pVmcsInfo The VMCS info. object.
2340	*/
2341	static int nemR3DarwinVmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2342	{
2343	//PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2344	uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
2345	uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2346
2347	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2348	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2349
2350	#if 0 /** @todo Use preemption timer */
2351	/* Enable the VMX-preemption timer. */
2352	if (pVM->hmr0.s.vmx.fUsePreemptTimer)
2353	{
2354	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2355	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2356	}
2357
2358	/* Enable posted-interrupt processing. */
2359	if (pVM->hm.s.fPostedIntrs)
2360	{
2361	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2362	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2363	fVal \|= VMX_PIN_CTLS_POSTED_INT;
2364	}
2365	#endif
2366
2367	if ((fVal & fZap) != fVal)
2368	{
2369	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2370	g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
2371	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2372	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2373	}
2374
2375	/* Commit it to the VMCS and update our cache. */
2376	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2377	AssertRC(rc);
2378	pVmcsInfo->u32PinCtls = fVal;
2379
2380	return VINF_SUCCESS;
2381	}
2382
2383
2384	/**
2385	* Sets up secondary processor-based VM-execution controls in the VMCS.
2386	*
2387	* @returns VBox status code.
2388	* @param pVCpu The cross context virtual CPU structure.
2389	* @param pVmcsInfo The VMCS info. object.
2390	*/
2391	static int nemR3DarwinVmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2392	{
2393	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2394	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2395	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2396
2397	/* WBINVD causes a VM-exit. */
2398	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2399	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2400
2401	/* Enable the INVPCID instruction if we expose it to the guest and is supported
2402	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
2403	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
2404	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
2405	fVal \|= VMX_PROC_CTLS2_INVPCID;
2406
2407	#if 0 /** @todo */
2408	/* Enable VPID. */
2409	if (pVM->hmr0.s.vmx.fVpid)
2410	fVal \|= VMX_PROC_CTLS2_VPID;
2411
2412	if (pVM->hm.s.fVirtApicRegs)
2413	{
2414	/* Enable APIC-register virtualization. */
2415	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2416	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2417
2418	/* Enable virtual-interrupt delivery. */
2419	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2420	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2421	}
2422
2423	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
2424	where the TPR shadow resides. */
2425	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2426	* done dynamically. */
2427	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2428	{
2429	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2430	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
2431	}
2432	#endif
2433
2434	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
2435	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
2436	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
2437	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
2438	fVal \|= VMX_PROC_CTLS2_RDTSCP;
2439
2440	/* Enable Pause-Loop exiting. */
2441	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
2442	&& pVM->nem.s.cPleGapTicks
2443	&& pVM->nem.s.cPleWindowTicks)
2444	{
2445	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2446
2447	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PLE_GAP, pVM->nem.s.cPleGapTicks); AssertRC(rc);
2448	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PLE_WINDOW, pVM->nem.s.cPleWindowTicks); AssertRC(rc);
2449	}
2450
2451	if ((fVal & fZap) != fVal)
2452	{
2453	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2454	g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
2455	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2456	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2457	}
2458
2459	/* Commit it to the VMCS and update our cache. */
2460	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2461	AssertRC(rc);
2462	pVmcsInfo->u32ProcCtls2 = fVal;
2463
2464	return VINF_SUCCESS;
2465	}
2466
2467
2468	/**
2469	* Enables native access for the given MSR.
2470	*
2471	* @returns VBox status code.
2472	* @param pVCpu The cross context virtual CPU structure.
2473	* @param idMsr The MSR to enable native access for.
2474	*/
2475	static int nemR3DarwinMsrSetNative(PVMCPUCC pVCpu, uint32_t idMsr)
2476	{
2477	hv_return_t hrc = hv_vcpu_enable_native_msr(pVCpu->nem.s.hVCpuId, idMsr, true /enable/);
2478	if (hrc == HV_SUCCESS)
2479	return VINF_SUCCESS;
2480
2481	return nemR3DarwinHvSts2Rc(hrc);
2482	}
2483
2484
2485	/**
2486	* Sets the MSR to managed for the given vCPU allowing the guest to access it.
2487	*
2488	* @returns VBox status code.
2489	* @param pVCpu The cross context virtual CPU structure.
2490	* @param idMsr The MSR to enable managed access for.
2491	* @param fMsrPerm The MSR permissions flags.
2492	*/
2493	static int nemR3DarwinMsrSetManaged(PVMCPUCC pVCpu, uint32_t idMsr, hv_msr_flags_t fMsrPerm)
2494	{
2495	Assert(hv_vcpu_enable_managed_msr);
2496
2497	hv_return_t hrc = hv_vcpu_enable_managed_msr(pVCpu->nem.s.hVCpuId, idMsr, true /enable/);
2498	if (hrc == HV_SUCCESS)
2499	{
2500	hrc = hv_vcpu_set_msr_access(pVCpu->nem.s.hVCpuId, idMsr, fMsrPerm);
2501	if (hrc == HV_SUCCESS)
2502	return VINF_SUCCESS;
2503	}
2504
2505	return nemR3DarwinHvSts2Rc(hrc);
2506	}
2507
2508
2509	/**
2510	* Sets up the MSR permissions which don't change through the lifetime of the VM.
2511	*
2512	* @returns VBox status code.
2513	* @param pVCpu The cross context virtual CPU structure.
2514	* @param pVmcsInfo The VMCS info. object.
2515	*/
2516	static int nemR3DarwinSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2517	{
2518	RT_NOREF(pVmcsInfo);
2519
2520	/*
2521	* The guest can access the following MSRs (read, write) without causing
2522	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2523	*/
2524	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2525	int rc;
2526	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_CS); AssertRCReturn(rc, rc);
2527	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_ESP); AssertRCReturn(rc, rc);
2528	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_EIP); AssertRCReturn(rc, rc);
2529	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_GS_BASE); AssertRCReturn(rc, rc);
2530	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_FS_BASE); AssertRCReturn(rc, rc);
2531
2532	/*
2533	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2534	* associated with then. We never need to intercept access (writes need to be
2535	* executed without causing a VM-exit, reads will #GP fault anyway).
2536	*
2537	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2538	* read/write them. We swap the guest/host MSR value using the
2539	* auto-load/store MSR area.
2540	*/
2541	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2542	{
2543	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_PRED_CMD);
2544	AssertRCReturn(rc, rc);
2545	}
2546	#if 0 /* Doesn't work. */
2547	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2548	{
2549	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_FLUSH_CMD);
2550	AssertRCReturn(rc, rc);
2551	}
2552	#endif
2553	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2554	{
2555	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SPEC_CTRL);
2556	AssertRCReturn(rc, rc);
2557	}
2558
2559	/*
2560	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2561	* required for 64-bit guests.
2562	*/
2563	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_LSTAR); AssertRCReturn(rc, rc);
2564	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K6_STAR); AssertRCReturn(rc, rc);
2565	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_SF_MASK); AssertRCReturn(rc, rc);
2566	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_KERNEL_GS_BASE); AssertRCReturn(rc, rc);
2567
2568	/* Required for enabling the RDTSCP instruction. */
2569	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_TSC_AUX); AssertRCReturn(rc, rc);
2570
2571	/* Last Branch Record. */
2572	if (pVM->nem.s.fLbr)
2573	{
2574	uint32_t const idFromIpMsrStart = pVM->nem.s.idLbrFromIpMsrFirst;
2575	uint32_t const idToIpMsrStart = pVM->nem.s.idLbrToIpMsrFirst;
2576	uint32_t const idInfoMsrStart = pVM->nem.s.idLbrInfoMsrFirst;
2577	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
2578	Assert(cLbrStack <= 32);
2579	for (uint32_t i = 0; i < cLbrStack; i++)
2580	{
2581	rc = nemR3DarwinMsrSetManaged(pVCpu, idFromIpMsrStart + i, HV_MSR_READ \| HV_MSR_WRITE);
2582	AssertRCReturn(rc, rc);
2583
2584	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
2585	if (idToIpMsrStart != 0)
2586	{
2587	rc = nemR3DarwinMsrSetManaged(pVCpu, idToIpMsrStart + i, HV_MSR_READ \| HV_MSR_WRITE);
2588	AssertRCReturn(rc, rc);
2589	}
2590
2591	if (idInfoMsrStart != 0)
2592	{
2593	rc = nemR3DarwinMsrSetManaged(pVCpu, idInfoMsrStart + i, HV_MSR_READ \| HV_MSR_WRITE);
2594	AssertRCReturn(rc, rc);
2595	}
2596	}
2597
2598	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLbrTosMsr, HV_MSR_READ \| HV_MSR_WRITE);
2599	AssertRCReturn(rc, rc);
2600
2601	if (pVM->nem.s.idLerFromIpMsr)
2602	{
2603	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLerFromIpMsr, HV_MSR_READ \| HV_MSR_WRITE);
2604	AssertRCReturn(rc, rc);
2605	}
2606
2607	if (pVM->nem.s.idLerToIpMsr)
2608	{
2609	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLerToIpMsr, HV_MSR_READ \| HV_MSR_WRITE);
2610	AssertRCReturn(rc, rc);
2611	}
2612
2613	if (pVM->nem.s.idLbrSelectMsr)
2614	{
2615	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLbrSelectMsr, HV_MSR_READ \| HV_MSR_WRITE);
2616	AssertRCReturn(rc, rc);
2617	}
2618	}
2619
2620	return VINF_SUCCESS;
2621	}
2622
2623
2624	/**
2625	* Sets up processor-based VM-execution controls in the VMCS.
2626	*
2627	* @returns VBox status code.
2628	* @param pVCpu The cross context virtual CPU structure.
2629	* @param pVmcsInfo The VMCS info. object.
2630	*/
2631	static int nemR3DarwinVmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2632	{
2633	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2634	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2635
2636	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2637	// \| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2638	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2639	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2640	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2641	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2642	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2643
2644	#ifdef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
2645	fVal \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
2646	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2647	#endif
2648
2649	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2650	if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2651	\|\| (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2652	{
2653	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2654	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2655	}
2656
2657	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2658	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2659	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2660
2661	if ((fVal & fZap) != fVal)
2662	{
2663	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2664	g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2665	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2666	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2667	}
2668
2669	/* Commit it to the VMCS and update our cache. */
2670	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2671	AssertRC(rc);
2672	pVmcsInfo->u32ProcCtls = fVal;
2673
2674	/* Set up MSR permissions that don't change through the lifetime of the VM. */
2675	rc = nemR3DarwinSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2676	AssertRCReturn(rc, rc);
2677
2678	/*
2679	* Set up secondary processor-based VM-execution controls
2680	* (we assume the CPU to always support it as we rely on unrestricted guest execution support).
2681	*/
2682	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
2683	return nemR3DarwinVmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2684	}
2685
2686
2687	/**
2688	* Sets up miscellaneous (everything other than Pin, Processor and secondary
2689	* Processor-based VM-execution) control fields in the VMCS.
2690	*
2691	* @returns VBox status code.
2692	* @param pVCpu The cross context virtual CPU structure.
2693	* @param pVmcsInfo The VMCS info. object.
2694	*/
2695	static int nemR3DarwinVmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2696	{
2697	int rc = VINF_SUCCESS;
2698	//rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo); TODO
2699	if (RT_SUCCESS(rc))
2700	{
2701	uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2702	uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2703
2704	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2705	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2706
2707	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2708	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2709
2710	if (pVCpu->CTX_SUFF(pVM)->nem.s.fLbr)
2711	{
2712	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2713	AssertRC(rc);
2714	}
2715	return VINF_SUCCESS;
2716	}
2717	else
2718	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2719	return rc;
2720	}
2721
2722
2723	/**
2724	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2725	*
2726	* We shall setup those exception intercepts that don't change during the
2727	* lifetime of the VM here. The rest are done dynamically while loading the
2728	* guest state.
2729	*
2730	* @param pVCpu The cross context virtual CPU structure.
2731	* @param pVmcsInfo The VMCS info. object.
2732	*/
2733	static void nemR3DarwinVmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2734	{
2735	/*
2736	* The following exceptions are always intercepted:
2737	*
2738	* #AC - To prevent the guest from hanging the CPU and for dealing with
2739	* split-lock detecting host configs.
2740	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2741	* recursive #DBs can cause a CPU hang.
2742	*/
2743	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2744	\| RT_BIT(X86_XCPT_DB);
2745
2746	/* Commit it to the VMCS. */
2747	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2748	AssertRC(rc);
2749
2750	/* Update our cache of the exception bitmap. */
2751	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2752	}
2753
2754
2755	/**
2756	* Initialize the VMCS information field for the given vCPU.
2757	*
2758	* @returns VBox status code.
2759	* @param pVCpu The cross context virtual CPU structure of the
2760	* calling EMT.
2761	*/
2762	static int nemR3DarwinInitVmcs(PVMCPU pVCpu)
2763	{
2764	int rc = nemR3DarwinVmxSetupVmcsPinCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2765	if (RT_SUCCESS(rc))
2766	{
2767	rc = nemR3DarwinVmxSetupVmcsProcCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2768	if (RT_SUCCESS(rc))
2769	{
2770	rc = nemR3DarwinVmxSetupVmcsMiscCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2771	if (RT_SUCCESS(rc))
2772	{
2773	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &pVCpu->nem.s.VmcsInfo.u32EntryCtls);
2774	if (RT_SUCCESS(rc))
2775	{
2776	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_EXIT, &pVCpu->nem.s.VmcsInfo.u32ExitCtls);
2777	if (RT_SUCCESS(rc))
2778	{
2779	nemR3DarwinVmxSetupVmcsXcptBitmap(pVCpu, &pVCpu->nem.s.VmcsInfo);
2780	return VINF_SUCCESS;
2781	}
2782	LogRelFunc(("Failed to read the exit controls. rc=%Rrc\n", rc));
2783	}
2784	else
2785	LogRelFunc(("Failed to read the entry controls. rc=%Rrc\n", rc));
2786	}
2787	else
2788	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
2789	}
2790	else
2791	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
2792	}
2793	else
2794	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
2795
2796	return rc;
2797	}
2798
2799
2800	/**
2801	* Registers statistics for the given vCPU.
2802	*
2803	* @returns VBox status code.
2804	* @param pVM The cross context VM structure.
2805	* @param idCpu The CPU ID.
2806	* @param pNemCpu The NEM CPU structure.
2807	*/
2808	static int nemR3DarwinStatisticsRegister(PVM pVM, VMCPUID idCpu, PNEMCPU pNemCpu)
2809	{
2810	#define NEM_REG_STAT(a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szNmFmt, a_szDesc) do { \
2811	int rc = STAMR3RegisterF(pVM, a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szDesc, a_szNmFmt, idCpu); \
2812	AssertRC(rc); \
2813	} while (0)
2814	#define NEM_REG_PROFILE(a_pVar, a_szNmFmt, a_szDesc) \
2815	NEM_REG_STAT(a_pVar, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, a_szNmFmt, a_szDesc)
2816	#define NEM_REG_COUNTER(a, b, desc) NEM_REG_STAT(a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, b, desc)
2817
2818	PVMXSTATISTICS const pVmxStats = pNemCpu->pVmxStats;
2819
2820	NEM_REG_COUNTER(&pVmxStats->StatExitCR0Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR0", "CR0 read.");
2821	NEM_REG_COUNTER(&pVmxStats->StatExitCR2Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR2", "CR2 read.");
2822	NEM_REG_COUNTER(&pVmxStats->StatExitCR3Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR3", "CR3 read.");
2823	NEM_REG_COUNTER(&pVmxStats->StatExitCR4Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR4", "CR4 read.");
2824	NEM_REG_COUNTER(&pVmxStats->StatExitCR8Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR8", "CR8 read.");
2825	NEM_REG_COUNTER(&pVmxStats->StatExitCR0Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR0", "CR0 write.");
2826	NEM_REG_COUNTER(&pVmxStats->StatExitCR2Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR2", "CR2 write.");
2827	NEM_REG_COUNTER(&pVmxStats->StatExitCR3Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR3", "CR3 write.");
2828	NEM_REG_COUNTER(&pVmxStats->StatExitCR4Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR4", "CR4 write.");
2829	NEM_REG_COUNTER(&pVmxStats->StatExitCR8Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR8", "CR8 write.");
2830
2831	NEM_REG_COUNTER(&pVmxStats->StatExitAll, "/NEM/CPU%u/Exit/All", "Total exits (including nested-guest exits).");
2832
2833	NEM_REG_COUNTER(&pVmxStats->StatImportGuestStateFallback, "/NEM/CPU%u/ImportGuestStateFallback", "Times vmxHCImportGuestState took the fallback code path.");
2834	NEM_REG_COUNTER(&pVmxStats->StatReadToTransientFallback, "/NEM/CPU%u/ReadToTransientFallback", "Times vmxHCReadToTransient took the fallback code path.");
2835
2836	#ifdef VBOX_WITH_STATISTICS
2837	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateImport, "/NEM/CPU%u/ImportGuestState", "Profiling of importing guest state from hardware after VM-exit.");
2838	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateExport, "/NEM/CPU%u/ExportGuestState", "Profiling of exporting guest state from hardware after VM-exit.");
2839
2840	for (int j = 0; j < MAX_EXITREASON_STAT; j++)
2841	{
2842	const char *pszExitName = HMGetVmxExitName(j);
2843	if (pszExitName)
2844	{
2845	int rc = STAMR3RegisterF(pVM, &pVmxStats->aStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
2846	STAMUNIT_OCCURENCES, pszExitName, "/NEM/CPU%u/Exit/Reason/%02x", idCpu, j);
2847	AssertRCReturn(rc, rc);
2848	}
2849	}
2850	#endif
2851
2852	return VINF_SUCCESS;
2853
2854	#undef NEM_REG_COUNTER
2855	#undef NEM_REG_PROFILE
2856	#undef NEM_REG_STAT
2857	}
2858
2859
2860	/**
2861	* Displays the HM Last-Branch-Record info. for the guest.
2862	*
2863	* @param pVM The cross context VM structure.
2864	* @param pHlp The info helper functions.
2865	* @param pszArgs Arguments, ignored.
2866	*/
2867	static DECLCALLBACK(void) nemR3DarwinInfoLbr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2868	{
2869	NOREF(pszArgs);
2870	PVMCPU pVCpu = VMMGetCpu(pVM);
2871	if (!pVCpu)
2872	pVCpu = pVM->apCpusR3[0];
2873
2874	Assert(pVM->nem.s.fLbr);
2875
2876	PCVMXVMCSINFOSHARED pVmcsInfoShared = &pVCpu->nem.s.vmx.VmcsInfo;
2877	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
2878
2879	/** @todo r=ramshankar: The index technically varies depending on the CPU, but
2880	* 0xf should cover everything we support thus far. Fix if necessary
2881	* later. */
2882	uint32_t const idxTopOfStack = pVmcsInfoShared->u64LbrTosMsr & 0xf;
2883	if (idxTopOfStack > cLbrStack)
2884	{
2885	pHlp->pfnPrintf(pHlp, "Top-of-stack LBR MSR seems corrupt (index=%u, msr=%#RX64) expected index < %u\n",
2886	idxTopOfStack, pVmcsInfoShared->u64LbrTosMsr, cLbrStack);
2887	return;
2888	}
2889
2890	/*
2891	* Dump the circular buffer of LBR records starting from the most recent record (contained in idxTopOfStack).
2892	*/
2893	pHlp->pfnPrintf(pHlp, "CPU[%u]: LBRs (most-recent first)\n", pVCpu->idCpu);
2894	if (pVM->nem.s.idLerFromIpMsr)
2895	pHlp->pfnPrintf(pHlp, "LER: From IP=%#016RX64 - To IP=%#016RX64\n",
2896	pVmcsInfoShared->u64LerFromIpMsr, pVmcsInfoShared->u64LerToIpMsr);
2897	uint32_t idxCurrent = idxTopOfStack;
2898	Assert(idxTopOfStack < cLbrStack);
2899	Assert(RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr) <= cLbrStack);
2900	Assert(RT_ELEMENTS(pVmcsInfoShared->au64LbrToIpMsr) <= cLbrStack);
2901	for (;;)
2902	{
2903	if (pVM->nem.s.idLbrToIpMsrFirst)
2904	pHlp->pfnPrintf(pHlp, " Branch (%2u): From IP=%#016RX64 - To IP=%#016RX64 (Info: %#016RX64)\n", idxCurrent,
2905	pVmcsInfoShared->au64LbrFromIpMsr[idxCurrent],
2906	pVmcsInfoShared->au64LbrToIpMsr[idxCurrent],
2907	pVmcsInfoShared->au64LbrInfoMsr[idxCurrent]);
2908	else
2909	pHlp->pfnPrintf(pHlp, " Branch (%2u): LBR=%#RX64\n", idxCurrent, pVmcsInfoShared->au64LbrFromIpMsr[idxCurrent]);
2910
2911	idxCurrent = (idxCurrent - 1) % cLbrStack;
2912	if (idxCurrent == idxTopOfStack)
2913	break;
2914	}
2915	}
2916
2917
2918	/**
2919	* Try initialize the native API.
2920	*
2921	* This may only do part of the job, more can be done in
2922	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
2923	*
2924	* @returns VBox status code.
2925	* @param pVM The cross context VM structure.
2926	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
2927	* the latter we'll fail if we cannot initialize.
2928	* @param fForced Whether the HMForced flag is set and we should
2929	* fail if we cannot initialize.
2930	*/
2931	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
2932	{
2933	AssertReturn(!pVM->nem.s.fCreatedVm, VERR_WRONG_ORDER);
2934
2935	/*
2936	* Some state init.
2937	*/
2938	PCFGMNODE pCfgNem = CFGMR3GetChild(CFGMR3GetRoot(pVM), "NEM/");
2939
2940	/** @cfgm{/NEM/VmxPleGap, uint32_t, 0}
2941	* The pause-filter exiting gap in TSC ticks. When the number of ticks between
2942	* two successive PAUSE instructions exceeds VmxPleGap, the CPU considers the
2943	* latest PAUSE instruction to be start of a new PAUSE loop.
2944	*/
2945	int rc = CFGMR3QueryU32Def(pCfgNem, "VmxPleGap", &pVM->nem.s.cPleGapTicks, 0);
2946	AssertRCReturn(rc, rc);
2947
2948	/** @cfgm{/NEM/VmxPleWindow, uint32_t, 0}
2949	* The pause-filter exiting window in TSC ticks. When the number of ticks
2950	* between the current PAUSE instruction and first PAUSE of a loop exceeds
2951	* VmxPleWindow, a VM-exit is triggered.
2952	*
2953	* Setting VmxPleGap and VmxPleGap to 0 disables pause-filter exiting.
2954	*/
2955	rc = CFGMR3QueryU32Def(pCfgNem, "VmxPleWindow", &pVM->nem.s.cPleWindowTicks, 0);
2956	AssertRCReturn(rc, rc);
2957
2958	/** @cfgm{/NEM/VmxLbr, bool, false}
2959	* Whether to enable LBR for the guest. This is disabled by default as it's only
2960	* useful while debugging and enabling it causes a noticeable performance hit. */
2961	rc = CFGMR3QueryBoolDef(pCfgNem, "VmxLbr", &pVM->nem.s.fLbr, false);
2962	AssertRCReturn(rc, rc);
2963
2964	/*
2965	* Error state.
2966	* The error message will be non-empty on failure and 'rc' will be set too.
2967	*/
2968	RTERRINFOSTATIC ErrInfo;
2969	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
2970	rc = nemR3DarwinLoadHv(fForced, pErrInfo);
2971	if (RT_SUCCESS(rc))
2972	{
2973	if ( !hv_vcpu_enable_managed_msr
2974	&& pVM->nem.s.fLbr)
2975	{
2976	LogRel(("NEM: LBR recording is disabled because the Hypervisor API misses hv_vcpu_enable_managed_msr/hv_vcpu_set_msr_access functionality\n"));
2977	pVM->nem.s.fLbr = false;
2978	}
2979
2980	/*
2981	* While hv_vcpu_run_until() is available starting with Catalina (10.15) it sometimes returns
2982	* an error there for no obvious reasons and there is no indication as to why this happens
2983	* and Apple doesn't document anything. Starting with BigSur (11.0) it appears to work correctly
2984	* so pretend that hv_vcpu_run_until() doesn't exist on Catalina which can be determined by checking
2985	* whether another method is available which was introduced with BigSur.
2986	*/
2987	if (!hv_vmx_get_msr_info) /* Not available means this runs on < 11.0 */
2988	hv_vcpu_run_until = NULL;
2989
2990	if (hv_vcpu_run_until)
2991	{
2992	struct mach_timebase_info TimeInfo;
2993
2994	if (mach_timebase_info(&TimeInfo) == KERN_SUCCESS)
2995	{
2996	pVM->nem.s.cMachTimePerNs = RT_MIN(1, (double)TimeInfo.denom / (double)TimeInfo.numer);
2997	LogRel(("NEM: cMachTimePerNs=%llu (TimeInfo.numer=%u TimeInfo.denom=%u)\n",
2998	pVM->nem.s.cMachTimePerNs, TimeInfo.numer, TimeInfo.denom));
2999	}
3000	else
3001	hv_vcpu_run_until = NULL; /* To avoid running forever (TM asserts when the guest runs for longer than 4 seconds). */
3002	}
3003
3004	hv_return_t hrc = hv_vm_create(HV_VM_DEFAULT);
3005	if (hrc == HV_SUCCESS)
3006	{
3007	if (hv_vm_space_create)
3008	{
3009	hrc = hv_vm_space_create(&pVM->nem.s.uVmAsid);
3010	if (hrc == HV_SUCCESS)
3011	{
3012	LogRel(("NEM: Successfully created ASID: %u\n", pVM->nem.s.uVmAsid));
3013	pVM->nem.s.fCreatedAsid = true;
3014	}
3015	else
3016	LogRel(("NEM: Failed to create ASID for VM (hrc=%#x), continuing...\n", pVM->nem.s.uVmAsid));
3017	}
3018	pVM->nem.s.fCreatedVm = true;
3019
3020	/* Register release statistics */
3021	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3022	{
3023	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
3024	PVMXSTATISTICS pVmxStats = (PVMXSTATISTICS)RTMemAllocZ(sizeof(*pVmxStats));
3025	if (RT_LIKELY(pVmxStats))
3026	{
3027	pNemCpu->pVmxStats = pVmxStats;
3028	rc = nemR3DarwinStatisticsRegister(pVM, idCpu, pNemCpu);
3029	AssertRC(rc);
3030	}
3031	else
3032	{
3033	rc = VERR_NO_MEMORY;
3034	break;
3035	}
3036	}
3037
3038	if (RT_SUCCESS(rc))
3039	{
3040	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
3041	Log(("NEM: Marked active!\n"));
3042	PGMR3EnableNemMode(pVM);
3043	}
3044	}
3045	else
3046	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
3047	"hv_vm_create() failed: %#x", hrc);
3048	}
3049
3050	/*
3051	* We only fail if in forced mode, otherwise just log the complaint and return.
3052	*/
3053	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
3054	if ( (fForced \|\| !fFallback)
3055	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
3056	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
3057
3058	if (pVM->nem.s.fLbr)
3059	{
3060	rc = DBGFR3InfoRegisterInternalEx(pVM, "lbr", "Dumps the NEM LBR info.", nemR3DarwinInfoLbr, DBGFINFO_FLAGS_ALL_EMTS);
3061	AssertRCReturn(rc, rc);
3062	}
3063
3064	if (RTErrInfoIsSet(pErrInfo))
3065	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
3066	return VINF_SUCCESS;
3067	}
3068
3069
3070	/**
3071	* Worker to create the vCPU handle on the EMT running it later on (as required by HV).
3072	*
3073	* @returns VBox status code
3074	* @param pVM The VM handle.
3075	* @param pVCpu The vCPU handle.
3076	* @param idCpu ID of the CPU to create.
3077	*/
3078	static DECLCALLBACK(int) nemR3DarwinNativeInitVCpuOnEmt(PVM pVM, PVMCPU pVCpu, VMCPUID idCpu)
3079	{
3080	hv_return_t hrc = hv_vcpu_create(&pVCpu->nem.s.hVCpuId, HV_VCPU_DEFAULT);
3081	if (hrc != HV_SUCCESS)
3082	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
3083	"Call to hv_vcpu_create failed on vCPU %u: %#x (%Rrc)", idCpu, hrc, nemR3DarwinHvSts2Rc(hrc));
3084
3085	if (idCpu == 0)
3086	{
3087	/* First call initializs the MSR structure holding the capabilities of the host CPU. */
3088	int rc = nemR3DarwinCapsInit();
3089	AssertRCReturn(rc, rc);
3090
3091	if (hv_vmx_vcpu_get_cap_write_vmcs)
3092	{
3093	/* Log the VMCS field write capabilities. */
3094	for (uint32_t i = 0; i < RT_ELEMENTS(g_aVmcsFieldsCap); i++)
3095	{
3096	uint64_t u64Allowed0 = 0;
3097	uint64_t u64Allowed1 = 0;
3098
3099	hrc = hv_vmx_vcpu_get_cap_write_vmcs(pVCpu->nem.s.hVCpuId, g_aVmcsFieldsCap[i].u32VmcsFieldId,
3100	&u64Allowed0, &u64Allowed1);
3101	if (hrc == HV_SUCCESS)
3102	{
3103	if (g_aVmcsFieldsCap[i].f64Bit)
3104	LogRel(("NEM: %s = (allowed_0=%#016RX64 allowed_1=%#016RX64)\n",
3105	g_aVmcsFieldsCap[i].pszVmcsField, u64Allowed0, u64Allowed1));
3106	else
3107	LogRel(("NEM: %s = (allowed_0=%#08RX32 allowed_1=%#08RX32)\n",
3108	g_aVmcsFieldsCap[i].pszVmcsField, (uint32_t)u64Allowed0, (uint32_t)u64Allowed1));
3109
3110	uint32_t cBits = g_aVmcsFieldsCap[i].f64Bit ? 64 : 32;
3111	for (uint32_t iBit = 0; iBit < cBits; iBit++)
3112	{
3113	bool fAllowed0 = RT_BOOL(u64Allowed0 & RT_BIT_64(iBit));
3114	bool fAllowed1 = RT_BOOL(u64Allowed1 & RT_BIT_64(iBit));
3115
3116	if (!fAllowed0 && !fAllowed1)
3117	LogRel(("NEM: Bit %02u = Must NOT be set\n", iBit));
3118	else if (!fAllowed0 && fAllowed1)
3119	LogRel(("NEM: Bit %02u = Can be set or not be set\n", iBit));
3120	else if (fAllowed0 && !fAllowed1)
3121	LogRel(("NEM: Bit %02u = UNDEFINED (AppleHV error)!\n", iBit));
3122	else if (fAllowed0 && fAllowed1)
3123	LogRel(("NEM: Bit %02u = MUST be set\n", iBit));
3124	else
3125	AssertFailed();
3126	}
3127	}
3128	else
3129	LogRel(("NEM: %s = failed to query (hrc=%d)\n", g_aVmcsFieldsCap[i].pszVmcsField, hrc));
3130	}
3131	}
3132	}
3133
3134	int rc = nemR3DarwinInitVmcs(pVCpu);
3135	AssertRCReturn(rc, rc);
3136
3137	if (pVM->nem.s.fCreatedAsid)
3138	{
3139	hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, pVM->nem.s.uVmAsid);
3140	AssertReturn(hrc == HV_SUCCESS, VERR_NEM_VM_CREATE_FAILED);
3141	}
3142
3143	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3144
3145	return VINF_SUCCESS;
3146	}
3147
3148
3149	/**
3150	* Worker to destroy the vCPU handle on the EMT running it later on (as required by HV).
3151	*
3152	* @returns VBox status code
3153	* @param pVCpu The vCPU handle.
3154	*/
3155	static DECLCALLBACK(int) nemR3DarwinNativeTermVCpuOnEmt(PVMCPU pVCpu)
3156	{
3157	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
3158	Assert(hrc == HV_SUCCESS);
3159
3160	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
3161	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
3162	return VINF_SUCCESS;
3163	}
3164
3165
3166	/**
3167	* Worker to setup the TPR shadowing feature if available on the CPU and the VM has an APIC enabled.
3168	*
3169	* @returns VBox status code
3170	* @param pVM The VM handle.
3171	* @param pVCpu The vCPU handle.
3172	*/
3173	static DECLCALLBACK(int) nemR3DarwinNativeInitTprShadowing(PVM pVM, PVMCPU pVCpu)
3174	{
3175	PVMXVMCSINFO pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
3176	uint32_t fVal = pVmcsInfo->u32ProcCtls;
3177
3178	/* Use TPR shadowing if supported by the CPU. */
3179	if ( PDMHasApic(pVM)
3180	&& (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
3181	{
3182	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3183	/* CR8 writes cause a VM-exit based on TPR threshold. */
3184	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3185	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3186	}
3187	else
3188	{
3189	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3190	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3191	}
3192
3193	/* Commit it to the VMCS and update our cache. */
3194	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3195	AssertRC(rc);
3196	pVmcsInfo->u32ProcCtls = fVal;
3197
3198	return VINF_SUCCESS;
3199	}
3200
3201
3202	/**
3203	* This is called after CPUMR3Init is done.
3204	*
3205	* @returns VBox status code.
3206	* @param pVM The VM handle..
3207	*/
3208	int nemR3NativeInitAfterCPUM(PVM pVM)
3209	{
3210	/*
3211	* Validate sanity.
3212	*/
3213	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
3214	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
3215
3216	if (pVM->nem.s.fLbr)
3217	{
3218	int rc = nemR3DarwinSetupLbrMsrRange(pVM);
3219	AssertRCReturn(rc, rc);
3220	}
3221
3222	/*
3223	* Setup the EMTs.
3224	*/
3225	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3226	{
3227	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
3228
3229	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitVCpuOnEmt, 3, pVM, pVCpu, idCpu);
3230	if (RT_FAILURE(rc))
3231	{
3232	/* Rollback. */
3233	while (idCpu--)
3234	VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeTermVCpuOnEmt, 1, pVCpu);
3235
3236	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to hv_vcpu_create failed: %Rrc", rc);
3237	}
3238	}
3239
3240	pVM->nem.s.fCreatedEmts = true;
3241	return VINF_SUCCESS;
3242	}
3243
3244
3245	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
3246	{
3247	if (enmWhat == VMINITCOMPLETED_RING3)
3248	{
3249	/* Now that PDM is initialized the APIC state is known in order to enable the TPR shadowing feature on all EMTs. */
3250	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3251	{
3252	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
3253
3254	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitTprShadowing, 2, pVM, pVCpu);
3255	if (RT_FAILURE(rc))
3256	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Setting up TPR shadowing failed: %Rrc", rc);
3257	}
3258	}
3259	return VINF_SUCCESS;
3260	}
3261
3262
3263	int nemR3NativeTerm(PVM pVM)
3264	{
3265	/*
3266	* Delete the VM.
3267	*/
3268
3269	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu--)
3270	{
3271	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
3272
3273	/*
3274	* Need to do this or hv_vm_space_destroy() fails later on (on 10.15 at least). Could've been documented in
3275	* API reference so I wouldn't have to decompile the kext to find this out but we are talking
3276	* about Apple here unfortunately, API documentation is not their strong suit...
3277	* Would have been of course even better to just automatically drop the address space reference when the vCPU
3278	* gets destroyed.
3279	*/
3280	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
3281	Assert(hrc == HV_SUCCESS);
3282
3283	/*
3284	* Apple's documentation states that the vCPU should be destroyed
3285	* on the thread running the vCPU but as all the other EMTs are gone
3286	* at this point, destroying the VM would hang.
3287	*
3288	* We seem to be at luck here though as destroying apparently works
3289	* from EMT(0) as well.
3290	*/
3291	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
3292	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
3293
3294	if (pVCpu->nem.s.pVmxStats)
3295	{
3296	RTMemFree(pVCpu->nem.s.pVmxStats);
3297	pVCpu->nem.s.pVmxStats = NULL;
3298	}
3299	}
3300
3301	pVM->nem.s.fCreatedEmts = false;
3302
3303	if (pVM->nem.s.fCreatedAsid)
3304	{
3305	hv_return_t hrc = hv_vm_space_destroy(pVM->nem.s.uVmAsid);
3306	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
3307	pVM->nem.s.fCreatedAsid = false;
3308	}
3309
3310	if (pVM->nem.s.fCreatedVm)
3311	{
3312	hv_return_t hrc = hv_vm_destroy();
3313	if (hrc != HV_SUCCESS)
3314	LogRel(("NEM: hv_vm_destroy() failed with %#x\n", hrc));
3315
3316	pVM->nem.s.fCreatedVm = false;
3317	}
3318	return VINF_SUCCESS;
3319	}
3320
3321
3322	/**
3323	* VM reset notification.
3324	*
3325	* @param pVM The cross context VM structure.
3326	*/
3327	void nemR3NativeReset(PVM pVM)
3328	{
3329	RT_NOREF(pVM);
3330	}
3331
3332
3333	/**
3334	* Reset CPU due to INIT IPI or hot (un)plugging.
3335	*
3336	* @param pVCpu The cross context virtual CPU structure of the CPU being
3337	* reset.
3338	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
3339	*/
3340	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
3341	{
3342	RT_NOREF(fInitIpi);
3343	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3344	}
3345
3346
3347	/**
3348	* Dumps the VMCS in response to a faild hv_vcpu_run{_until}() call.
3349	*
3350	* @returns nothing.
3351	* @param pVCpu The cross context virtual CPU structure.
3352	*/
3353	static void nemR3DarwinVmcsDump(PVMCPU pVCpu)
3354	{
3355	static const struct
3356	{
3357	uint32_t u32VmcsFieldId; /*< The VMCS field identifier. /
3358	const char pszVmcsField; /< The VMCS field name. /
3359	bool f64Bit;
3360	} s_aVmcsFieldsDump[] =
3361	{
3362	#define NEM_DARWIN_VMCSNW_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, true }
3363	#define NEM_DARWIN_VMCS64_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, true }
3364	#define NEM_DARWIN_VMCS32_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, false }
3365	#define NEM_DARWIN_VMCS16_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, false }
3366	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_VPID),
3367	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR),
3368	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_EPTP_INDEX),
3369	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_ES_SEL),
3370	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_CS_SEL),
3371	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_SS_SEL),
3372	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_DS_SEL),
3373	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_FS_SEL),
3374	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_GS_SEL),
3375	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_LDTR_SEL),
3376	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_TR_SEL),
3377	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_INTR_STATUS),
3378	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_PML_INDEX),
3379	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_ES_SEL),
3380	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_CS_SEL),
3381	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_SS_SEL),
3382	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_DS_SEL),
3383	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_FS_SEL),
3384	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_GS_SEL),
3385	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_TR_SEL),
3386
3387	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL),
3388	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH),
3389	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL),
3390	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH),
3391	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_MSR_BITMAP_FULL),
3392	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_MSR_BITMAP_HIGH),
3393	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL),
3394	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH),
3395	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL),
3396	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH),
3397	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL),
3398	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH),
3399	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL),
3400	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH),
3401	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL),
3402	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH),
3403	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_OFFSET_FULL),
3404	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_OFFSET_HIGH),
3405	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL),
3406	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH),
3407	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL),
3408	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH),
3409	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL),
3410	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH),
3411	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL),
3412	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH),
3413	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_FULL),
3414	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_HIGH),
3415	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL),
3416	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH),
3417	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL),
3418	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH),
3419	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL),
3420	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH),
3421	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL),
3422	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH),
3423	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_LIST_FULL),
3424	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_LIST_HIGH),
3425	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL),
3426	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH),
3427	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL),
3428	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH),
3429	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL),
3430	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH),
3431	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL),
3432	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH),
3433	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL),
3434	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH),
3435	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_SPPTP_FULL),
3436	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_SPPTP_HIGH),
3437	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL),
3438	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH),
3439	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_PROC_EXEC3_FULL),
3440	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_PROC_EXEC3_HIGH),
3441	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL),
3442	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH),
3443	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL),
3444	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH),
3445	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL),
3446	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH),
3447	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_DEBUGCTL_FULL),
3448	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_DEBUGCTL_HIGH),
3449	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PAT_FULL),
3450	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PAT_HIGH),
3451	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_EFER_FULL),
3452	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_EFER_HIGH),
3453	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL),
3454	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH),
3455	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE0_FULL),
3456	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE0_HIGH),
3457	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE1_FULL),
3458	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE1_HIGH),
3459	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE2_FULL),
3460	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE2_HIGH),
3461	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE3_FULL),
3462	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE3_HIGH),
3463	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_BNDCFGS_FULL),
3464	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_BNDCFGS_HIGH),
3465	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_RTIT_CTL_FULL),
3466	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_RTIT_CTL_HIGH),
3467	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PKRS_FULL),
3468	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PKRS_HIGH),
3469	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PAT_FULL),
3470	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PAT_HIGH),
3471	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_EFER_FULL),
3472	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_EFER_HIGH),
3473	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL),
3474	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH),
3475	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PKRS_FULL),
3476	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PKRS_HIGH),
3477
3478	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PIN_EXEC),
3479	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PROC_EXEC),
3480	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXCEPTION_BITMAP),
3481	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK),
3482	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH),
3483	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_CR3_TARGET_COUNT),
3484	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXIT),
3485	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT),
3486	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT),
3487	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY),
3488	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT),
3489	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO),
3490	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE),
3491	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH),
3492	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_TPR_THRESHOLD),
3493	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PROC_EXEC2),
3494	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PLE_GAP),
3495	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PLE_WINDOW),
3496	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_VM_INSTR_ERROR),
3497	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_REASON),
3498	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO),
3499	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE),
3500	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_IDT_VECTORING_INFO),
3501	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE),
3502	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INSTR_LENGTH),
3503	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INSTR_INFO),
3504	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_ES_LIMIT),
3505	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_CS_LIMIT),
3506	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SS_LIMIT),
3507	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_DS_LIMIT),
3508	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_FS_LIMIT),
3509	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_GS_LIMIT),
3510	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_LDTR_LIMIT),
3511	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_TR_LIMIT),
3512	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_GDTR_LIMIT),
3513	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_IDTR_LIMIT),
3514	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS),
3515	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS),
3516	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS),
3517	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS),
3518	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS),
3519	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS),
3520	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS),
3521	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS),
3522	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_INT_STATE),
3523	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_ACTIVITY_STATE),
3524	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SMBASE),
3525	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SYSENTER_CS),
3526	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_PREEMPT_TIMER_VALUE),
3527	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_HOST_SYSENTER_CS),
3528
3529	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR0_MASK),
3530	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR4_MASK),
3531	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR0_READ_SHADOW),
3532	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR4_READ_SHADOW),
3533	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL0),
3534	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL1),
3535	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL2),
3536	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL3),
3537	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_EXIT_QUALIFICATION),
3538	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RCX),
3539	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RSI),
3540	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RDI),
3541	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RIP),
3542	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_GUEST_LINEAR_ADDR),
3543	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CR0),
3544	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CR3),
3545	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CR4),
3546	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_ES_BASE),
3547	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CS_BASE),
3548	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SS_BASE),
3549	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_DS_BASE),
3550	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_FS_BASE),
3551	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_GS_BASE),
3552	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_LDTR_BASE),
3553	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_TR_BASE),
3554	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_GDTR_BASE),
3555	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_IDTR_BASE),
3556	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_DR7),
3557	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_RSP),
3558	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_RIP),
3559	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_RFLAGS),
3560	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS),
3561	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SYSENTER_ESP),
3562	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SYSENTER_EIP),
3563	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_S_CET),
3564	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SSP),
3565	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR),
3566	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_CR0),
3567	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_CR3),
3568	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_CR4),
3569	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_FS_BASE),
3570	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_GS_BASE),
3571	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_TR_BASE),
3572	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_GDTR_BASE),
3573	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_IDTR_BASE),
3574	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_SYSENTER_ESP),
3575	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_SYSENTER_EIP),
3576	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_RSP),
3577	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_RIP),
3578	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_S_CET),
3579	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_SSP),
3580	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR)
3581	#undef NEM_DARWIN_VMCSNW_FIELD_DUMP
3582	#undef NEM_DARWIN_VMCS64_FIELD_DUMP
3583	#undef NEM_DARWIN_VMCS32_FIELD_DUMP
3584	#undef NEM_DARWIN_VMCS16_FIELD_DUMP
3585	};
3586
3587	for (uint32_t i = 0; i < RT_ELEMENTS(s_aVmcsFieldsDump); i++)
3588	{
3589	if (s_aVmcsFieldsDump[i].f64Bit)
3590	{
3591	uint64_t u64Val;
3592	int rc = nemR3DarwinReadVmcs64(pVCpu, s_aVmcsFieldsDump[i].u32VmcsFieldId, &u64Val);
3593	if (RT_SUCCESS(rc))
3594	LogRel(("NEM/VMCS: %040s: 0x%016RX64\n", s_aVmcsFieldsDump[i].pszVmcsField, u64Val));
3595	else
3596	LogRel(("NEM/VMCS: %040s: rc=%Rrc\n", s_aVmcsFieldsDump[i].pszVmcsField, rc));
3597	}
3598	else
3599	{
3600	uint32_t u32Val;
3601	int rc = nemR3DarwinReadVmcs32(pVCpu, s_aVmcsFieldsDump[i].u32VmcsFieldId, &u32Val);
3602	if (RT_SUCCESS(rc))
3603	LogRel(("NEM/VMCS: %040s: 0x%08RX32\n", s_aVmcsFieldsDump[i].pszVmcsField, u32Val));
3604	else
3605	LogRel(("NEM/VMCS: %040s: rc=%Rrc\n", s_aVmcsFieldsDump[i].pszVmcsField, rc));
3606	}
3607	}
3608	}
3609
3610
3611	/**
3612	* Runs the guest once until an exit occurs.
3613	*
3614	* @returns HV status code.
3615	* @param pVM The cross context VM structure.
3616	* @param pVCpu The cross context virtual CPU structure.
3617	* @param pVmxTransient The transient VMX execution structure.
3618	*/
3619	static hv_return_t nemR3DarwinRunGuest(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
3620	{
3621	TMNotifyStartOfExecution(pVM, pVCpu);
3622
3623	Assert(!pVCpu->nem.s.fCtxChanged);
3624	hv_return_t hrc;
3625	if (hv_vcpu_run_until) /** @todo Configur the deadline dynamically based on when the next timer triggers. */
3626	hrc = hv_vcpu_run_until(pVCpu->nem.s.hVCpuId, mach_absolute_time() + 2 * RT_NS_1SEC_64 * pVM->nem.s.cMachTimePerNs);
3627	else
3628	hrc = hv_vcpu_run(pVCpu->nem.s.hVCpuId);
3629
3630	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
3631
3632	if (hrc != HV_SUCCESS)
3633	nemR3DarwinVmcsDump(pVCpu);
3634
3635	/*
3636	* Sync the TPR shadow with our APIC state.
3637	*/
3638	if ( !pVmxTransient->fIsNestedGuest
3639	&& (pVCpu->nem.s.VmcsInfo.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
3640	{
3641	uint64_t u64Tpr;
3642	hv_return_t hrc2 = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, HV_X86_TPR, &u64Tpr);
3643	Assert(hrc2 == HV_SUCCESS); RT_NOREF(hrc2);
3644
3645	if (pVmxTransient->u8GuestTpr != (uint8_t)u64Tpr)
3646	{
3647	int rc = APICSetTpr(pVCpu, (uint8_t)u64Tpr);
3648	AssertRC(rc);
3649	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
3650	}
3651	}
3652
3653	return hrc;
3654	}
3655
3656
3657	/**
3658	* Prepares the VM to run the guest.
3659	*
3660	* @returns Strict VBox status code.
3661	* @param pVM The cross context VM structure.
3662	* @param pVCpu The cross context virtual CPU structure.
3663	* @param pVmxTransient The VMX transient state.
3664	* @param fSingleStepping Flag whether we run in single stepping mode.
3665	*/
3666	static VBOXSTRICTRC nemR3DarwinPreRunGuest(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fSingleStepping)
3667	{
3668	/*
3669	* Check and process force flag actions, some of which might require us to go back to ring-3.
3670	*/
3671	VBOXSTRICTRC rcStrict = vmxHCCheckForceFlags(pVCpu, false /fIsNestedGuest/, fSingleStepping);
3672	if (rcStrict == VINF_SUCCESS)
3673	{ /likely / }
3674	else
3675	return rcStrict;
3676
3677	/*
3678	* Do not execute in HV if the A20 isn't enabled.
3679	*/
3680	if (PGMPhysIsA20Enabled(pVCpu))
3681	{ /* likely */ }
3682	else
3683	{
3684	LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
3685	return VINF_EM_RESCHEDULE_REM;
3686	}
3687
3688	/*
3689	* Evaluate events to be injected into the guest.
3690	*
3691	* Events in TRPM can be injected without inspecting the guest state.
3692	* If any new events (interrupts/NMI) are pending currently, we try to set up the
3693	* guest to cause a VM-exit the next time they are ready to receive the event.
3694	*/
3695	if (TRPMHasTrap(pVCpu))
3696	vmxHCTrpmTrapToPendingEvent(pVCpu);
3697
3698	uint32_t fIntrState;
3699	rcStrict = vmxHCEvaluatePendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, &fIntrState);
3700
3701	/*
3702	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
3703	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
3704	* also result in triple-faulting the VM.
3705	*
3706	* With nested-guests, the above does not apply since unrestricted guest execution is a
3707	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
3708	*/
3709	rcStrict = vmxHCInjectPendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, fIntrState, fSingleStepping);
3710	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3711	{ /* likely */ }
3712	else
3713	return rcStrict;
3714
3715	int rc = nemR3DarwinExportGuestState(pVM, pVCpu, pVmxTransient);
3716	AssertRCReturn(rc, rc);
3717
3718	LogFlowFunc(("Running vCPU\n"));
3719	pVCpu->nem.s.Event.fPending = false;
3720	return VINF_SUCCESS;
3721	}
3722
3723
3724	/**
3725	* The normal runloop (no debugging features enabled).
3726	*
3727	* @returns Strict VBox status code.
3728	* @param pVM The cross context VM structure.
3729	* @param pVCpu The cross context virtual CPU structure.
3730	*/
3731	static VBOXSTRICTRC nemR3DarwinRunGuestNormal(PVM pVM, PVMCPU pVCpu)
3732	{
3733	/*
3734	* The run loop.
3735	*
3736	* Current approach to state updating to use the sledgehammer and sync
3737	* everything every time. This will be optimized later.
3738	*/
3739	VMXTRANSIENT VmxTransient;
3740	RT_ZERO(VmxTransient);
3741	VmxTransient.pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
3742
3743	/*
3744	* Poll timers and run for a bit.
3745	*/
3746	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
3747	* the whole polling job when timers have changed... */
3748	uint64_t offDeltaIgnored;
3749	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
3750	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
3751	for (unsigned iLoop = 0;; iLoop++)
3752	{
3753	rcStrict = nemR3DarwinPreRunGuest(pVM, pVCpu, &VmxTransient, false /* fSingleStepping */);
3754	if (rcStrict != VINF_SUCCESS)
3755	break;
3756
3757	hv_return_t hrc = nemR3DarwinRunGuest(pVM, pVCpu, &VmxTransient);
3758	if (hrc == HV_SUCCESS)
3759	{
3760	/*
3761	* Deal with the message.
3762	*/
3763	rcStrict = nemR3DarwinHandleExit(pVM, pVCpu, &VmxTransient);
3764	if (rcStrict == VINF_SUCCESS)
3765	{ /* hopefully likely */ }
3766	else
3767	{
3768	LogFlow(("NEM/%u: breaking: nemR3DarwinHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
3769	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
3770	break;
3771	}
3772	}
3773	else
3774	{
3775	AssertLogRelMsgFailedReturn(("hv_vcpu_run()) failed for CPU #%u: %#x %u\n",
3776	pVCpu->idCpu, hrc, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
3777	VERR_NEM_IPE_0);
3778	}
3779	} /* the run loop */
3780
3781	return rcStrict;
3782	}
3783
3784
3785	/**
3786	* Checks if any expensive dtrace probes are enabled and we should go to the
3787	* debug loop.
3788	*
3789	* @returns true if we should use debug loop, false if not.
3790	*/
3791	static bool nemR3DarwinAnyExpensiveProbesEnabled(void)
3792	{
3793	/** @todo Check performance penalty when checking these over and over */
3794	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED() /* expensive too due to context */
3795	\| VBOXVMM_XCPT_DE_ENABLED()
3796	\| VBOXVMM_XCPT_DB_ENABLED()
3797	\| VBOXVMM_XCPT_BP_ENABLED()
3798	\| VBOXVMM_XCPT_OF_ENABLED()
3799	\| VBOXVMM_XCPT_BR_ENABLED()
3800	\| VBOXVMM_XCPT_UD_ENABLED()
3801	\| VBOXVMM_XCPT_NM_ENABLED()
3802	\| VBOXVMM_XCPT_DF_ENABLED()
3803	\| VBOXVMM_XCPT_TS_ENABLED()
3804	\| VBOXVMM_XCPT_NP_ENABLED()
3805	\| VBOXVMM_XCPT_SS_ENABLED()
3806	\| VBOXVMM_XCPT_GP_ENABLED()
3807	\| VBOXVMM_XCPT_PF_ENABLED()
3808	\| VBOXVMM_XCPT_MF_ENABLED()
3809	\| VBOXVMM_XCPT_AC_ENABLED()
3810	\| VBOXVMM_XCPT_XF_ENABLED()
3811	\| VBOXVMM_XCPT_VE_ENABLED()
3812	\| VBOXVMM_XCPT_SX_ENABLED()
3813	\| VBOXVMM_INT_SOFTWARE_ENABLED()
3814	/* not available in R3 \| VBOXVMM_INT_HARDWARE_ENABLED()*/
3815	) != 0
3816	\|\| ( VBOXVMM_INSTR_HALT_ENABLED()
3817	\| VBOXVMM_INSTR_MWAIT_ENABLED()
3818	\| VBOXVMM_INSTR_MONITOR_ENABLED()
3819	\| VBOXVMM_INSTR_CPUID_ENABLED()
3820	\| VBOXVMM_INSTR_INVD_ENABLED()
3821	\| VBOXVMM_INSTR_WBINVD_ENABLED()
3822	\| VBOXVMM_INSTR_INVLPG_ENABLED()
3823	\| VBOXVMM_INSTR_RDTSC_ENABLED()
3824	\| VBOXVMM_INSTR_RDTSCP_ENABLED()
3825	\| VBOXVMM_INSTR_RDPMC_ENABLED()
3826	\| VBOXVMM_INSTR_RDMSR_ENABLED()
3827	\| VBOXVMM_INSTR_WRMSR_ENABLED()
3828	\| VBOXVMM_INSTR_CRX_READ_ENABLED()
3829	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED()
3830	\| VBOXVMM_INSTR_DRX_READ_ENABLED()
3831	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED()
3832	\| VBOXVMM_INSTR_PAUSE_ENABLED()
3833	\| VBOXVMM_INSTR_XSETBV_ENABLED()
3834	\| VBOXVMM_INSTR_SIDT_ENABLED()
3835	\| VBOXVMM_INSTR_LIDT_ENABLED()
3836	\| VBOXVMM_INSTR_SGDT_ENABLED()
3837	\| VBOXVMM_INSTR_LGDT_ENABLED()
3838	\| VBOXVMM_INSTR_SLDT_ENABLED()
3839	\| VBOXVMM_INSTR_LLDT_ENABLED()
3840	\| VBOXVMM_INSTR_STR_ENABLED()
3841	\| VBOXVMM_INSTR_LTR_ENABLED()
3842	\| VBOXVMM_INSTR_GETSEC_ENABLED()
3843	\| VBOXVMM_INSTR_RSM_ENABLED()
3844	\| VBOXVMM_INSTR_RDRAND_ENABLED()
3845	\| VBOXVMM_INSTR_RDSEED_ENABLED()
3846	\| VBOXVMM_INSTR_XSAVES_ENABLED()
3847	\| VBOXVMM_INSTR_XRSTORS_ENABLED()
3848	\| VBOXVMM_INSTR_VMM_CALL_ENABLED()
3849	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED()
3850	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED()
3851	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED()
3852	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED()
3853	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED()
3854	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED()
3855	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED()
3856	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED()
3857	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED()
3858	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED()
3859	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED()
3860	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED()
3861	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED()
3862	) != 0
3863	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED()
3864	\| VBOXVMM_EXIT_HALT_ENABLED()
3865	\| VBOXVMM_EXIT_MWAIT_ENABLED()
3866	\| VBOXVMM_EXIT_MONITOR_ENABLED()
3867	\| VBOXVMM_EXIT_CPUID_ENABLED()
3868	\| VBOXVMM_EXIT_INVD_ENABLED()
3869	\| VBOXVMM_EXIT_WBINVD_ENABLED()
3870	\| VBOXVMM_EXIT_INVLPG_ENABLED()
3871	\| VBOXVMM_EXIT_RDTSC_ENABLED()
3872	\| VBOXVMM_EXIT_RDTSCP_ENABLED()
3873	\| VBOXVMM_EXIT_RDPMC_ENABLED()
3874	\| VBOXVMM_EXIT_RDMSR_ENABLED()
3875	\| VBOXVMM_EXIT_WRMSR_ENABLED()
3876	\| VBOXVMM_EXIT_CRX_READ_ENABLED()
3877	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED()
3878	\| VBOXVMM_EXIT_DRX_READ_ENABLED()
3879	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED()
3880	\| VBOXVMM_EXIT_PAUSE_ENABLED()
3881	\| VBOXVMM_EXIT_XSETBV_ENABLED()
3882	\| VBOXVMM_EXIT_SIDT_ENABLED()
3883	\| VBOXVMM_EXIT_LIDT_ENABLED()
3884	\| VBOXVMM_EXIT_SGDT_ENABLED()
3885	\| VBOXVMM_EXIT_LGDT_ENABLED()
3886	\| VBOXVMM_EXIT_SLDT_ENABLED()
3887	\| VBOXVMM_EXIT_LLDT_ENABLED()
3888	\| VBOXVMM_EXIT_STR_ENABLED()
3889	\| VBOXVMM_EXIT_LTR_ENABLED()
3890	\| VBOXVMM_EXIT_GETSEC_ENABLED()
3891	\| VBOXVMM_EXIT_RSM_ENABLED()
3892	\| VBOXVMM_EXIT_RDRAND_ENABLED()
3893	\| VBOXVMM_EXIT_RDSEED_ENABLED()
3894	\| VBOXVMM_EXIT_XSAVES_ENABLED()
3895	\| VBOXVMM_EXIT_XRSTORS_ENABLED()
3896	\| VBOXVMM_EXIT_VMM_CALL_ENABLED()
3897	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED()
3898	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED()
3899	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED()
3900	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED()
3901	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED()
3902	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED()
3903	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED()
3904	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED()
3905	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED()
3906	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED()
3907	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED()
3908	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED()
3909	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED()
3910	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED()
3911	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED()
3912	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED()
3913	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED()
3914	) != 0;
3915	}
3916
3917
3918	/**
3919	* The debug runloop.
3920	*
3921	* @returns Strict VBox status code.
3922	* @param pVM The cross context VM structure.
3923	* @param pVCpu The cross context virtual CPU structure.
3924	*/
3925	static VBOXSTRICTRC nemR3DarwinRunGuestDebug(PVM pVM, PVMCPU pVCpu)
3926	{
3927	/*
3928	* The run loop.
3929	*
3930	* Current approach to state updating to use the sledgehammer and sync
3931	* everything every time. This will be optimized later.
3932	*/
3933	VMXTRANSIENT VmxTransient;
3934	RT_ZERO(VmxTransient);
3935	VmxTransient.pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
3936
3937	bool const fSavedSingleInstruction = pVCpu->nem.s.fSingleInstruction;
3938	pVCpu->nem.s.fSingleInstruction = pVCpu->nem.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
3939	pVCpu->nem.s.fDebugWantRdTscExit = false;
3940	pVCpu->nem.s.fUsingDebugLoop = true;
3941
3942	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
3943	VMXRUNDBGSTATE DbgState;
3944	vmxHCRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
3945	vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
3946
3947	/*
3948	* Poll timers and run for a bit.
3949	*/
3950	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
3951	* the whole polling job when timers have changed... */
3952	uint64_t offDeltaIgnored;
3953	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
3954	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
3955	for (unsigned iLoop = 0;; iLoop++)
3956	{
3957	bool fStepping = pVCpu->nem.s.fSingleInstruction;
3958
3959	/* Set up VM-execution controls the next two can respond to. */
3960	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
3961
3962	rcStrict = nemR3DarwinPreRunGuest(pVM, pVCpu, &VmxTransient, fStepping);
3963	if (rcStrict != VINF_SUCCESS)
3964	break;
3965
3966	/* Override any obnoxious code in the above call. */
3967	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
3968
3969	hv_return_t hrc = nemR3DarwinRunGuest(pVM, pVCpu, &VmxTransient);
3970	if (hrc == HV_SUCCESS)
3971	{
3972	/*
3973	* Deal with the message.
3974	*/
3975	rcStrict = nemR3DarwinHandleExitDebug(pVM, pVCpu, &VmxTransient, &DbgState);
3976	if (rcStrict == VINF_SUCCESS)
3977	{ /* hopefully likely */ }
3978	else
3979	{
3980	LogFlow(("NEM/%u: breaking: nemR3DarwinHandleExitDebug -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
3981	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
3982	break;
3983	}
3984
3985	/*
3986	* Stepping: Did the RIP change, if so, consider it a single step.
3987	* Otherwise, make sure one of the TFs gets set.
3988	*/
3989	if (fStepping)
3990	{
3991	int rc = vmxHCImportGuestStateEx(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
3992	AssertRC(rc);
3993	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
3994	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
3995	{
3996	rcStrict = VINF_EM_DBG_STEPPED;
3997	break;
3998	}
3999	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
4000	}
4001	}
4002	else
4003	{
4004	AssertLogRelMsgFailedReturn(("hv_vcpu_run()) failed for CPU #%u: %#x %u\n",
4005	pVCpu->idCpu, hrc, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
4006	VERR_NEM_IPE_0);
4007	}
4008	} /* the run loop */
4009
4010	/*
4011	* Clear the X86_EFL_TF if necessary.
4012	*/
4013	if (pVCpu->nem.s.fClearTrapFlag)
4014	{
4015	int rc = vmxHCImportGuestStateEx(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
4016	AssertRC(rc);
4017	pVCpu->nem.s.fClearTrapFlag = false;
4018	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
4019	}
4020
4021	pVCpu->nem.s.fUsingDebugLoop = false;
4022	pVCpu->nem.s.fDebugWantRdTscExit = false;
4023	pVCpu->nem.s.fSingleInstruction = fSavedSingleInstruction;
4024
4025	/* Restore all controls applied by vmxHCPreRunGuestDebugStateApply above. */
4026	return vmxHCRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
4027	}
4028
4029
4030	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
4031	{
4032	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u));
4033	#ifdef LOG_ENABLED
4034	if (LogIs3Enabled())
4035	nemR3DarwinLogState(pVM, pVCpu);
4036	#endif
4037
4038	AssertReturn(NEMR3CanExecuteGuest(pVM, pVCpu), VERR_NEM_IPE_9);
4039
4040	/*
4041	* Try switch to NEM runloop state.
4042	*/
4043	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
4044	{ /* likely */ }
4045	else
4046	{
4047	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4048	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
4049	return VINF_SUCCESS;
4050	}
4051
4052	VBOXSTRICTRC rcStrict;
4053	if ( !pVCpu->nem.s.fUseDebugLoop
4054	&& !nemR3DarwinAnyExpensiveProbesEnabled()
4055	&& !DBGFIsStepping(pVCpu)
4056	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4057	rcStrict = nemR3DarwinRunGuestNormal(pVM, pVCpu);
4058	else
4059	rcStrict = nemR3DarwinRunGuestDebug(pVM, pVCpu);
4060
4061	if (rcStrict == VINF_EM_RAW_TO_R3)
4062	rcStrict = VINF_SUCCESS;
4063
4064	/*
4065	* Convert any pending HM events back to TRPM due to premature exits.
4066	*
4067	* This is because execution may continue from IEM and we would need to inject
4068	* the event from there (hence place it back in TRPM).
4069	*/
4070	if (pVCpu->nem.s.Event.fPending)
4071	{
4072	vmxHCPendingEventToTrpmTrap(pVCpu);
4073	Assert(!pVCpu->nem.s.Event.fPending);
4074
4075	/* Clear the events from the VMCS. */
4076	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
4077	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
4078	}
4079
4080
4081	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
4082	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4083
4084	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL))
4085	{
4086	/* Try anticipate what we might need. */
4087	uint64_t fImport = NEM_DARWIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
4088	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
4089	\|\| RT_FAILURE(rcStrict))
4090	fImport = CPUMCTX_EXTRN_ALL;
4091	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
4092	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4093	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
4094
4095	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
4096	{
4097	/* Only import what is external currently. */
4098	int rc2 = nemR3DarwinCopyStateFromHv(pVM, pVCpu, fImport);
4099	if (RT_SUCCESS(rc2))
4100	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4101	else if (RT_SUCCESS(rcStrict))
4102	rcStrict = rc2;
4103	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
4104	{
4105	pVCpu->cpum.GstCtx.fExtrn = 0;
4106	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4107	}
4108	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
4109	}
4110	else
4111	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4112	}
4113	else
4114	{
4115	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4116	pVCpu->cpum.GstCtx.fExtrn = 0;
4117	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4118	}
4119
4120	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
4121	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, VBOXSTRICTRC_VAL(rcStrict) ));
4122	return rcStrict;
4123	}
4124
4125
4126	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
4127	{
4128	NOREF(pVM);
4129	return PGMPhysIsA20Enabled(pVCpu);
4130	}
4131
4132
4133	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
4134	{
4135	VMCPU_ASSERT_EMT(pVCpu);
4136	bool fOld = pVCpu->nem.s.fSingleInstruction;
4137	pVCpu->nem.s.fSingleInstruction = fEnable;
4138	pVCpu->nem.s.fUseDebugLoop = fEnable \|\| pVM->nem.s.fUseDebugLoop;
4139	return fOld;
4140	}
4141
4142
4143	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
4144	{
4145	LogFlowFunc(("pVM=%p pVCpu=%p fFlags=%#x\n", pVM, pVCpu, fFlags));
4146
4147	RT_NOREF(pVM, fFlags);
4148
4149	hv_return_t hrc = hv_vcpu_interrupt(&pVCpu->nem.s.hVCpuId, 1);
4150	if (hrc != HV_SUCCESS)
4151	LogRel(("NEM: hv_vcpu_interrupt(%u, 1) failed with %#x\n", pVCpu->nem.s.hVCpuId, hrc));
4152	}
4153
4154
4155	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
4156	{
4157	for (DBGFEVENTTYPE enmEvent = DBGFEVENT_EXIT_VMX_FIRST;
4158	!fUseDebugLoop && enmEvent <= DBGFEVENT_EXIT_VMX_LAST;
4159	enmEvent = (DBGFEVENTTYPE)(enmEvent + 1))
4160	fUseDebugLoop = DBGF_IS_EVENT_ENABLED(pVM, enmEvent);
4161
4162	return fUseDebugLoop;
4163	}
4164
4165
4166	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
4167	{
4168	RT_NOREF(pVM, pVCpu);
4169	return fUseDebugLoop;
4170	}
4171
4172
4173	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
4174	uint8_t pu2State, uint32_t puNemRange)
4175	{
4176	RT_NOREF(pVM, puNemRange);
4177
4178	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p\n", GCPhys, cb, pvR3));
4179	#if defined(VBOX_WITH_PGM_NEM_MODE)
4180	if (pvR3)
4181	{
4182	int rc = nemR3DarwinMap(pVM, GCPhys, pvR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE, pu2State);
4183	if (RT_FAILURE(rc))
4184	{
4185	LogRel(("NEMR3NotifyPhysRamRegister: GCPhys=%RGp LB %RGp pvR3=%p rc=%Rrc\n", GCPhys, cb, pvR3, rc));
4186	return VERR_NEM_MAP_PAGES_FAILED;
4187	}
4188	}
4189	return VINF_SUCCESS;
4190	#else
4191	RT_NOREF(pVM, GCPhys, cb, pvR3);
4192	return VERR_NEM_MAP_PAGES_FAILED;
4193	#endif
4194	}
4195
4196
4197	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
4198	{
4199	RT_NOREF(pVM);
4200	return false;
4201	}
4202
4203
4204	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
4205	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
4206	{
4207	RT_NOREF(pVM, puNemRange, pvRam, fFlags);
4208
4209	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d)\n",
4210	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, *pu2State));
4211
4212	#if defined(VBOX_WITH_PGM_NEM_MODE)
4213	/*
4214	* Unmap the RAM we're replacing.
4215	*/
4216	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
4217	{
4218	int rc = nemR3DarwinUnmap(pVM, GCPhys, cb, pu2State);
4219	if (RT_SUCCESS(rc))
4220	{ /* likely */ }
4221	else if (pvMmio2)
4222	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rc(ignored)\n",
4223	GCPhys, cb, fFlags, rc));
4224	else
4225	{
4226	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
4227	GCPhys, cb, fFlags, rc));
4228	return VERR_NEM_UNMAP_PAGES_FAILED;
4229	}
4230	}
4231
4232	/*
4233	* Map MMIO2 if any.
4234	*/
4235	if (pvMmio2)
4236	{
4237	Assert(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2);
4238	int rc = nemR3DarwinMap(pVM, GCPhys, pvMmio2, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE, pu2State);
4239	if (RT_FAILURE(rc))
4240	{
4241	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x pvMmio2=%p: Map -> rc=%Rrc\n",
4242	GCPhys, cb, fFlags, pvMmio2, rc));
4243	return VERR_NEM_MAP_PAGES_FAILED;
4244	}
4245	}
4246	else
4247	Assert(!(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2));
4248
4249	#else
4250	RT_NOREF(pVM, GCPhys, cb, pvRam, pvMmio2);
4251	*pu2State = (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE) ? UINT8_MAX : NEM_DARWIN_PAGE_STATE_UNMAPPED;
4252	#endif
4253	return VINF_SUCCESS;
4254	}
4255
4256
4257	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
4258	void pvRam, void pvMmio2, uint32_t *puNemRange)
4259	{
4260	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
4261	return VINF_SUCCESS;
4262	}
4263
4264
4265	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
4266	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
4267	{
4268	RT_NOREF(pVM, puNemRange);
4269
4270	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p uNemRange=%#x (%#x)\n",
4271	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
4272
4273	int rc = VINF_SUCCESS;
4274	#if defined(VBOX_WITH_PGM_NEM_MODE)
4275	/*
4276	* Unmap the MMIO2 pages.
4277	*/
4278	/** @todo If we implement aliasing (MMIO2 page aliased into MMIO range),
4279	* we may have more stuff to unmap even in case of pure MMIO... */
4280	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
4281	{
4282	rc = nemR3DarwinUnmap(pVM, GCPhys, cb, pu2State);
4283	if (RT_FAILURE(rc))
4284	{
4285	LogRel2(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
4286	GCPhys, cb, fFlags, rc));
4287	rc = VERR_NEM_UNMAP_PAGES_FAILED;
4288	}
4289	}
4290
4291	/* Ensure the page is masked as unmapped if relevant. */
4292	Assert(!pu2State \|\| *pu2State == NEM_DARWIN_PAGE_STATE_UNMAPPED);
4293
4294	/*
4295	* Restore the RAM we replaced.
4296	*/
4297	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
4298	{
4299	AssertPtr(pvRam);
4300	rc = nemR3DarwinMap(pVM, GCPhys, pvRam, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE, pu2State);
4301	if (RT_SUCCESS(rc))
4302	{ /* likely */ }
4303	else
4304	{
4305	LogRel(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp pvMmio2=%p rc=%Rrc\n", GCPhys, cb, pvMmio2, rc));
4306	rc = VERR_NEM_MAP_PAGES_FAILED;
4307	}
4308	}
4309
4310	RT_NOREF(pvMmio2);
4311	#else
4312	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
4313	if (pu2State)
4314	*pu2State = UINT8_MAX;
4315	rc = VERR_NEM_UNMAP_PAGES_FAILED;
4316	#endif
4317	return rc;
4318	}
4319
4320
4321	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
4322	void *pvBitmap, size_t cbBitmap)
4323	{
4324	RT_NOREF(pVM, GCPhys, cb, uNemRange, pvBitmap, cbBitmap);
4325	AssertFailed();
4326	return VERR_NOT_IMPLEMENTED;
4327	}
4328
4329
4330	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
4331	uint8_t pu2State, uint32_t puNemRange)
4332	{
4333	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
4334
4335	Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
4336	*pu2State = UINT8_MAX;
4337	*puNemRange = 0;
4338	return VINF_SUCCESS;
4339	}
4340
4341
4342	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
4343	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
4344	{
4345	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
4346	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
4347	*pu2State = UINT8_MAX;
4348
4349	#if defined(VBOX_WITH_PGM_NEM_MODE)
4350	/*
4351	* (Re-)map readonly.
4352	*/
4353	AssertPtrReturn(pvPages, VERR_INVALID_POINTER);
4354	int rc = nemR3DarwinMap(pVM, GCPhys, pvPages, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE, pu2State);
4355	if (RT_FAILURE(rc))
4356	{
4357	LogRel(("nemR3NativeNotifyPhysRomRegisterLate: GCPhys=%RGp LB %RGp pvPages=%p fFlags=%#x rc=%Rrc\n",
4358	GCPhys, cb, pvPages, fFlags, rc));
4359	return VERR_NEM_MAP_PAGES_FAILED;
4360	}
4361	RT_NOREF(fFlags, puNemRange);
4362	return VINF_SUCCESS;
4363	#else
4364	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
4365	return VERR_NEM_MAP_PAGES_FAILED;
4366	#endif
4367	}
4368
4369
4370	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
4371	RTR3PTR pvMemR3, uint8_t *pu2State)
4372	{
4373	RT_NOREF(pVM);
4374
4375	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
4376	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
4377
4378	*pu2State = UINT8_MAX;
4379	#if defined(VBOX_WITH_PGM_NEM_MODE)
4380	if (pvMemR3)
4381	{
4382	int rc = nemR3DarwinMap(pVM, GCPhys, pvMemR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE, pu2State);
4383	AssertLogRelMsgRC(rc, ("NEMHCNotifyHandlerPhysicalDeregister: nemR3DarwinMap(,%p,%RGp,%RGp,) -> %Rrc\n",
4384	pvMemR3, GCPhys, cb, rc));
4385	}
4386	RT_NOREF(enmKind);
4387	#else
4388	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
4389	AssertFailed();
4390	#endif
4391	}
4392
4393
4394	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
4395	{
4396	Log(("NEMR3NotifySetA20: fEnabled=%RTbool\n", fEnabled));
4397	RT_NOREF(pVCpu, fEnabled);
4398	}
4399
4400
4401	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
4402	{
4403	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
4404	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
4405	}
4406
4407
4408	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
4409	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
4410	{
4411	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
4412	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
4413	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
4414	}
4415
4416
4417	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
4418	PGMPAGETYPE enmType, uint8_t *pu2State)
4419	{
4420	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4421	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
4422	RT_NOREF(HCPhys, fPageProt, enmType);
4423
4424	return nemR3DarwinUnmap(pVM, GCPhys, X86_PAGE_SIZE, pu2State);
4425	}
4426
4427
4428	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
4429	PGMPAGETYPE enmType, uint8_t *pu2State)
4430	{
4431	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4432	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
4433	RT_NOREF(HCPhys, pvR3, fPageProt, enmType)
4434
4435	nemR3DarwinUnmap(pVM, GCPhys, X86_PAGE_SIZE, pu2State);
4436	}
4437
4438
4439	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
4440	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
4441	{
4442	Log5(("NEMHCNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4443	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
4444	RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType);
4445
4446	nemR3DarwinUnmap(pVM, GCPhys, X86_PAGE_SIZE, pu2State);
4447	}
4448
4449
4450	/**
4451	* Interface for importing state on demand (used by IEM).
4452	*
4453	* @returns VBox status code.
4454	* @param pVCpu The cross context CPU structure.
4455	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
4456	*/
4457	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
4458	{
4459	LogFlowFunc(("pVCpu=%p fWhat=%RX64\n", pVCpu, fWhat));
4460	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
4461
4462	return nemR3DarwinCopyStateFromHv(pVCpu->pVMR3, pVCpu, fWhat);
4463	}
4464
4465
4466	/**
4467	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
4468	*
4469	* @returns VBox status code.
4470	* @param pVCpu The cross context CPU structure.
4471	* @param pcTicks Where to return the CPU tick count.
4472	* @param puAux Where to return the TSC_AUX register value.
4473	*/
4474	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
4475	{
4476	LogFlowFunc(("pVCpu=%p pcTicks=%RX64 puAux=%RX32\n", pVCpu, pcTicks, puAux));
4477	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
4478
4479	int rc = nemR3DarwinMsrRead(pVCpu, MSR_IA32_TSC, pcTicks);
4480	if ( RT_SUCCESS(rc)
4481	&& puAux)
4482	{
4483	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX)
4484	{
4485	uint64_t u64Aux;
4486	rc = nemR3DarwinMsrRead(pVCpu, MSR_K8_TSC_AUX, &u64Aux);
4487	if (RT_SUCCESS(rc))
4488	*puAux = (uint32_t)u64Aux;
4489	}
4490	else
4491	*puAux = CPUMGetGuestTscAux(pVCpu);
4492	}
4493
4494	return rc;
4495	}
4496
4497
4498	/**
4499	* Resumes CPU clock (TSC) on all virtual CPUs.
4500	*
4501	* This is called by TM when the VM is started, restored, resumed or similar.
4502	*
4503	* @returns VBox status code.
4504	* @param pVM The cross context VM structure.
4505	* @param pVCpu The cross context CPU structure of the calling EMT.
4506	* @param uPausedTscValue The TSC value at the time of pausing.
4507	*/
4508	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
4509	{
4510	LogFlowFunc(("pVM=%p pVCpu=%p uPausedTscValue=%RX64\n", pVCpu, uPausedTscValue));
4511	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
4512	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
4513
4514	hv_return_t hrc = hv_vm_sync_tsc(uPausedTscValue);
4515	if (RT_LIKELY(hrc == HV_SUCCESS))
4516	{
4517	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_TSC_AUX);
4518	return VINF_SUCCESS;
4519	}
4520
4521	return nemR3DarwinHvSts2Rc(hrc);
4522	}
4523
4524
4525	/**
4526	* Returns features supported by the NEM backend.
4527	*
4528	* @returns Flags of features supported by the native NEM backend.
4529	* @param pVM The cross context VM structure.
4530	*/
4531	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
4532	{
4533	RT_NOREF(pVM);
4534	/*
4535	* Apple's Hypervisor.framework is not supported if the CPU doesn't support nested paging
4536	* and unrestricted guest execution support so we can safely return these flags here always.
4537	*/
4538	return NEM_FEAT_F_NESTED_PAGING \| NEM_FEAT_F_FULL_GST_EXEC \| NEM_FEAT_F_XSAVE_XRSTOR;
4539	}
4540
4541
4542	/** @page pg_nem_darwin NEM/darwin - Native Execution Manager, macOS.
4543	*
4544	* @todo Add notes as the implementation progresses...
4545	*/
4546

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-darwin.cpp@ 98045

Download in other formats: