NEMR3Native-darwin.cpp@ 103583

Last change on this file since 103583 was 100878, checked in by vboxsync, 16 months ago
VMM/NEMR3Native-darwin: Disable intercepting #DB exceptions and mov drX instructions as it currently breaks using a debugger inside the guest. This will break using the hypervisor debugger but only very few people use it anyway and even less on macOS, bugref:10504 [doxygen]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 182.2 KB

Line
1	/* $Id: NEMR3Native-darwin.cpp 100878 2023-08-15 08:24:46Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 macOS backend using Hypervisor.framework.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	*/
9
10	/*
11	* Copyright (C) 2020-2023 Oracle and/or its affiliates.
12	*
13	* This file is part of VirtualBox base platform packages, as
14	* available from https://www.virtualbox.org.
15	*
16	* This program is free software; you can redistribute it and/or
17	* modify it under the terms of the GNU General Public License
18	* as published by the Free Software Foundation, in version 3 of the
19	* License.
20	*
21	* This program is distributed in the hope that it will be useful, but
22	* WITHOUT ANY WARRANTY; without even the implied warranty of
23	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	* General Public License for more details.
25	*
26	* You should have received a copy of the GNU General Public License
27	* along with this program; if not, see <https://www.gnu.org/licenses>.
28	*
29	* SPDX-License-Identifier: GPL-3.0-only
30	*/
31
32
33	/*********************************************************************************************************************************
34	* Header Files *
35	*********************************************************************************************************************************/
36	#define LOG_GROUP LOG_GROUP_NEM
37	#define VMCPU_INCL_CPUM_GST_CTX
38	#define CPUM_WITH_NONCONST_HOST_FEATURES /* required for initializing parts of the g_CpumHostFeatures structure here. */
39	#include <VBox/vmm/nem.h>
40	#include <VBox/vmm/iem.h>
41	#include <VBox/vmm/em.h>
42	#include <VBox/vmm/apic.h>
43	#include <VBox/vmm/pdm.h>
44	#include <VBox/vmm/hm.h>
45	#include <VBox/vmm/hm_vmx.h>
46	#include <VBox/vmm/dbgftrace.h>
47	#include <VBox/vmm/gcm.h>
48	#include "VMXInternal.h"
49	#include "NEMInternal.h"
50	#include <VBox/vmm/vmcc.h>
51	#include "dtrace/VBoxVMM.h"
52
53	#include <iprt/asm.h>
54	#include <iprt/ldr.h>
55	#include <iprt/mem.h>
56	#include <iprt/path.h>
57	#include <iprt/string.h>
58	#include <iprt/system.h>
59	#include <iprt/utf16.h>
60
61	#include <mach/mach_time.h>
62	#include <mach/kern_return.h>
63
64
65	/*********************************************************************************************************************************
66	* Defined Constants And Macros *
67	*********************************************************************************************************************************/
68	/* No nested hwvirt (for now). */
69	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
70	# undef VBOX_WITH_NESTED_HWVIRT_VMX
71	#endif
72
73
74	/** @name HV return codes.
75	* @{ */
76	/** Operation was successful. */
77	#define HV_SUCCESS 0
78	/** An error occurred during operation. */
79	#define HV_ERROR 0xfae94001
80	/** The operation could not be completed right now, try again. */
81	#define HV_BUSY 0xfae94002
82	/** One of the parameters passed wis invalid. */
83	#define HV_BAD_ARGUMENT 0xfae94003
84	/** Not enough resources left to fulfill the operation. */
85	#define HV_NO_RESOURCES 0xfae94005
86	/** The device could not be found. */
87	#define HV_NO_DEVICE 0xfae94006
88	/** The operation is not supportd on this platform with this configuration. */
89	#define HV_UNSUPPORTED 0xfae94007
90	/** @} */
91
92
93	/** @name HV memory protection flags.
94	* @{ */
95	/** Memory is readable. */
96	#define HV_MEMORY_READ RT_BIT_64(0)
97	/** Memory is writeable. */
98	#define HV_MEMORY_WRITE RT_BIT_64(1)
99	/** Memory is executable. */
100	#define HV_MEMORY_EXEC RT_BIT_64(2)
101	/** @} */
102
103
104	/** @name HV shadow VMCS protection flags.
105	* @{ */
106	/** Shadow VMCS field is not accessible. */
107	#define HV_SHADOW_VMCS_NONE 0
108	/** Shadow VMCS fild is readable. */
109	#define HV_SHADOW_VMCS_READ RT_BIT_64(0)
110	/** Shadow VMCS field is writeable. */
111	#define HV_SHADOW_VMCS_WRITE RT_BIT_64(1)
112	/** @} */
113
114
115	/** Default VM creation flags. */
116	#define HV_VM_DEFAULT 0
117	/** Default guest address space creation flags. */
118	#define HV_VM_SPACE_DEFAULT 0
119	/** Default vCPU creation flags. */
120	#define HV_VCPU_DEFAULT 0
121
122	#define HV_DEADLINE_FOREVER UINT64_MAX
123
124
125	/*********************************************************************************************************************************
126	* Structures and Typedefs *
127	*********************************************************************************************************************************/
128
129	/** HV return code type. */
130	typedef uint32_t hv_return_t;
131	/** HV capability bitmask. */
132	typedef uint64_t hv_capability_t;
133	/** Option bitmask type when creating a VM. */
134	typedef uint64_t hv_vm_options_t;
135	/** Option bitmask when creating a vCPU. */
136	typedef uint64_t hv_vcpu_options_t;
137	/** HV memory protection flags type. */
138	typedef uint64_t hv_memory_flags_t;
139	/** Shadow VMCS protection flags. */
140	typedef uint64_t hv_shadow_flags_t;
141	/** Guest physical address type. */
142	typedef uint64_t hv_gpaddr_t;
143
144
145	/**
146	* VMX Capability enumeration.
147	*/
148	typedef enum
149	{
150	HV_VMX_CAP_PINBASED = 0,
151	HV_VMX_CAP_PROCBASED,
152	HV_VMX_CAP_PROCBASED2,
153	HV_VMX_CAP_ENTRY,
154	HV_VMX_CAP_EXIT,
155	HV_VMX_CAP_BASIC, /* Since 11.0 */
156	HV_VMX_CAP_TRUE_PINBASED, /* Since 11.0 */
157	HV_VMX_CAP_TRUE_PROCBASED, /* Since 11.0 */
158	HV_VMX_CAP_TRUE_ENTRY, /* Since 11.0 */
159	HV_VMX_CAP_TRUE_EXIT, /* Since 11.0 */
160	HV_VMX_CAP_MISC, /* Since 11.0 */
161	HV_VMX_CAP_CR0_FIXED0, /* Since 11.0 */
162	HV_VMX_CAP_CR0_FIXED1, /* Since 11.0 */
163	HV_VMX_CAP_CR4_FIXED0, /* Since 11.0 */
164	HV_VMX_CAP_CR4_FIXED1, /* Since 11.0 */
165	HV_VMX_CAP_VMCS_ENUM, /* Since 11.0 */
166	HV_VMX_CAP_EPT_VPID_CAP, /* Since 11.0 */
167	HV_VMX_CAP_PREEMPTION_TIMER = 32
168	} hv_vmx_capability_t;
169
170
171	/**
172	* MSR information.
173	*/
174	typedef enum
175	{
176	HV_VMX_INFO_MSR_IA32_ARCH_CAPABILITIES = 0,
177	HV_VMX_INFO_MSR_IA32_PERF_CAPABILITIES,
178	HV_VMX_VALID_MSR_IA32_PERFEVNTSEL,
179	HV_VMX_VALID_MSR_IA32_FIXED_CTR_CTRL,
180	HV_VMX_VALID_MSR_IA32_PERF_GLOBAL_CTRL,
181	HV_VMX_VALID_MSR_IA32_PERF_GLOBAL_STATUS,
182	HV_VMX_VALID_MSR_IA32_DEBUGCTL,
183	HV_VMX_VALID_MSR_IA32_SPEC_CTRL,
184	HV_VMX_NEED_MSR_IA32_SPEC_CTRL
185	} hv_vmx_msr_info_t;
186
187
188	/**
189	* HV x86 register enumeration.
190	*/
191	typedef enum
192	{
193	HV_X86_RIP = 0,
194	HV_X86_RFLAGS,
195	HV_X86_RAX,
196	HV_X86_RCX,
197	HV_X86_RDX,
198	HV_X86_RBX,
199	HV_X86_RSI,
200	HV_X86_RDI,
201	HV_X86_RSP,
202	HV_X86_RBP,
203	HV_X86_R8,
204	HV_X86_R9,
205	HV_X86_R10,
206	HV_X86_R11,
207	HV_X86_R12,
208	HV_X86_R13,
209	HV_X86_R14,
210	HV_X86_R15,
211	HV_X86_CS,
212	HV_X86_SS,
213	HV_X86_DS,
214	HV_X86_ES,
215	HV_X86_FS,
216	HV_X86_GS,
217	HV_X86_IDT_BASE,
218	HV_X86_IDT_LIMIT,
219	HV_X86_GDT_BASE,
220	HV_X86_GDT_LIMIT,
221	HV_X86_LDTR,
222	HV_X86_LDT_BASE,
223	HV_X86_LDT_LIMIT,
224	HV_X86_LDT_AR,
225	HV_X86_TR,
226	HV_X86_TSS_BASE,
227	HV_X86_TSS_LIMIT,
228	HV_X86_TSS_AR,
229	HV_X86_CR0,
230	HV_X86_CR1,
231	HV_X86_CR2,
232	HV_X86_CR3,
233	HV_X86_CR4,
234	HV_X86_DR0,
235	HV_X86_DR1,
236	HV_X86_DR2,
237	HV_X86_DR3,
238	HV_X86_DR4,
239	HV_X86_DR5,
240	HV_X86_DR6,
241	HV_X86_DR7,
242	HV_X86_TPR,
243	HV_X86_XCR0,
244	HV_X86_REGISTERS_MAX
245	} hv_x86_reg_t;
246
247
248	/** MSR permission flags type. */
249	typedef uint32_t hv_msr_flags_t;
250	/** MSR can't be accessed. */
251	#define HV_MSR_NONE 0
252	/** MSR is readable by the guest. */
253	#define HV_MSR_READ RT_BIT(0)
254	/** MSR is writeable by the guest. */
255	#define HV_MSR_WRITE RT_BIT(1)
256
257
258	typedef hv_return_t FN_HV_CAPABILITY(hv_capability_t capability, uint64_t *valu);
259	typedef hv_return_t FN_HV_VM_CREATE(hv_vm_options_t flags);
260	typedef hv_return_t FN_HV_VM_DESTROY(void);
261	typedef hv_return_t FN_HV_VM_SPACE_CREATE(hv_vm_space_t *asid);
262	typedef hv_return_t FN_HV_VM_SPACE_DESTROY(hv_vm_space_t asid);
263	typedef hv_return_t FN_HV_VM_MAP(const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
264	typedef hv_return_t FN_HV_VM_UNMAP(hv_gpaddr_t gpa, size_t size);
265	typedef hv_return_t FN_HV_VM_PROTECT(hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
266	typedef hv_return_t FN_HV_VM_MAP_SPACE(hv_vm_space_t asid, const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
267	typedef hv_return_t FN_HV_VM_UNMAP_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size);
268	typedef hv_return_t FN_HV_VM_PROTECT_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
269	typedef hv_return_t FN_HV_VM_SYNC_TSC(uint64_t tsc);
270
271	typedef hv_return_t FN_HV_VCPU_CREATE(hv_vcpuid_t *vcpu, hv_vcpu_options_t flags);
272	typedef hv_return_t FN_HV_VCPU_DESTROY(hv_vcpuid_t vcpu);
273	typedef hv_return_t FN_HV_VCPU_SET_SPACE(hv_vcpuid_t vcpu, hv_vm_space_t asid);
274	typedef hv_return_t FN_HV_VCPU_READ_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t *value);
275	typedef hv_return_t FN_HV_VCPU_WRITE_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t value);
276	typedef hv_return_t FN_HV_VCPU_READ_FPSTATE(hv_vcpuid_t vcpu, void *buffer, size_t size);
277	typedef hv_return_t FN_HV_VCPU_WRITE_FPSTATE(hv_vcpuid_t vcpu, const void *buffer, size_t size);
278	typedef hv_return_t FN_HV_VCPU_ENABLE_NATIVE_MSR(hv_vcpuid_t vcpu, uint32_t msr, bool enable);
279	typedef hv_return_t FN_HV_VCPU_READ_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t *value);
280	typedef hv_return_t FN_HV_VCPU_WRITE_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t value);
281	typedef hv_return_t FN_HV_VCPU_FLUSH(hv_vcpuid_t vcpu);
282	typedef hv_return_t FN_HV_VCPU_INVALIDATE_TLB(hv_vcpuid_t vcpu);
283	typedef hv_return_t FN_HV_VCPU_RUN(hv_vcpuid_t vcpu);
284	typedef hv_return_t FN_HV_VCPU_RUN_UNTIL(hv_vcpuid_t vcpu, uint64_t deadline);
285	typedef hv_return_t FN_HV_VCPU_INTERRUPT(hv_vcpuid_t *vcpus, unsigned int vcpu_count);
286	typedef hv_return_t FN_HV_VCPU_GET_EXEC_TIME(hv_vcpuid_t vcpus, uint64_t time);
287
288	typedef hv_return_t FN_HV_VMX_VCPU_READ_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
289	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
290
291	typedef hv_return_t FN_HV_VMX_VCPU_READ_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
292	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
293	typedef hv_return_t FN_HV_VMX_VCPU_SET_SHADOW_ACCESS(hv_vcpuid_t vcpu, uint32_t field, hv_shadow_flags_t flags);
294
295	typedef hv_return_t FN_HV_VMX_READ_CAPABILITY(hv_vmx_capability_t field, uint64_t *value);
296	typedef hv_return_t FN_HV_VMX_VCPU_SET_APIC_ADDRESS(hv_vcpuid_t vcpu, hv_gpaddr_t gpa);
297
298	/* Since 11.0 */
299	typedef hv_return_t FN_HV_VMX_GET_MSR_INFO(hv_vmx_msr_info_t field, uint64_t *value);
300	typedef hv_return_t FN_HV_VMX_VCPU_GET_CAP_WRITE_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t allowed_0, uint64_t allowed_1);
301	typedef hv_return_t FN_HV_VCPU_ENABLE_MANAGED_MSR(hv_vcpuid_t vcpu, uint32_t msr, bool enable);
302	typedef hv_return_t FN_HV_VCPU_SET_MSR_ACCESS(hv_vcpuid_t vcpu, uint32_t msr, hv_msr_flags_t flags);
303
304
305	/*********************************************************************************************************************************
306	* Global Variables *
307	*********************************************************************************************************************************/
308	static void nemR3DarwinVmcsDump(PVMCPU pVCpu);
309
310	/** NEM_DARWIN_PAGE_STATE_XXX names. */
311	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
312	/** MSRs. */
313	static SUPHWVIRTMSRS g_HmMsrs;
314	/** VMX: Set if swapping EFER is supported. */
315	static bool g_fHmVmxSupportsVmcsEfer = false;
316	/** @name APIs imported from Hypervisor.framework.
317	* @{ */
318	static FN_HV_CAPABILITY g_pfnHvCapability = NULL; / Since 10.15 */
319	static FN_HV_VM_CREATE g_pfnHvVmCreate = NULL; / Since 10.10 */
320	static FN_HV_VM_DESTROY g_pfnHvVmDestroy = NULL; / Since 10.10 */
321	static FN_HV_VM_SPACE_CREATE g_pfnHvVmSpaceCreate = NULL; / Since 10.15 */
322	static FN_HV_VM_SPACE_DESTROY g_pfnHvVmSpaceDestroy = NULL; / Since 10.15 */
323	static FN_HV_VM_MAP g_pfnHvVmMap = NULL; / Since 10.10 */
324	static FN_HV_VM_UNMAP g_pfnHvVmUnmap = NULL; / Since 10.10 */
325	static FN_HV_VM_PROTECT g_pfnHvVmProtect = NULL; / Since 10.10 */
326	static FN_HV_VM_MAP_SPACE g_pfnHvVmMapSpace = NULL; / Since 10.15 */
327	static FN_HV_VM_UNMAP_SPACE g_pfnHvVmUnmapSpace = NULL; / Since 10.15 */
328	static FN_HV_VM_PROTECT_SPACE g_pfnHvVmProtectSpace = NULL; / Since 10.15 */
329	static FN_HV_VM_SYNC_TSC g_pfnHvVmSyncTsc = NULL; / Since 10.10 */
330
331	static FN_HV_VCPU_CREATE g_pfnHvVCpuCreate = NULL; / Since 10.10 */
332	static FN_HV_VCPU_DESTROY g_pfnHvVCpuDestroy = NULL; / Since 10.10 */
333	static FN_HV_VCPU_SET_SPACE g_pfnHvVCpuSetSpace = NULL; / Since 10.15 */
334	static FN_HV_VCPU_READ_REGISTER g_pfnHvVCpuReadRegister = NULL; / Since 10.10 */
335	static FN_HV_VCPU_WRITE_REGISTER g_pfnHvVCpuWriteRegister = NULL; / Since 10.10 */
336	static FN_HV_VCPU_READ_FPSTATE g_pfnHvVCpuReadFpState = NULL; / Since 10.10 */
337	static FN_HV_VCPU_WRITE_FPSTATE g_pfnHvVCpuWriteFpState = NULL; / Since 10.10 */
338	static FN_HV_VCPU_ENABLE_NATIVE_MSR g_pfnHvVCpuEnableNativeMsr = NULL; / Since 10.10 */
339	static FN_HV_VCPU_READ_MSR g_pfnHvVCpuReadMsr = NULL; / Since 10.10 */
340	static FN_HV_VCPU_WRITE_MSR g_pfnHvVCpuWriteMsr = NULL; / Since 10.10 */
341	static FN_HV_VCPU_FLUSH g_pfnHvVCpuFlush = NULL; / Since 10.10 */
342	static FN_HV_VCPU_INVALIDATE_TLB g_pfnHvVCpuInvalidateTlb = NULL; / Since 10.10 */
343	static FN_HV_VCPU_RUN g_pfnHvVCpuRun = NULL; / Since 10.10 */
344	static FN_HV_VCPU_RUN_UNTIL g_pfnHvVCpuRunUntil = NULL; / Since 10.15 */
345	static FN_HV_VCPU_INTERRUPT g_pfnHvVCpuInterrupt = NULL; / Since 10.10 */
346	static FN_HV_VCPU_GET_EXEC_TIME g_pfnHvVCpuGetExecTime = NULL; / Since 10.10 */
347
348	static FN_HV_VMX_READ_CAPABILITY g_pfnHvVmxReadCapability = NULL; / Since 10.10 */
349	static FN_HV_VMX_VCPU_READ_VMCS g_pfnHvVmxVCpuReadVmcs = NULL; / Since 10.10 */
350	static FN_HV_VMX_VCPU_WRITE_VMCS g_pfnHvVmxVCpuWriteVmcs = NULL; / Since 10.10 */
351	static FN_HV_VMX_VCPU_READ_SHADOW_VMCS g_pfnHvVmxVCpuReadShadowVmcs = NULL; / Since 10.15 */
352	static FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS g_pfnHvVmxVCpuWriteShadowVmcs = NULL; / Since 10.15 */
353	static FN_HV_VMX_VCPU_SET_SHADOW_ACCESS g_pfnHvVmxVCpuSetShadowAccess = NULL; / Since 10.15 */
354	static FN_HV_VMX_VCPU_SET_APIC_ADDRESS g_pfnHvVmxVCpuSetApicAddress = NULL; / Since 10.10 */
355
356	static FN_HV_VMX_GET_MSR_INFO g_pfnHvVmxGetMsrInfo = NULL; / Since 11.0 */
357	static FN_HV_VMX_VCPU_GET_CAP_WRITE_VMCS g_pfnHvVmxVCpuGetCapWriteVmcs = NULL; / Since 11.0 */
358	static FN_HV_VCPU_ENABLE_MANAGED_MSR g_pfnHvVCpuEnableManagedMsr = NULL; / Since 11.0 */
359	static FN_HV_VCPU_SET_MSR_ACCESS g_pfnHvVCpuSetMsrAccess = NULL; / Since 11.0 */
360	/** @} */
361
362
363	/**
364	* Import instructions.
365	*/
366	static const struct
367	{
368	bool fOptional; /*< Set if import is optional. /
369	void ppfn; /< The function pointer variable. */
370	const char pszName; /< The function name. /
371	} g_aImports[] =
372	{
373	#define NEM_DARWIN_IMPORT(a_fOptional, a_Pfn, a_Name) { (a_fOptional), (void **)&(a_Pfn), #a_Name }
374	NEM_DARWIN_IMPORT(true, g_pfnHvCapability, hv_capability),
375	NEM_DARWIN_IMPORT(false, g_pfnHvVmCreate, hv_vm_create),
376	NEM_DARWIN_IMPORT(false, g_pfnHvVmDestroy, hv_vm_destroy),
377	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceCreate, hv_vm_space_create),
378	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceDestroy, hv_vm_space_destroy),
379	NEM_DARWIN_IMPORT(false, g_pfnHvVmMap, hv_vm_map),
380	NEM_DARWIN_IMPORT(false, g_pfnHvVmUnmap, hv_vm_unmap),
381	NEM_DARWIN_IMPORT(false, g_pfnHvVmProtect, hv_vm_protect),
382	NEM_DARWIN_IMPORT(true, g_pfnHvVmMapSpace, hv_vm_map_space),
383	NEM_DARWIN_IMPORT(true, g_pfnHvVmUnmapSpace, hv_vm_unmap_space),
384	NEM_DARWIN_IMPORT(true, g_pfnHvVmProtectSpace, hv_vm_protect_space),
385	NEM_DARWIN_IMPORT(false, g_pfnHvVmSyncTsc, hv_vm_sync_tsc),
386
387	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuCreate, hv_vcpu_create),
388	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuDestroy, hv_vcpu_destroy),
389	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuSetSpace, hv_vcpu_set_space),
390	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadRegister, hv_vcpu_read_register),
391	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteRegister, hv_vcpu_write_register),
392	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadFpState, hv_vcpu_read_fpstate),
393	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteFpState, hv_vcpu_write_fpstate),
394	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuEnableNativeMsr, hv_vcpu_enable_native_msr),
395	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadMsr, hv_vcpu_read_msr),
396	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteMsr, hv_vcpu_write_msr),
397	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuFlush, hv_vcpu_flush),
398	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInvalidateTlb, hv_vcpu_invalidate_tlb),
399	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuRun, hv_vcpu_run),
400	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuRunUntil, hv_vcpu_run_until),
401	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInterrupt, hv_vcpu_interrupt),
402	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuGetExecTime, hv_vcpu_get_exec_time),
403	NEM_DARWIN_IMPORT(false, g_pfnHvVmxReadCapability, hv_vmx_read_capability),
404	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuReadVmcs, hv_vmx_vcpu_read_vmcs),
405	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuWriteVmcs, hv_vmx_vcpu_write_vmcs),
406	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuReadShadowVmcs, hv_vmx_vcpu_read_shadow_vmcs),
407	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuWriteShadowVmcs, hv_vmx_vcpu_write_shadow_vmcs),
408	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuSetShadowAccess, hv_vmx_vcpu_set_shadow_access),
409	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuSetApicAddress, hv_vmx_vcpu_set_apic_address),
410	NEM_DARWIN_IMPORT(true, g_pfnHvVmxGetMsrInfo, hv_vmx_get_msr_info),
411	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuGetCapWriteVmcs, hv_vmx_vcpu_get_cap_write_vmcs),
412	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuEnableManagedMsr, hv_vcpu_enable_managed_msr),
413	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuSetMsrAccess, hv_vcpu_set_msr_access)
414	#undef NEM_DARWIN_IMPORT
415	};
416
417
418	/*
419	* Let the preprocessor alias the APIs to import variables for better autocompletion.
420	*/
421	#ifndef IN_SLICKEDIT
422	# define hv_capability g_pfnHvCapability
423	# define hv_vm_create g_pfnHvVmCreate
424	# define hv_vm_destroy g_pfnHvVmDestroy
425	# define hv_vm_space_create g_pfnHvVmSpaceCreate
426	# define hv_vm_space_destroy g_pfnHvVmSpaceDestroy
427	# define hv_vm_map g_pfnHvVmMap
428	# define hv_vm_unmap g_pfnHvVmUnmap
429	# define hv_vm_protect g_pfnHvVmProtect
430	# define hv_vm_map_space g_pfnHvVmMapSpace
431	# define hv_vm_unmap_space g_pfnHvVmUnmapSpace
432	# define hv_vm_protect_space g_pfnHvVmProtectSpace
433	# define hv_vm_sync_tsc g_pfnHvVmSyncTsc
434
435	# define hv_vcpu_create g_pfnHvVCpuCreate
436	# define hv_vcpu_destroy g_pfnHvVCpuDestroy
437	# define hv_vcpu_set_space g_pfnHvVCpuSetSpace
438	# define hv_vcpu_read_register g_pfnHvVCpuReadRegister
439	# define hv_vcpu_write_register g_pfnHvVCpuWriteRegister
440	# define hv_vcpu_read_fpstate g_pfnHvVCpuReadFpState
441	# define hv_vcpu_write_fpstate g_pfnHvVCpuWriteFpState
442	# define hv_vcpu_enable_native_msr g_pfnHvVCpuEnableNativeMsr
443	# define hv_vcpu_read_msr g_pfnHvVCpuReadMsr
444	# define hv_vcpu_write_msr g_pfnHvVCpuWriteMsr
445	# define hv_vcpu_flush g_pfnHvVCpuFlush
446	# define hv_vcpu_invalidate_tlb g_pfnHvVCpuInvalidateTlb
447	# define hv_vcpu_run g_pfnHvVCpuRun
448	# define hv_vcpu_run_until g_pfnHvVCpuRunUntil
449	# define hv_vcpu_interrupt g_pfnHvVCpuInterrupt
450	# define hv_vcpu_get_exec_time g_pfnHvVCpuGetExecTime
451
452	# define hv_vmx_read_capability g_pfnHvVmxReadCapability
453	# define hv_vmx_vcpu_read_vmcs g_pfnHvVmxVCpuReadVmcs
454	# define hv_vmx_vcpu_write_vmcs g_pfnHvVmxVCpuWriteVmcs
455	# define hv_vmx_vcpu_read_shadow_vmcs g_pfnHvVmxVCpuReadShadowVmcs
456	# define hv_vmx_vcpu_write_shadow_vmcs g_pfnHvVmxVCpuWriteShadowVmcs
457	# define hv_vmx_vcpu_set_shadow_access g_pfnHvVmxVCpuSetShadowAccess
458	# define hv_vmx_vcpu_set_apic_address g_pfnHvVmxVCpuSetApicAddress
459
460	# define hv_vmx_get_msr_info g_pfnHvVmxGetMsrInfo
461	# define hv_vmx_vcpu_get_cap_write_vmcs g_pfnHvVmxVCpuGetCapWriteVmcs
462	# define hv_vcpu_enable_managed_msr g_pfnHvVCpuEnableManagedMsr
463	# define hv_vcpu_set_msr_access g_pfnHvVCpuSetMsrAccess
464	#endif
465
466	static const struct
467	{
468	uint32_t u32VmcsFieldId; /*< The VMCS field identifier. /
469	const char pszVmcsField; /< The VMCS field name. /
470	bool f64Bit;
471	} g_aVmcsFieldsCap[] =
472	{
473	#define NEM_DARWIN_VMCS64_FIELD_CAP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, true }
474	#define NEM_DARWIN_VMCS32_FIELD_CAP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, false }
475
476	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PIN_EXEC),
477	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PROC_EXEC),
478	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_EXCEPTION_BITMAP),
479	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_EXIT),
480	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_ENTRY),
481	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PROC_EXEC2),
482	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PLE_GAP),
483	NEM_DARWIN_VMCS32_FIELD_CAP(VMX_VMCS32_CTRL_PLE_WINDOW),
484	NEM_DARWIN_VMCS64_FIELD_CAP(VMX_VMCS64_CTRL_TSC_OFFSET_FULL),
485	NEM_DARWIN_VMCS64_FIELD_CAP(VMX_VMCS64_GUEST_DEBUGCTL_FULL)
486	#undef NEM_DARWIN_VMCS64_FIELD_CAP
487	#undef NEM_DARWIN_VMCS32_FIELD_CAP
488	};
489
490
491	/*********************************************************************************************************************************
492	* Internal Functions *
493	*********************************************************************************************************************************/
494	DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo);
495
496
497	/**
498	* Converts a HV return code to a VBox status code.
499	*
500	* @returns VBox status code.
501	* @param hrc The HV return code to convert.
502	*/
503	DECLINLINE(int) nemR3DarwinHvSts2Rc(hv_return_t hrc)
504	{
505	if (hrc == HV_SUCCESS)
506	return VINF_SUCCESS;
507
508	switch (hrc)
509	{
510	case HV_ERROR: return VERR_INVALID_STATE;
511	case HV_BUSY: return VERR_RESOURCE_BUSY;
512	case HV_BAD_ARGUMENT: return VERR_INVALID_PARAMETER;
513	case HV_NO_RESOURCES: return VERR_OUT_OF_RESOURCES;
514	case HV_NO_DEVICE: return VERR_NOT_FOUND;
515	case HV_UNSUPPORTED: return VERR_NOT_SUPPORTED;
516	}
517
518	return VERR_IPE_UNEXPECTED_STATUS;
519	}
520
521
522	/**
523	* Unmaps the given guest physical address range (page aligned).
524	*
525	* @returns VBox status code.
526	* @param pVM The cross context VM structure.
527	* @param GCPhys The guest physical address to start unmapping at.
528	* @param cb The size of the range to unmap in bytes.
529	* @param pu2State Where to store the new state of the unmappd page, optional.
530	*/
531	DECLINLINE(int) nemR3DarwinUnmap(PVM pVM, RTGCPHYS GCPhys, size_t cb, uint8_t *pu2State)
532	{
533	if (*pu2State == NEM_DARWIN_PAGE_STATE_UNMAPPED)
534	{
535	Log5(("nemR3DarwinUnmap: %RGp == unmapped\n", GCPhys));
536	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
537	return VINF_SUCCESS;
538	}
539
540	LogFlowFunc(("Unmapping %RGp LB %zu\n", GCPhys, cb));
541	hv_return_t hrc;
542	if (pVM->nem.s.fCreatedAsid)
543	hrc = hv_vm_unmap_space(pVM->nem.s.uVmAsid, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, cb);
544	else
545	hrc = hv_vm_unmap(GCPhys, cb);
546	if (RT_LIKELY(hrc == HV_SUCCESS))
547	{
548	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
549	if (pu2State)
550	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
551	Log5(("nemR3DarwinUnmap: %RGp => unmapped\n", GCPhys));
552	return VINF_SUCCESS;
553	}
554
555	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
556	LogRel(("nemR3DarwinUnmap(%RGp): failed! hrc=%#x\n",
557	GCPhys, hrc));
558	return VERR_NEM_IPE_6;
559	}
560
561
562	/**
563	* Resolves a NEM page state from the given protection flags.
564	*
565	* @returns NEM page state.
566	* @param fPageProt The page protection flags.
567	*/
568	DECLINLINE(uint8_t) nemR3DarwinPageStateFromProt(uint32_t fPageProt)
569	{
570	switch (fPageProt)
571	{
572	case NEM_PAGE_PROT_NONE:
573	return NEM_DARWIN_PAGE_STATE_UNMAPPED;
574	case NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE:
575	return NEM_DARWIN_PAGE_STATE_RX;
576	case NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE:
577	return NEM_DARWIN_PAGE_STATE_RW;
578	case NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE:
579	return NEM_DARWIN_PAGE_STATE_RWX;
580	default:
581	break;
582	}
583
584	AssertLogRelMsgFailed(("Invalid combination of page protection flags %#x, can't map to page state!\n", fPageProt));
585	return NEM_DARWIN_PAGE_STATE_UNMAPPED;
586	}
587
588
589	/**
590	* Maps a given guest physical address range backed by the given memory with the given
591	* protection flags.
592	*
593	* @returns VBox status code.
594	* @param pVM The cross context VM structure.
595	* @param GCPhys The guest physical address to start mapping.
596	* @param pvRam The R3 pointer of the memory to back the range with.
597	* @param cb The size of the range, page aligned.
598	* @param fPageProt The page protection flags to use for this range, combination of NEM_PAGE_PROT_XXX
599	* @param pu2State Where to store the state for the new page, optional.
600	*/
601	DECLINLINE(int) nemR3DarwinMap(PVM pVM, RTGCPHYS GCPhys, const void pvRam, size_t cb, uint32_t fPageProt, uint8_t pu2State)
602	{
603	LogFlowFunc(("Mapping %RGp LB %zu fProt=%#x\n", GCPhys, cb, fPageProt));
604
605	Assert(fPageProt != NEM_PAGE_PROT_NONE);
606
607	hv_memory_flags_t fHvMemProt = 0;
608	if (fPageProt & NEM_PAGE_PROT_READ)
609	fHvMemProt \|= HV_MEMORY_READ;
610	if (fPageProt & NEM_PAGE_PROT_WRITE)
611	fHvMemProt \|= HV_MEMORY_WRITE;
612	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
613	fHvMemProt \|= HV_MEMORY_EXEC;
614
615	hv_return_t hrc;
616	if (pVM->nem.s.fCreatedAsid)
617	hrc = hv_vm_map_space(pVM->nem.s.uVmAsid, pvRam, GCPhys, cb, fHvMemProt);
618	else
619	hrc = hv_vm_map(pvRam, GCPhys, cb, fHvMemProt);
620	if (hrc == HV_SUCCESS)
621	{
622	if (pu2State)
623	*pu2State = nemR3DarwinPageStateFromProt(fPageProt);
624	return VINF_SUCCESS;
625	}
626
627	return nemR3DarwinHvSts2Rc(hrc);
628	}
629
630
631	/**
632	* Changes the protection flags for the given guest physical address range.
633	*
634	* @returns VBox status code.
635	* @param pVM The cross context VM structure.
636	* @param GCPhys The guest physical address to start mapping.
637	* @param cb The size of the range, page aligned.
638	* @param fPageProt The page protection flags to use for this range, combination of NEM_PAGE_PROT_XXX
639	* @param pu2State Where to store the state for the new page, optional.
640	*/
641	DECLINLINE(int) nemR3DarwinProtect(PVM pVM, RTGCPHYS GCPhys, size_t cb, uint32_t fPageProt, uint8_t *pu2State)
642	{
643	hv_memory_flags_t fHvMemProt = 0;
644	if (fPageProt & NEM_PAGE_PROT_READ)
645	fHvMemProt \|= HV_MEMORY_READ;
646	if (fPageProt & NEM_PAGE_PROT_WRITE)
647	fHvMemProt \|= HV_MEMORY_WRITE;
648	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
649	fHvMemProt \|= HV_MEMORY_EXEC;
650
651	hv_return_t hrc;
652	if (pVM->nem.s.fCreatedAsid)
653	hrc = hv_vm_protect_space(pVM->nem.s.uVmAsid, GCPhys, cb, fHvMemProt);
654	else
655	hrc = hv_vm_protect(GCPhys, cb, fHvMemProt);
656	if (hrc == HV_SUCCESS)
657	{
658	if (pu2State)
659	*pu2State = nemR3DarwinPageStateFromProt(fPageProt);
660	return VINF_SUCCESS;
661	}
662
663	return nemR3DarwinHvSts2Rc(hrc);
664	}
665
666
667	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
668	{
669	PGMPAGEMAPLOCK Lock;
670	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
671	if (RT_SUCCESS(rc))
672	PGMPhysReleasePageMappingLock(pVM, &Lock);
673	return rc;
674	}
675
676
677	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
678	{
679	PGMPAGEMAPLOCK Lock;
680	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
681	if (RT_SUCCESS(rc))
682	PGMPhysReleasePageMappingLock(pVM, &Lock);
683	return rc;
684	}
685
686
687	#ifdef LOG_ENABLED
688	/**
689	* Logs the current CPU state.
690	*/
691	static void nemR3DarwinLogState(PVMCC pVM, PVMCPUCC pVCpu)
692	{
693	if (LogIs3Enabled())
694	{
695	#if 0
696	char szRegs[4096];
697	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
698	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
699	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
700	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
701	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
702	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
703	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
704	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
705	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
706	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
707	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
708	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
709	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
710	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
711	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
712	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
713	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
714	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
715	" efer=%016VR{efer}\n"
716	" pat=%016VR{pat}\n"
717	" sf_mask=%016VR{sf_mask}\n"
718	"krnl_gs_base=%016VR{krnl_gs_base}\n"
719	" lstar=%016VR{lstar}\n"
720	" star=%016VR{star} cstar=%016VR{cstar}\n"
721	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
722	);
723
724	char szInstr[256];
725	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
726	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
727	szInstr, sizeof(szInstr), NULL);
728	Log3(("%s%s\n", szRegs, szInstr));
729	#else
730	RT_NOREF(pVM, pVCpu);
731	#endif
732	}
733	}
734	#endif /* LOG_ENABLED */
735
736
737	DECLINLINE(int) nemR3DarwinReadVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t *pData)
738	{
739	uint64_t u64Data;
740	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
741	if (RT_LIKELY(hrc == HV_SUCCESS))
742	{
743	*pData = (uint16_t)u64Data;
744	return VINF_SUCCESS;
745	}
746
747	return nemR3DarwinHvSts2Rc(hrc);
748	}
749
750
751	DECLINLINE(int) nemR3DarwinReadVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t *pData)
752	{
753	uint64_t u64Data;
754	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
755	if (RT_LIKELY(hrc == HV_SUCCESS))
756	{
757	*pData = (uint32_t)u64Data;
758	return VINF_SUCCESS;
759	}
760
761	return nemR3DarwinHvSts2Rc(hrc);
762	}
763
764
765	DECLINLINE(int) nemR3DarwinReadVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t *pData)
766	{
767	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, pData);
768	if (RT_LIKELY(hrc == HV_SUCCESS))
769	return VINF_SUCCESS;
770
771	return nemR3DarwinHvSts2Rc(hrc);
772	}
773
774
775	DECLINLINE(int) nemR3DarwinWriteVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t u16Val)
776	{
777	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u16Val);
778	if (RT_LIKELY(hrc == HV_SUCCESS))
779	return VINF_SUCCESS;
780
781	return nemR3DarwinHvSts2Rc(hrc);
782	}
783
784
785	DECLINLINE(int) nemR3DarwinWriteVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t u32Val)
786	{
787	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u32Val);
788	if (RT_LIKELY(hrc == HV_SUCCESS))
789	return VINF_SUCCESS;
790
791	return nemR3DarwinHvSts2Rc(hrc);
792	}
793
794
795	DECLINLINE(int) nemR3DarwinWriteVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t u64Val)
796	{
797	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u64Val);
798	if (RT_LIKELY(hrc == HV_SUCCESS))
799	return VINF_SUCCESS;
800
801	return nemR3DarwinHvSts2Rc(hrc);
802	}
803
804	DECLINLINE(int) nemR3DarwinMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Val)
805	{
806	hv_return_t hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, idMsr, pu64Val);
807	if (RT_LIKELY(hrc == HV_SUCCESS))
808	return VINF_SUCCESS;
809
810	return nemR3DarwinHvSts2Rc(hrc);
811	}
812
813	#if 0 /unused/
814	DECLINLINE(int) nemR3DarwinMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Val)
815	{
816	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, idMsr, u64Val);
817	if (RT_LIKELY(hrc == HV_SUCCESS))
818	return VINF_SUCCESS;
819
820	return nemR3DarwinHvSts2Rc(hrc);
821	}
822	#endif
823
824	static int nemR3DarwinCopyStateFromHv(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
825	{
826	#define READ_GREG(a_GReg, a_Value) \
827	do \
828	{ \
829	hrc = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, (a_GReg), &(a_Value)); \
830	if (RT_LIKELY(hrc == HV_SUCCESS)) \
831	{ /* likely */ } \
832	else \
833	return VERR_INTERNAL_ERROR; \
834	} while(0)
835	#define READ_VMCS_FIELD(a_Field, a_Value) \
836	do \
837	{ \
838	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &(a_Value)); \
839	if (RT_LIKELY(hrc == HV_SUCCESS)) \
840	{ /* likely */ } \
841	else \
842	return VERR_INTERNAL_ERROR; \
843	} while(0)
844	#define READ_VMCS16_FIELD(a_Field, a_Value) \
845	do \
846	{ \
847	uint64_t u64Data; \
848	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
849	if (RT_LIKELY(hrc == HV_SUCCESS)) \
850	{ (a_Value) = (uint16_t)u64Data; } \
851	else \
852	return VERR_INTERNAL_ERROR; \
853	} while(0)
854	#define READ_VMCS32_FIELD(a_Field, a_Value) \
855	do \
856	{ \
857	uint64_t u64Data; \
858	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
859	if (RT_LIKELY(hrc == HV_SUCCESS)) \
860	{ (a_Value) = (uint32_t)u64Data; } \
861	else \
862	return VERR_INTERNAL_ERROR; \
863	} while(0)
864	#define READ_MSR(a_Msr, a_Value) \
865	do \
866	{ \
867	hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, (a_Msr), &(a_Value)); \
868	if (RT_LIKELY(hrc == HV_SUCCESS)) \
869	{ /* likely */ } \
870	else \
871	AssertFailedReturn(VERR_INTERNAL_ERROR); \
872	} while(0)
873
874	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateImport, x);
875
876	RT_NOREF(pVM);
877	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
878
879	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
880	vmxHCImportGuestIntrState(pVCpu, &pVCpu->nem.s.VmcsInfo);
881
882	/* GPRs */
883	hv_return_t hrc;
884	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
885	{
886	if (fWhat & CPUMCTX_EXTRN_RAX)
887	READ_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
888	if (fWhat & CPUMCTX_EXTRN_RCX)
889	READ_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
890	if (fWhat & CPUMCTX_EXTRN_RDX)
891	READ_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
892	if (fWhat & CPUMCTX_EXTRN_RBX)
893	READ_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
894	if (fWhat & CPUMCTX_EXTRN_RSP)
895	READ_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
896	if (fWhat & CPUMCTX_EXTRN_RBP)
897	READ_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
898	if (fWhat & CPUMCTX_EXTRN_RSI)
899	READ_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
900	if (fWhat & CPUMCTX_EXTRN_RDI)
901	READ_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
902	if (fWhat & CPUMCTX_EXTRN_R8_R15)
903	{
904	READ_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
905	READ_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
906	READ_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
907	READ_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
908	READ_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
909	READ_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
910	READ_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
911	READ_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
912	}
913	}
914
915	/* RIP & Flags */
916	if (fWhat & CPUMCTX_EXTRN_RIP)
917	READ_GREG(HV_X86_RIP, pVCpu->cpum.GstCtx.rip);
918	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
919	{
920	uint64_t fRFlagsTmp = 0;
921	READ_GREG(HV_X86_RFLAGS, fRFlagsTmp);
922	pVCpu->cpum.GstCtx.rflags.u = fRFlagsTmp;
923	}
924
925	/* Segments */
926	#define READ_SEG(a_SReg, a_enmName) \
927	do { \
928	READ_VMCS16_FIELD(VMX_VMCS16_GUEST_ ## a_enmName ## _SEL, (a_SReg).Sel); \
929	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _LIMIT, (a_SReg).u32Limit); \
930	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _ACCESS_RIGHTS, (a_SReg).Attr.u); \
931	READ_VMCS_FIELD(VMX_VMCS_GUEST_ ## a_enmName ## _BASE, (a_SReg).u64Base); \
932	(a_SReg).ValidSel = (a_SReg).Sel; \
933	} while (0)
934	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
935	{
936	if (fWhat & CPUMCTX_EXTRN_ES)
937	READ_SEG(pVCpu->cpum.GstCtx.es, ES);
938	if (fWhat & CPUMCTX_EXTRN_CS)
939	READ_SEG(pVCpu->cpum.GstCtx.cs, CS);
940	if (fWhat & CPUMCTX_EXTRN_SS)
941	READ_SEG(pVCpu->cpum.GstCtx.ss, SS);
942	if (fWhat & CPUMCTX_EXTRN_DS)
943	READ_SEG(pVCpu->cpum.GstCtx.ds, DS);
944	if (fWhat & CPUMCTX_EXTRN_FS)
945	READ_SEG(pVCpu->cpum.GstCtx.fs, FS);
946	if (fWhat & CPUMCTX_EXTRN_GS)
947	READ_SEG(pVCpu->cpum.GstCtx.gs, GS);
948	}
949
950	/* Descriptor tables and the task segment. */
951	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
952	{
953	if (fWhat & CPUMCTX_EXTRN_LDTR)
954	READ_SEG(pVCpu->cpum.GstCtx.ldtr, LDTR);
955
956	if (fWhat & CPUMCTX_EXTRN_TR)
957	{
958	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
959	avoid to trigger sanity assertions around the code, always fix this. */
960	READ_SEG(pVCpu->cpum.GstCtx.tr, TR);
961	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
962	{
963	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
964	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
965	break;
966	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
967	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
968	break;
969	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
970	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
971	break;
972	}
973	}
974	if (fWhat & CPUMCTX_EXTRN_IDTR)
975	{
976	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_IDTR_LIMIT, pVCpu->cpum.GstCtx.idtr.cbIdt);
977	READ_VMCS_FIELD(VMX_VMCS_GUEST_IDTR_BASE, pVCpu->cpum.GstCtx.idtr.pIdt);
978	}
979	if (fWhat & CPUMCTX_EXTRN_GDTR)
980	{
981	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_GDTR_LIMIT, pVCpu->cpum.GstCtx.gdtr.cbGdt);
982	READ_VMCS_FIELD(VMX_VMCS_GUEST_GDTR_BASE, pVCpu->cpum.GstCtx.gdtr.pGdt);
983	}
984	}
985
986	/* Control registers. */
987	bool fMaybeChangedMode = false;
988	bool fUpdateCr3 = false;
989	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
990	{
991	uint64_t u64CrTmp = 0;
992
993	if (fWhat & CPUMCTX_EXTRN_CR0)
994	{
995	READ_GREG(HV_X86_CR0, u64CrTmp);
996	if (pVCpu->cpum.GstCtx.cr0 != u64CrTmp)
997	{
998	CPUMSetGuestCR0(pVCpu, u64CrTmp);
999	fMaybeChangedMode = true;
1000	}
1001	}
1002	if (fWhat & CPUMCTX_EXTRN_CR2)
1003	READ_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
1004	if (fWhat & CPUMCTX_EXTRN_CR3)
1005	{
1006	READ_GREG(HV_X86_CR3, u64CrTmp);
1007	if (pVCpu->cpum.GstCtx.cr3 != u64CrTmp)
1008	{
1009	CPUMSetGuestCR3(pVCpu, u64CrTmp);
1010	fUpdateCr3 = true;
1011	}
1012
1013	/*
1014	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
1015	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
1016	*/
1017	if (CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx))
1018	{
1019	X86PDPE aPaePdpes[4];
1020	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE0_FULL, aPaePdpes[0].u);
1021	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE1_FULL, aPaePdpes[1].u);
1022	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE2_FULL, aPaePdpes[2].u);
1023	READ_VMCS_FIELD(VMX_VMCS64_GUEST_PDPTE3_FULL, aPaePdpes[3].u);
1024	if (memcmp(&aPaePdpes[0], &pVCpu->cpum.GstCtx.aPaePdpes[0], sizeof(aPaePdpes)))
1025	{
1026	memcpy(&pVCpu->cpum.GstCtx.aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
1027	fUpdateCr3 = true;
1028	}
1029	}
1030	}
1031	if (fWhat & CPUMCTX_EXTRN_CR4)
1032	{
1033	READ_GREG(HV_X86_CR4, u64CrTmp);
1034	u64CrTmp &= ~VMX_V_CR4_FIXED0;
1035
1036	if (pVCpu->cpum.GstCtx.cr4 != u64CrTmp)
1037	{
1038	CPUMSetGuestCR4(pVCpu, u64CrTmp);
1039	fMaybeChangedMode = true;
1040	}
1041	}
1042	}
1043
1044	#if 0 /* Always done. */
1045	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1046	{
1047	uint64_t u64Cr8 = 0;
1048
1049	READ_GREG(HV_X86_TPR, u64Cr8);
1050	APICSetTpr(pVCpu, u64Cr8 << 4);
1051	}
1052	#endif
1053
1054	if (fWhat & CPUMCTX_EXTRN_XCRx)
1055	READ_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
1056
1057	/* Debug registers. */
1058	if (fWhat & CPUMCTX_EXTRN_DR7)
1059	{
1060	uint64_t u64Dr7;
1061	READ_GREG(HV_X86_DR7, u64Dr7);
1062	if (pVCpu->cpum.GstCtx.dr[7] != u64Dr7)
1063	CPUMSetGuestDR7(pVCpu, u64Dr7);
1064	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
1065	}
1066	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1067	{
1068	uint64_t u64DrTmp;
1069
1070	READ_GREG(HV_X86_DR0, u64DrTmp);
1071	if (pVCpu->cpum.GstCtx.dr[0] != u64DrTmp)
1072	CPUMSetGuestDR0(pVCpu, u64DrTmp);
1073	READ_GREG(HV_X86_DR1, u64DrTmp);
1074	if (pVCpu->cpum.GstCtx.dr[1] != u64DrTmp)
1075	CPUMSetGuestDR1(pVCpu, u64DrTmp);
1076	READ_GREG(HV_X86_DR2, u64DrTmp);
1077	if (pVCpu->cpum.GstCtx.dr[2] != u64DrTmp)
1078	CPUMSetGuestDR2(pVCpu, u64DrTmp);
1079	READ_GREG(HV_X86_DR3, u64DrTmp);
1080	if (pVCpu->cpum.GstCtx.dr[3] != u64DrTmp)
1081	CPUMSetGuestDR3(pVCpu, u64DrTmp);
1082	}
1083	if (fWhat & CPUMCTX_EXTRN_DR6)
1084	{
1085	uint64_t u64Dr6;
1086	READ_GREG(HV_X86_DR6, u64Dr6);
1087	if (pVCpu->cpum.GstCtx.dr[6] != u64Dr6)
1088	CPUMSetGuestDR6(pVCpu, u64Dr6);
1089	}
1090
1091	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
1092	{
1093	hrc = hv_vcpu_read_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1094	if (hrc == HV_SUCCESS)
1095	{ /* likely */ }
1096	else
1097	{
1098	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1099	return nemR3DarwinHvSts2Rc(hrc);
1100	}
1101	}
1102
1103	/* MSRs */
1104	if (fWhat & CPUMCTX_EXTRN_EFER)
1105	{
1106	uint64_t u64Efer;
1107
1108	READ_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, u64Efer);
1109	if (u64Efer != pVCpu->cpum.GstCtx.msrEFER)
1110	{
1111	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, u64Efer));
1112	if ((u64Efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1113	PGMNotifyNxeChanged(pVCpu, RT_BOOL(u64Efer & MSR_K6_EFER_NXE));
1114	pVCpu->cpum.GstCtx.msrEFER = u64Efer;
1115	fMaybeChangedMode = true;
1116	}
1117	}
1118
1119	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1120	READ_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1121	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1122	{
1123	uint64_t u64Tmp;
1124	READ_MSR(MSR_IA32_SYSENTER_EIP, u64Tmp);
1125	pVCpu->cpum.GstCtx.SysEnter.eip = u64Tmp;
1126	READ_MSR(MSR_IA32_SYSENTER_ESP, u64Tmp);
1127	pVCpu->cpum.GstCtx.SysEnter.esp = u64Tmp;
1128	READ_MSR(MSR_IA32_SYSENTER_CS, u64Tmp);
1129	pVCpu->cpum.GstCtx.SysEnter.cs = u64Tmp;
1130	}
1131	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1132	{
1133	READ_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1134	READ_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1135	READ_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1136	READ_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1137	}
1138	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1139	{
1140	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1141	READ_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1142	}
1143	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1144	{
1145	/* Last Branch Record. */
1146	if (pVM->nem.s.fLbr)
1147	{
1148	PVMXVMCSINFOSHARED const pVmcsInfoShared = &pVCpu->nem.s.vmx.VmcsInfo;
1149	uint32_t const idFromIpMsrStart = pVM->nem.s.idLbrFromIpMsrFirst;
1150	uint32_t const idToIpMsrStart = pVM->nem.s.idLbrToIpMsrFirst;
1151	uint32_t const idInfoMsrStart = pVM->nem.s.idLbrInfoMsrFirst;
1152	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
1153	Assert(cLbrStack <= 32);
1154	for (uint32_t i = 0; i < cLbrStack; i++)
1155	{
1156	READ_MSR(idFromIpMsrStart + i, pVmcsInfoShared->au64LbrFromIpMsr[i]);
1157
1158	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
1159	if (idToIpMsrStart != 0)
1160	READ_MSR(idToIpMsrStart + i, pVmcsInfoShared->au64LbrToIpMsr[i]);
1161	if (idInfoMsrStart != 0)
1162	READ_MSR(idInfoMsrStart + i, pVmcsInfoShared->au64LbrInfoMsr[i]);
1163	}
1164
1165	READ_MSR(pVM->nem.s.idLbrTosMsr, pVmcsInfoShared->u64LbrTosMsr);
1166
1167	if (pVM->nem.s.idLerFromIpMsr)
1168	READ_MSR(pVM->nem.s.idLerFromIpMsr, pVmcsInfoShared->u64LerFromIpMsr);
1169	if (pVM->nem.s.idLerToIpMsr)
1170	READ_MSR(pVM->nem.s.idLerToIpMsr, pVmcsInfoShared->u64LerToIpMsr);
1171	}
1172	}
1173
1174	/* Almost done, just update extrn flags and maybe change PGM mode. */
1175	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1176	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1177	pVCpu->cpum.GstCtx.fExtrn = 0;
1178
1179	#ifdef LOG_ENABLED
1180	nemR3DarwinLogState(pVM, pVCpu);
1181	#endif
1182
1183	/* Typical. */
1184	if (!fMaybeChangedMode && !fUpdateCr3)
1185	{
1186	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1187	return VINF_SUCCESS;
1188	}
1189
1190	/*
1191	* Slow.
1192	*/
1193	if (fMaybeChangedMode)
1194	{
1195	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
1196	false /* fForce */);
1197	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1198	}
1199
1200	if (fUpdateCr3)
1201	{
1202	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
1203	if (rc == VINF_SUCCESS)
1204	{ /* likely */ }
1205	else
1206	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1207	}
1208
1209	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1210
1211	return VINF_SUCCESS;
1212	#undef READ_GREG
1213	#undef READ_VMCS_FIELD
1214	#undef READ_VMCS32_FIELD
1215	#undef READ_SEG
1216	#undef READ_MSR
1217	}
1218
1219
1220	/**
1221	* State to pass between vmxHCExitEptViolation
1222	* and nemR3DarwinHandleMemoryAccessPageCheckerCallback.
1223	*/
1224	typedef struct NEMHCDARWINHMACPCCSTATE
1225	{
1226	/** Input: Write access. */
1227	bool fWriteAccess;
1228	/** Output: Set if we did something. */
1229	bool fDidSomething;
1230	/** Output: Set it we should resume. */
1231	bool fCanResume;
1232	} NEMHCDARWINHMACPCCSTATE;
1233
1234	/**
1235	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1236	* Worker for vmxHCExitEptViolation; pvUser points to a
1237	* NEMHCDARWINHMACPCCSTATE structure. }
1238	*/
1239	static DECLCALLBACK(int)
1240	nemR3DarwinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1241	{
1242	RT_NOREF(pVCpu);
1243
1244	NEMHCDARWINHMACPCCSTATE pState = (NEMHCDARWINHMACPCCSTATE )pvUser;
1245	pState->fDidSomething = false;
1246	pState->fCanResume = false;
1247
1248	uint8_t u2State = pInfo->u2NemState;
1249
1250	/*
1251	* Consolidate current page state with actual page protection and access type.
1252	* We don't really consider downgrades here, as they shouldn't happen.
1253	*/
1254	switch (u2State)
1255	{
1256	case NEM_DARWIN_PAGE_STATE_UNMAPPED:
1257	{
1258	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1259	{
1260	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1261	return VINF_SUCCESS;
1262	}
1263
1264	/* Don't bother remapping it if it's a write request to a non-writable page. */
1265	if ( pState->fWriteAccess
1266	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1267	{
1268	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1269	return VINF_SUCCESS;
1270	}
1271
1272	int rc = VINF_SUCCESS;
1273	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1274	{
1275	void *pvPage;
1276	rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhys, &pvPage);
1277	if (RT_SUCCESS(rc))
1278	rc = nemR3DarwinMap(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, pvPage, X86_PAGE_SIZE, pInfo->fNemProt, &u2State);
1279	}
1280	else if (pInfo->fNemProt & NEM_PAGE_PROT_READ)
1281	{
1282	const void *pvPage;
1283	rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhys, &pvPage);
1284	if (RT_SUCCESS(rc))
1285	rc = nemR3DarwinMap(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, pvPage, X86_PAGE_SIZE, pInfo->fNemProt, &u2State);
1286	}
1287	else /* Only EXECUTE doesn't work. */
1288	AssertReleaseFailed();
1289
1290	pInfo->u2NemState = u2State;
1291	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1292	GCPhys, g_apszPageStates[u2State], rc));
1293	pState->fDidSomething = true;
1294	pState->fCanResume = true;
1295	return rc;
1296	}
1297	case NEM_DARWIN_PAGE_STATE_RX:
1298	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1299	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1300	{
1301	pState->fCanResume = true;
1302	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1303	return VINF_SUCCESS;
1304	}
1305	break;
1306
1307	case NEM_DARWIN_PAGE_STATE_RW:
1308	case NEM_DARWIN_PAGE_STATE_RWX:
1309	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1310	{
1311	pState->fCanResume = true;
1312	if ( pInfo->u2OldNemState == NEM_DARWIN_PAGE_STATE_RW
1313	\|\| pInfo->u2OldNemState == NEM_DARWIN_PAGE_STATE_RWX)
1314	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: Spurious EPT fault\n", GCPhys));
1315	return VINF_SUCCESS;
1316	}
1317	break;
1318
1319	default:
1320	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1321	}
1322
1323	/* Unmap and restart the instruction. */
1324	int rc = nemR3DarwinUnmap(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE, &u2State);
1325	if (RT_SUCCESS(rc))
1326	{
1327	pInfo->u2NemState = u2State;
1328	pState->fDidSomething = true;
1329	pState->fCanResume = true;
1330	Log5(("NEM GPA unmapped/exit: %RGp (was %s)\n", GCPhys, g_apszPageStates[u2State]));
1331	return VINF_SUCCESS;
1332	}
1333
1334	LogRel(("nemR3DarwinHandleMemoryAccessPageCheckerCallback/unmap: GCPhys=%RGp %s rc=%Rrc\n",
1335	GCPhys, g_apszPageStates[u2State], rc));
1336	return VERR_NEM_UNMAP_PAGES_FAILED;
1337	}
1338
1339
1340	DECL_FORCE_INLINE(bool) nemR3DarwinIsUnrestrictedGuest(PCVMCC pVM)
1341	{
1342	RT_NOREF(pVM);
1343	return true;
1344	}
1345
1346
1347	DECL_FORCE_INLINE(bool) nemR3DarwinIsNestedPaging(PCVMCC pVM)
1348	{
1349	RT_NOREF(pVM);
1350	return true;
1351	}
1352
1353
1354	DECL_FORCE_INLINE(bool) nemR3DarwinIsPreemptTimerUsed(PCVMCC pVM)
1355	{
1356	RT_NOREF(pVM);
1357	return false;
1358	}
1359
1360
1361	#if 0 /* unused */
1362	DECL_FORCE_INLINE(bool) nemR3DarwinIsVmxLbr(PCVMCC pVM)
1363	{
1364	RT_NOREF(pVM);
1365	return false;
1366	}
1367	#endif
1368
1369
1370	/*
1371	* Instantiate the code we share with ring-0.
1372	*/
1373	#define IN_NEM_DARWIN
1374	//#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
1375	//#define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
1376	//#define HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
1377	#define VCPU_2_VMXSTATE(a_pVCpu) (a_pVCpu)->nem.s
1378	#define VCPU_2_VMXSTATS(a_pVCpu) (*(a_pVCpu)->nem.s.pVmxStats)
1379
1380	#define VM_IS_VMX_UNRESTRICTED_GUEST(a_pVM) nemR3DarwinIsUnrestrictedGuest((a_pVM))
1381	#define VM_IS_VMX_NESTED_PAGING(a_pVM) nemR3DarwinIsNestedPaging((a_pVM))
1382	#define VM_IS_VMX_PREEMPT_TIMER_USED(a_pVM) nemR3DarwinIsPreemptTimerUsed((a_pVM))
1383	#define VM_IS_VMX_LBR(a_pVM) nemR3DarwinIsVmxLbr((a_pVM))
1384
1385	#define VMX_VMCS_WRITE_16(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs16((a_pVCpu), (a_FieldEnc), (a_Val))
1386	#define VMX_VMCS_WRITE_32(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs32((a_pVCpu), (a_FieldEnc), (a_Val))
1387	#define VMX_VMCS_WRITE_64(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1388	#define VMX_VMCS_WRITE_NW(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1389
1390	#define VMX_VMCS_READ_16(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs16((a_pVCpu), (a_FieldEnc), (a_pVal))
1391	#define VMX_VMCS_READ_32(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs32((a_pVCpu), (a_FieldEnc), (a_pVal))
1392	#define VMX_VMCS_READ_64(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1393	#define VMX_VMCS_READ_NW(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1394
1395	#include "../VMMAll/VMXAllTemplate.cpp.h"
1396
1397	#undef VMX_VMCS_WRITE_16
1398	#undef VMX_VMCS_WRITE_32
1399	#undef VMX_VMCS_WRITE_64
1400	#undef VMX_VMCS_WRITE_NW
1401
1402	#undef VMX_VMCS_READ_16
1403	#undef VMX_VMCS_READ_32
1404	#undef VMX_VMCS_READ_64
1405	#undef VMX_VMCS_READ_NW
1406
1407	#undef VM_IS_VMX_PREEMPT_TIMER_USED
1408	#undef VM_IS_VMX_NESTED_PAGING
1409	#undef VM_IS_VMX_UNRESTRICTED_GUEST
1410	#undef VCPU_2_VMXSTATS
1411	#undef VCPU_2_VMXSTATE
1412
1413
1414	/**
1415	* Exports the guest GP registers to HV for execution.
1416	*
1417	* @returns VBox status code.
1418	* @param pVCpu The cross context virtual CPU structure of the
1419	* calling EMT.
1420	*/
1421	static int nemR3DarwinExportGuestGprs(PVMCPUCC pVCpu)
1422	{
1423	#define WRITE_GREG(a_GReg, a_Value) \
1424	do \
1425	{ \
1426	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1427	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1428	{ /* likely */ } \
1429	else \
1430	return VERR_INTERNAL_ERROR; \
1431	} while(0)
1432
1433	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->nem.s.fCtxChanged);
1434	if (fCtxChanged & HM_CHANGED_GUEST_GPRS_MASK)
1435	{
1436	if (fCtxChanged & HM_CHANGED_GUEST_RAX)
1437	WRITE_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
1438	if (fCtxChanged & HM_CHANGED_GUEST_RCX)
1439	WRITE_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
1440	if (fCtxChanged & HM_CHANGED_GUEST_RDX)
1441	WRITE_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
1442	if (fCtxChanged & HM_CHANGED_GUEST_RBX)
1443	WRITE_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
1444	if (fCtxChanged & HM_CHANGED_GUEST_RSP)
1445	WRITE_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
1446	if (fCtxChanged & HM_CHANGED_GUEST_RBP)
1447	WRITE_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
1448	if (fCtxChanged & HM_CHANGED_GUEST_RSI)
1449	WRITE_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
1450	if (fCtxChanged & HM_CHANGED_GUEST_RDI)
1451	WRITE_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
1452	if (fCtxChanged & HM_CHANGED_GUEST_R8_R15)
1453	{
1454	WRITE_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
1455	WRITE_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
1456	WRITE_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
1457	WRITE_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
1458	WRITE_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
1459	WRITE_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
1460	WRITE_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
1461	WRITE_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
1462	}
1463
1464	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_GPRS_MASK);
1465	}
1466
1467	if (fCtxChanged & HM_CHANGED_GUEST_CR2)
1468	{
1469	WRITE_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
1470	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_CR2);
1471	}
1472
1473	return VINF_SUCCESS;
1474	#undef WRITE_GREG
1475	}
1476
1477
1478	/**
1479	* Exports the guest debug registers into the guest-state applying any hypervisor
1480	* debug related states (hardware breakpoints from the debugger, etc.).
1481	*
1482	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
1483	*
1484	* @returns VBox status code.
1485	* @param pVCpu The cross context virtual CPU structure.
1486	* @param pVmxTransient The VMX-transient structure.
1487	*/
1488	static int nemR3DarwinExportDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1489	{
1490	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1491
1492	#ifdef VBOX_STRICT
1493	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
1494	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
1495	{
1496	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
1497	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
1498	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
1499	}
1500	#endif
1501
1502	bool fSteppingDB = false;
1503	bool fInterceptMovDRx = false;
1504	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
1505	if (pVCpu->nem.s.fSingleInstruction)
1506	{
1507	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
1508	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
1509	{
1510	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
1511	Assert(fSteppingDB == false);
1512	}
1513	else
1514	{
1515	pVCpu->cpum.GstCtx.eflags.u \|= X86_EFL_TF;
1516	pVCpu->nem.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
1517	pVCpu->nem.s.fClearTrapFlag = true;
1518	fSteppingDB = true;
1519	}
1520	}
1521
1522	uint64_t u64GuestDr7;
1523	if ( fSteppingDB
1524	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1525	{
1526	/*
1527	* Use the combined guest and host DRx values found in the hypervisor register set
1528	* because the hypervisor debugger has breakpoints active or someone is single stepping
1529	* on the host side without a monitor trap flag.
1530	*
1531	* Note! DBGF expects a clean DR6 state before executing guest code.
1532	*/
1533	if (!CPUMIsHyperDebugStateActive(pVCpu))
1534	{
1535	/*
1536	* Make sure the hypervisor values are up to date.
1537	*/
1538	CPUMRecalcHyperDRx(pVCpu, UINT8_MAX /* no loading, please */);
1539
1540	CPUMR3NemActivateHyperDebugState(pVCpu);
1541
1542	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1543	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1544	}
1545
1546	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
1547	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
1548	pVCpu->nem.s.fUsingHyperDR7 = true;
1549	fInterceptMovDRx = true;
1550	}
1551	else
1552	{
1553	/*
1554	* If the guest has enabled debug registers, we need to load them prior to
1555	* executing guest code so they'll trigger at the right time.
1556	*/
1557	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
1558	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
1559	{
1560	if (!CPUMIsGuestDebugStateActive(pVCpu))
1561	{
1562	CPUMR3NemActivateGuestDebugState(pVCpu);
1563
1564	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1565	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1566	}
1567	Assert(!fInterceptMovDRx);
1568	}
1569	else if (!CPUMIsGuestDebugStateActive(pVCpu))
1570	{
1571	/*
1572	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
1573	* must intercept #DB in order to maintain a correct DR6 guest value, and
1574	* because we need to intercept it to prevent nested #DBs from hanging the
1575	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
1576	*/
1577	fInterceptMovDRx = true;
1578	}
1579
1580	/* Update DR7 with the actual guest value. */
1581	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
1582	pVCpu->nem.s.fUsingHyperDR7 = false;
1583	}
1584
1585	/** @todo The DRx handling is not quite correct breaking debugging inside the guest with gdb,
1586	* see @ticketref{21413} and @ticketref{21546}, so this is disabled for now. See @bugref{10504}
1587	* as well.
1588	*/
1589	#if 0
1590	if (fInterceptMovDRx)
1591	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
1592	else
1593	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
1594	#endif
1595
1596	/*
1597	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
1598	* monitor-trap flag and update our cache.
1599	*/
1600	if (uProcCtls != pVmcsInfo->u32ProcCtls)
1601	{
1602	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
1603	AssertRC(rc);
1604	pVmcsInfo->u32ProcCtls = uProcCtls;
1605	}
1606
1607	/*
1608	* Update guest DR7.
1609	*/
1610	int rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_GUEST_DR7, u64GuestDr7);
1611	AssertRC(rc);
1612
1613	/*
1614	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
1615	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
1616	*
1617	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
1618	*/
1619	if (fSteppingDB)
1620	{
1621	Assert(pVCpu->nem.s.fSingleInstruction);
1622	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
1623
1624	uint32_t fIntrState = 0;
1625	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
1626	AssertRC(rc);
1627
1628	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
1629	{
1630	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
1631	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
1632	AssertRC(rc);
1633	}
1634	}
1635
1636	/*
1637	* Store status of the shared guest/host debug state at the time of VM-entry.
1638	*/
1639	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
1640	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
1641
1642	return VINF_SUCCESS;
1643	}
1644
1645
1646	/**
1647	* Converts the given CPUM externalized bitmask to the appropriate HM changed bitmask.
1648	*
1649	* @returns Bitmask of HM changed flags.
1650	* @param fCpumExtrn The CPUM extern bitmask.
1651	*/
1652	static uint64_t nemR3DarwinCpumExtrnToHmChanged(uint64_t fCpumExtrn)
1653	{
1654	uint64_t fHmChanged = 0;
1655
1656	/* Invert to gt a mask of things which are kept in CPUM. */
1657	uint64_t fCpumIntern = ~fCpumExtrn;
1658
1659	if (fCpumIntern & CPUMCTX_EXTRN_GPRS_MASK)
1660	{
1661	if (fCpumIntern & CPUMCTX_EXTRN_RAX)
1662	fHmChanged \|= HM_CHANGED_GUEST_RAX;
1663	if (fCpumIntern & CPUMCTX_EXTRN_RCX)
1664	fHmChanged \|= HM_CHANGED_GUEST_RCX;
1665	if (fCpumIntern & CPUMCTX_EXTRN_RDX)
1666	fHmChanged \|= HM_CHANGED_GUEST_RDX;
1667	if (fCpumIntern & CPUMCTX_EXTRN_RBX)
1668	fHmChanged \|= HM_CHANGED_GUEST_RBX;
1669	if (fCpumIntern & CPUMCTX_EXTRN_RSP)
1670	fHmChanged \|= HM_CHANGED_GUEST_RSP;
1671	if (fCpumIntern & CPUMCTX_EXTRN_RBP)
1672	fHmChanged \|= HM_CHANGED_GUEST_RBP;
1673	if (fCpumIntern & CPUMCTX_EXTRN_RSI)
1674	fHmChanged \|= HM_CHANGED_GUEST_RSI;
1675	if (fCpumIntern & CPUMCTX_EXTRN_RDI)
1676	fHmChanged \|= HM_CHANGED_GUEST_RDI;
1677	if (fCpumIntern & CPUMCTX_EXTRN_R8_R15)
1678	fHmChanged \|= HM_CHANGED_GUEST_R8_R15;
1679	}
1680
1681	/* RIP & Flags */
1682	if (fCpumIntern & CPUMCTX_EXTRN_RIP)
1683	fHmChanged \|= HM_CHANGED_GUEST_RIP;
1684	if (fCpumIntern & CPUMCTX_EXTRN_RFLAGS)
1685	fHmChanged \|= HM_CHANGED_GUEST_RFLAGS;
1686
1687	/* Segments */
1688	if (fCpumIntern & CPUMCTX_EXTRN_SREG_MASK)
1689	{
1690	if (fCpumIntern & CPUMCTX_EXTRN_ES)
1691	fHmChanged \|= HM_CHANGED_GUEST_ES;
1692	if (fCpumIntern & CPUMCTX_EXTRN_CS)
1693	fHmChanged \|= HM_CHANGED_GUEST_CS;
1694	if (fCpumIntern & CPUMCTX_EXTRN_SS)
1695	fHmChanged \|= HM_CHANGED_GUEST_SS;
1696	if (fCpumIntern & CPUMCTX_EXTRN_DS)
1697	fHmChanged \|= HM_CHANGED_GUEST_DS;
1698	if (fCpumIntern & CPUMCTX_EXTRN_FS)
1699	fHmChanged \|= HM_CHANGED_GUEST_FS;
1700	if (fCpumIntern & CPUMCTX_EXTRN_GS)
1701	fHmChanged \|= HM_CHANGED_GUEST_GS;
1702	}
1703
1704	/* Descriptor tables & task segment. */
1705	if (fCpumIntern & CPUMCTX_EXTRN_TABLE_MASK)
1706	{
1707	if (fCpumIntern & CPUMCTX_EXTRN_LDTR)
1708	fHmChanged \|= HM_CHANGED_GUEST_LDTR;
1709	if (fCpumIntern & CPUMCTX_EXTRN_TR)
1710	fHmChanged \|= HM_CHANGED_GUEST_TR;
1711	if (fCpumIntern & CPUMCTX_EXTRN_IDTR)
1712	fHmChanged \|= HM_CHANGED_GUEST_IDTR;
1713	if (fCpumIntern & CPUMCTX_EXTRN_GDTR)
1714	fHmChanged \|= HM_CHANGED_GUEST_GDTR;
1715	}
1716
1717	/* Control registers. */
1718	if (fCpumIntern & CPUMCTX_EXTRN_CR_MASK)
1719	{
1720	if (fCpumIntern & CPUMCTX_EXTRN_CR0)
1721	fHmChanged \|= HM_CHANGED_GUEST_CR0;
1722	if (fCpumIntern & CPUMCTX_EXTRN_CR2)
1723	fHmChanged \|= HM_CHANGED_GUEST_CR2;
1724	if (fCpumIntern & CPUMCTX_EXTRN_CR3)
1725	fHmChanged \|= HM_CHANGED_GUEST_CR3;
1726	if (fCpumIntern & CPUMCTX_EXTRN_CR4)
1727	fHmChanged \|= HM_CHANGED_GUEST_CR4;
1728	}
1729	if (fCpumIntern & CPUMCTX_EXTRN_APIC_TPR)
1730	fHmChanged \|= HM_CHANGED_GUEST_APIC_TPR;
1731
1732	/* Debug registers. */
1733	if (fCpumIntern & CPUMCTX_EXTRN_DR0_DR3)
1734	fHmChanged \|= HM_CHANGED_GUEST_DR0_DR3;
1735	if (fCpumIntern & CPUMCTX_EXTRN_DR6)
1736	fHmChanged \|= HM_CHANGED_GUEST_DR6;
1737	if (fCpumIntern & CPUMCTX_EXTRN_DR7)
1738	fHmChanged \|= HM_CHANGED_GUEST_DR7;
1739
1740	/* Floating point state. */
1741	if (fCpumIntern & CPUMCTX_EXTRN_X87)
1742	fHmChanged \|= HM_CHANGED_GUEST_X87;
1743	if (fCpumIntern & CPUMCTX_EXTRN_SSE_AVX)
1744	fHmChanged \|= HM_CHANGED_GUEST_SSE_AVX;
1745	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_XSAVE)
1746	fHmChanged \|= HM_CHANGED_GUEST_OTHER_XSAVE;
1747	if (fCpumIntern & CPUMCTX_EXTRN_XCRx)
1748	fHmChanged \|= HM_CHANGED_GUEST_XCRx;
1749
1750	/* MSRs */
1751	if (fCpumIntern & CPUMCTX_EXTRN_EFER)
1752	fHmChanged \|= HM_CHANGED_GUEST_EFER_MSR;
1753	if (fCpumIntern & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1754	fHmChanged \|= HM_CHANGED_GUEST_KERNEL_GS_BASE;
1755	if (fCpumIntern & CPUMCTX_EXTRN_SYSENTER_MSRS)
1756	fHmChanged \|= HM_CHANGED_GUEST_SYSENTER_MSR_MASK;
1757	if (fCpumIntern & CPUMCTX_EXTRN_SYSCALL_MSRS)
1758	fHmChanged \|= HM_CHANGED_GUEST_SYSCALL_MSRS;
1759	if (fCpumIntern & CPUMCTX_EXTRN_TSC_AUX)
1760	fHmChanged \|= HM_CHANGED_GUEST_TSC_AUX;
1761	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_MSRS)
1762	fHmChanged \|= HM_CHANGED_GUEST_OTHER_MSRS;
1763
1764	return fHmChanged;
1765	}
1766
1767
1768	/**
1769	* Exports the guest state to HV for execution.
1770	*
1771	* @returns VBox status code.
1772	* @param pVM The cross context VM structure.
1773	* @param pVCpu The cross context virtual CPU structure of the
1774	* calling EMT.
1775	* @param pVmxTransient The transient VMX structure.
1776	*/
1777	static int nemR3DarwinExportGuestState(PVMCC pVM, PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1778	{
1779	#define WRITE_GREG(a_GReg, a_Value) \
1780	do \
1781	{ \
1782	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1783	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1784	{ /* likely */ } \
1785	else \
1786	return VERR_INTERNAL_ERROR; \
1787	} while(0)
1788	#define WRITE_VMCS_FIELD(a_Field, a_Value) \
1789	do \
1790	{ \
1791	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), (a_Value)); \
1792	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1793	{ /* likely */ } \
1794	else \
1795	return VERR_INTERNAL_ERROR; \
1796	} while(0)
1797	#define WRITE_MSR(a_Msr, a_Value) \
1798	do \
1799	{ \
1800	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, (a_Msr), (a_Value)); \
1801	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1802	{ /* likely */ } \
1803	else \
1804	AssertFailedReturn(VERR_INTERNAL_ERROR); \
1805	} while(0)
1806
1807	RT_NOREF(pVM);
1808
1809	#ifdef LOG_ENABLED
1810	nemR3DarwinLogState(pVM, pVCpu);
1811	#endif
1812
1813	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateExport, x);
1814
1815	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL;
1816	if (!fWhat)
1817	return VINF_SUCCESS;
1818
1819	pVCpu->nem.s.fCtxChanged \|= nemR3DarwinCpumExtrnToHmChanged(pVCpu->cpum.GstCtx.fExtrn);
1820
1821	int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
1822	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1823
1824	rc = nemR3DarwinExportGuestGprs(pVCpu);
1825	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1826
1827	rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
1828	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1829
1830	VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
1831	if (rcStrict == VINF_SUCCESS)
1832	{ /* likely */ }
1833	else
1834	{
1835	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
1836	return VBOXSTRICTRC_VAL(rcStrict);
1837	}
1838
1839	rc = nemR3DarwinExportDebugState(pVCpu, pVmxTransient);
1840	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1841
1842	vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
1843	vmxHCExportGuestRip(pVCpu);
1844	//vmxHCExportGuestRsp(pVCpu);
1845	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
1846
1847	rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
1848	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1849
1850	if (fWhat & CPUMCTX_EXTRN_XCRx)
1851	{
1852	WRITE_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
1853	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_XCRx);
1854	}
1855
1856	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1857	{
1858	Assert(pVCpu->nem.s.fCtxChanged & HM_CHANGED_GUEST_APIC_TPR);
1859	vmxHCExportGuestApicTpr(pVCpu, pVmxTransient);
1860
1861	rc = APICGetTpr(pVCpu, &pVmxTransient->u8GuestTpr, NULL /pfPending/, NULL /pu8PendingIntr/);
1862	AssertRC(rc);
1863
1864	WRITE_GREG(HV_X86_TPR, pVmxTransient->u8GuestTpr);
1865	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1866	}
1867
1868	/* Debug registers. */
1869	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1870	{
1871	WRITE_GREG(HV_X86_DR0, CPUMGetHyperDR0(pVCpu));
1872	WRITE_GREG(HV_X86_DR1, CPUMGetHyperDR1(pVCpu));
1873	WRITE_GREG(HV_X86_DR2, CPUMGetHyperDR2(pVCpu));
1874	WRITE_GREG(HV_X86_DR3, CPUMGetHyperDR3(pVCpu));
1875	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR0_DR3);
1876	}
1877	if (fWhat & CPUMCTX_EXTRN_DR6)
1878	{
1879	WRITE_GREG(HV_X86_DR6, CPUMGetHyperDR6(pVCpu));
1880	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR6);
1881	}
1882	if (fWhat & CPUMCTX_EXTRN_DR7)
1883	{
1884	WRITE_GREG(HV_X86_DR7, CPUMGetHyperDR7(pVCpu));
1885	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR7);
1886	}
1887
1888	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1889	{
1890	hv_return_t hrc = hv_vcpu_write_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1891	if (hrc == HV_SUCCESS)
1892	{ /* likely */ }
1893	else
1894	return nemR3DarwinHvSts2Rc(hrc);
1895
1896	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~(HM_CHANGED_GUEST_X87 \| HM_CHANGED_GUEST_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE));
1897	}
1898
1899	/* MSRs */
1900	if (fWhat & CPUMCTX_EXTRN_EFER)
1901	{
1902	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, pVCpu->cpum.GstCtx.msrEFER);
1903	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
1904	}
1905	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1906	{
1907	WRITE_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1908	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_KERNEL_GS_BASE);
1909	}
1910	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1911	{
1912	WRITE_MSR(MSR_IA32_SYSENTER_CS, pVCpu->cpum.GstCtx.SysEnter.cs);
1913	WRITE_MSR(MSR_IA32_SYSENTER_EIP, pVCpu->cpum.GstCtx.SysEnter.eip);
1914	WRITE_MSR(MSR_IA32_SYSENTER_ESP, pVCpu->cpum.GstCtx.SysEnter.esp);
1915	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
1916	}
1917	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1918	{
1919	WRITE_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1920	WRITE_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1921	WRITE_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1922	WRITE_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1923	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSCALL_MSRS);
1924	}
1925	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1926	{
1927	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1928
1929	WRITE_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1930	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_TSC_AUX);
1931	}
1932	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1933	{
1934	/* Last Branch Record. */
1935	if (pVM->nem.s.fLbr)
1936	{
1937	PVMXVMCSINFOSHARED const pVmcsInfoShared = &pVCpu->nem.s.vmx.VmcsInfo;
1938	uint32_t const idFromIpMsrStart = pVM->nem.s.idLbrFromIpMsrFirst;
1939	uint32_t const idToIpMsrStart = pVM->nem.s.idLbrToIpMsrFirst;
1940	uint32_t const idInfoMsrStart = pVM->nem.s.idLbrInfoMsrFirst;
1941	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
1942	Assert(cLbrStack <= 32);
1943	for (uint32_t i = 0; i < cLbrStack; i++)
1944	{
1945	WRITE_MSR(idFromIpMsrStart + i, pVmcsInfoShared->au64LbrFromIpMsr[i]);
1946
1947	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
1948	if (idToIpMsrStart != 0)
1949	WRITE_MSR(idToIpMsrStart + i, pVmcsInfoShared->au64LbrToIpMsr[i]);
1950	if (idInfoMsrStart != 0)
1951	WRITE_MSR(idInfoMsrStart + i, pVmcsInfoShared->au64LbrInfoMsr[i]);
1952	}
1953
1954	WRITE_MSR(pVM->nem.s.idLbrTosMsr, pVmcsInfoShared->u64LbrTosMsr);
1955	if (pVM->nem.s.idLerFromIpMsr)
1956	WRITE_MSR(pVM->nem.s.idLerFromIpMsr, pVmcsInfoShared->u64LerFromIpMsr);
1957	if (pVM->nem.s.idLerToIpMsr)
1958	WRITE_MSR(pVM->nem.s.idLerToIpMsr, pVmcsInfoShared->u64LerToIpMsr);
1959	}
1960
1961	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
1962	}
1963
1964	hv_vcpu_invalidate_tlb(pVCpu->nem.s.hVCpuId);
1965	hv_vcpu_flush(pVCpu->nem.s.hVCpuId);
1966
1967	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_NEM;
1968
1969	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
1970	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~( HM_CHANGED_GUEST_HWVIRT
1971	\| HM_CHANGED_VMX_GUEST_AUTO_MSRS
1972	\| HM_CHANGED_VMX_GUEST_LAZY_MSRS
1973	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
1974
1975	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateExport, x);
1976	return VINF_SUCCESS;
1977	#undef WRITE_GREG
1978	#undef WRITE_VMCS_FIELD
1979	}
1980
1981
1982	/**
1983	* Common worker for both nemR3DarwinHandleExit() and nemR3DarwinHandleExitDebug().
1984	*
1985	* @returns VBox strict status code.
1986	* @param pVM The cross context VM structure.
1987	* @param pVCpu The cross context virtual CPU structure of the
1988	* calling EMT.
1989	* @param pVmxTransient The transient VMX structure.
1990	*/
1991	DECLINLINE(int) nemR3DarwinHandleExitCommon(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1992	{
1993	uint32_t uExitReason;
1994	int rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
1995	AssertRC(rc);
1996	pVmxTransient->fVmcsFieldsRead = 0;
1997	pVmxTransient->fIsNestedGuest = false;
1998	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
1999	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
2000
2001	if (RT_UNLIKELY(pVmxTransient->fVMEntryFailed))
2002	AssertLogRelMsgFailedReturn(("Running guest failed for CPU #%u: %#x %u\n",
2003	pVCpu->idCpu, pVmxTransient->uExitReason, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
2004	VERR_NEM_IPE_0);
2005
2006	/** @todo Only copy the state on demand (the R0 VT-x code saves some stuff unconditionally and the VMX template assumes that
2007	* when handling exits). */
2008	/*
2009	* Note! What is being fetched here must match the default value for the
2010	* a_fDonePostExit parameter of vmxHCImportGuestState exactly!
2011	*/
2012	rc = nemR3DarwinCopyStateFromHv(pVM, pVCpu, CPUMCTX_EXTRN_ALL);
2013	AssertRCReturn(rc, rc);
2014
2015	STAM_COUNTER_INC(&pVCpu->nem.s.pVmxStats->aStatExitReason[pVmxTransient->uExitReason & MASK_EXITREASON_STAT]);
2016	STAM_REL_COUNTER_INC(&pVCpu->nem.s.pVmxStats->StatExitAll);
2017	return VINF_SUCCESS;
2018	}
2019
2020
2021	/**
2022	* Handles an exit from hv_vcpu_run().
2023	*
2024	* @returns VBox strict status code.
2025	* @param pVM The cross context VM structure.
2026	* @param pVCpu The cross context virtual CPU structure of the
2027	* calling EMT.
2028	* @param pVmxTransient The transient VMX structure.
2029	*/
2030	static VBOXSTRICTRC nemR3DarwinHandleExit(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
2031	{
2032	int rc = nemR3DarwinHandleExitCommon(pVM, pVCpu, pVmxTransient);
2033	AssertRCReturn(rc, rc);
2034
2035	#ifndef HMVMX_USE_FUNCTION_TABLE
2036	return vmxHCHandleExit(pVCpu, pVmxTransient);
2037	#else
2038	return g_aVMExitHandlers[pVmxTransient->uExitReason].pfn(pVCpu, pVmxTransient);
2039	#endif
2040	}
2041
2042
2043	/**
2044	* Handles an exit from hv_vcpu_run() - debug runloop variant.
2045	*
2046	* @returns VBox strict status code.
2047	* @param pVM The cross context VM structure.
2048	* @param pVCpu The cross context virtual CPU structure of the
2049	* calling EMT.
2050	* @param pVmxTransient The transient VMX structure.
2051	* @param pDbgState The debug state structure.
2052	*/
2053	static VBOXSTRICTRC nemR3DarwinHandleExitDebug(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
2054	{
2055	int rc = nemR3DarwinHandleExitCommon(pVM, pVCpu, pVmxTransient);
2056	AssertRCReturn(rc, rc);
2057
2058	return vmxHCRunDebugHandleExit(pVCpu, pVmxTransient, pDbgState);
2059	}
2060
2061
2062	/**
2063	* Worker for nemR3NativeInit that loads the Hypervisor.framework shared library.
2064	*
2065	* @returns VBox status code.
2066	* @param fForced Whether the HMForced flag is set and we should
2067	* fail if we cannot initialize.
2068	* @param pErrInfo Where to always return error info.
2069	*/
2070	static int nemR3DarwinLoadHv(bool fForced, PRTERRINFO pErrInfo)
2071	{
2072	RTLDRMOD hMod = NIL_RTLDRMOD;
2073	static const char *s_pszHvPath = "/System/Library/Frameworks/Hypervisor.framework/Hypervisor";
2074
2075	int rc = RTLdrLoadEx(s_pszHvPath, &hMod, RTLDRLOAD_FLAGS_NO_UNLOAD \| RTLDRLOAD_FLAGS_NO_SUFFIX, pErrInfo);
2076	if (RT_SUCCESS(rc))
2077	{
2078	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
2079	{
2080	int rc2 = RTLdrGetSymbol(hMod, g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
2081	if (RT_SUCCESS(rc2))
2082	{
2083	if (g_aImports[i].fOptional)
2084	LogRel(("NEM: info: Found optional import Hypervisor!%s.\n",
2085	g_aImports[i].pszName));
2086	}
2087	else
2088	{
2089	*g_aImports[i].ppfn = NULL;
2090
2091	LogRel(("NEM: %s: Failed to import Hypervisor!%s: %Rrc\n",
2092	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
2093	g_aImports[i].pszName, rc2));
2094	if (!g_aImports[i].fOptional)
2095	{
2096	if (RTErrInfoIsSet(pErrInfo))
2097	RTErrInfoAddF(pErrInfo, rc2, ", Hypervisor!%s", g_aImports[i].pszName);
2098	else
2099	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: Hypervisor!%s", g_aImports[i].pszName);
2100	Assert(RT_FAILURE(rc));
2101	}
2102	}
2103	}
2104	if (RT_SUCCESS(rc))
2105	{
2106	Assert(!RTErrInfoIsSet(pErrInfo));
2107	}
2108
2109	RTLdrClose(hMod);
2110	}
2111	else
2112	{
2113	RTErrInfoAddF(pErrInfo, rc, "Failed to load Hypervisor.framwork: %s: %Rrc", s_pszHvPath, rc);
2114	rc = VERR_NEM_INIT_FAILED;
2115	}
2116
2117	return rc;
2118	}
2119
2120
2121	/**
2122	* Read and initialize the global capabilities supported by this CPU.
2123	*
2124	* @returns VBox status code.
2125	*/
2126	static int nemR3DarwinCapsInit(void)
2127	{
2128	RT_ZERO(g_HmMsrs);
2129
2130	hv_return_t hrc = hv_vmx_read_capability(HV_VMX_CAP_PINBASED, &g_HmMsrs.u.vmx.PinCtls.u);
2131	if (hrc == HV_SUCCESS)
2132	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED, &g_HmMsrs.u.vmx.ProcCtls.u);
2133	if (hrc == HV_SUCCESS)
2134	hrc = hv_vmx_read_capability(HV_VMX_CAP_ENTRY, &g_HmMsrs.u.vmx.EntryCtls.u);
2135	if (hrc == HV_SUCCESS)
2136	hrc = hv_vmx_read_capability(HV_VMX_CAP_EXIT, &g_HmMsrs.u.vmx.ExitCtls.u);
2137	if (hrc == HV_SUCCESS)
2138	{
2139	hrc = hv_vmx_read_capability(HV_VMX_CAP_BASIC, &g_HmMsrs.u.vmx.u64Basic);
2140	if (hrc == HV_SUCCESS)
2141	{
2142	if (hrc == HV_SUCCESS)
2143	hrc = hv_vmx_read_capability(HV_VMX_CAP_MISC, &g_HmMsrs.u.vmx.u64Misc);
2144	if (hrc == HV_SUCCESS)
2145	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED0, &g_HmMsrs.u.vmx.u64Cr0Fixed0);
2146	if (hrc == HV_SUCCESS)
2147	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED1, &g_HmMsrs.u.vmx.u64Cr0Fixed1);
2148	if (hrc == HV_SUCCESS)
2149	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED0, &g_HmMsrs.u.vmx.u64Cr4Fixed0);
2150	if (hrc == HV_SUCCESS)
2151	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED1, &g_HmMsrs.u.vmx.u64Cr4Fixed1);
2152	if (hrc == HV_SUCCESS)
2153	hrc = hv_vmx_read_capability(HV_VMX_CAP_VMCS_ENUM, &g_HmMsrs.u.vmx.u64VmcsEnum);
2154	if ( hrc == HV_SUCCESS
2155	&& RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_TRUE_CTLS))
2156	{
2157	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PINBASED, &g_HmMsrs.u.vmx.TruePinCtls.u);
2158	if (hrc == HV_SUCCESS)
2159	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PROCBASED, &g_HmMsrs.u.vmx.TrueProcCtls.u);
2160	if (hrc == HV_SUCCESS)
2161	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_ENTRY, &g_HmMsrs.u.vmx.TrueEntryCtls.u);
2162	if (hrc == HV_SUCCESS)
2163	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_EXIT, &g_HmMsrs.u.vmx.TrueExitCtls.u);
2164	}
2165	}
2166	else
2167	{
2168	/* Likely running on anything < 11.0 (BigSur) so provide some sensible defaults. */
2169	g_HmMsrs.u.vmx.u64Cr0Fixed0 = 0x80000021;
2170	g_HmMsrs.u.vmx.u64Cr0Fixed1 = 0xffffffff;
2171	g_HmMsrs.u.vmx.u64Cr4Fixed0 = 0x2000;
2172	g_HmMsrs.u.vmx.u64Cr4Fixed1 = 0x1767ff;
2173	hrc = HV_SUCCESS;
2174	}
2175	}
2176
2177	if ( hrc == HV_SUCCESS
2178	&& g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2179	{
2180	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED2, &g_HmMsrs.u.vmx.ProcCtls2.u);
2181
2182	if ( hrc == HV_SUCCESS
2183	&& g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & (VMX_PROC_CTLS2_EPT \| VMX_PROC_CTLS2_VPID))
2184	{
2185	hrc = hv_vmx_read_capability(HV_VMX_CAP_EPT_VPID_CAP, &g_HmMsrs.u.vmx.u64EptVpidCaps);
2186	if (hrc != HV_SUCCESS)
2187	hrc = HV_SUCCESS; /* Probably just outdated OS. */
2188	}
2189
2190	g_HmMsrs.u.vmx.u64VmFunc = 0; /* No way to read that on macOS. */
2191	}
2192
2193	if (hrc == HV_SUCCESS)
2194	{
2195	/*
2196	* Check for EFER swapping support.
2197	*/
2198	g_fHmVmxSupportsVmcsEfer = true; //(g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2199	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2200	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
2201	}
2202
2203	/*
2204	* Get MSR_IA32_ARCH_CAPABILITIES and expand it into the host feature structure.
2205	* This is only available with 11.0+ (BigSur) as the required API is only available there,
2206	* we could in theory initialize this when creating the EMTs using hv_vcpu_read_msr() but
2207	* the required vCPU handle is created after CPUM was initialized which is too late.
2208	* Given that the majority of users is on 11.0 and later we don't care for now.
2209	*/
2210	if ( hrc == HV_SUCCESS
2211	&& hv_vmx_get_msr_info)
2212	{
2213	g_CpumHostFeatures.s.fArchRdclNo = 0;
2214	g_CpumHostFeatures.s.fArchIbrsAll = 0;
2215	g_CpumHostFeatures.s.fArchRsbOverride = 0;
2216	g_CpumHostFeatures.s.fArchVmmNeedNotFlushL1d = 0;
2217	g_CpumHostFeatures.s.fArchMdsNo = 0;
2218	uint32_t const cStdRange = ASMCpuId_EAX(0);
2219	if ( RTX86IsValidStdRange(cStdRange)
2220	&& cStdRange >= 7)
2221	{
2222	uint32_t const fStdFeaturesEdx = ASMCpuId_EDX(1);
2223	uint32_t fStdExtFeaturesEdx;
2224	ASMCpuIdExSlow(7, 0, 0, 0, NULL, NULL, NULL, &fStdExtFeaturesEdx);
2225	if ( (fStdExtFeaturesEdx & X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP)
2226	&& (fStdFeaturesEdx & X86_CPUID_FEATURE_EDX_MSR))
2227	{
2228	uint64_t fArchVal;
2229	hrc = hv_vmx_get_msr_info(HV_VMX_INFO_MSR_IA32_ARCH_CAPABILITIES, &fArchVal);
2230	if (hrc == HV_SUCCESS)
2231	{
2232	g_CpumHostFeatures.s.fArchRdclNo = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RDCL_NO);
2233	g_CpumHostFeatures.s.fArchIbrsAll = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_IBRS_ALL);
2234	g_CpumHostFeatures.s.fArchRsbOverride = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RSBO);
2235	g_CpumHostFeatures.s.fArchVmmNeedNotFlushL1d = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_VMM_NEED_NOT_FLUSH_L1D);
2236	g_CpumHostFeatures.s.fArchMdsNo = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_MDS_NO);
2237	}
2238	}
2239	else
2240	g_CpumHostFeatures.s.fArchCap = 0;
2241	}
2242	}
2243
2244	return nemR3DarwinHvSts2Rc(hrc);
2245	}
2246
2247
2248	/**
2249	* Sets up the LBR MSR ranges based on the host CPU.
2250	*
2251	* @returns VBox status code.
2252	* @param pVM The cross context VM structure.
2253	*
2254	* @sa hmR0VmxSetupLbrMsrRange
2255	*/
2256	static int nemR3DarwinSetupLbrMsrRange(PVMCC pVM)
2257	{
2258	Assert(pVM->nem.s.fLbr);
2259	uint32_t idLbrFromIpMsrFirst;
2260	uint32_t idLbrFromIpMsrLast;
2261	uint32_t idLbrToIpMsrFirst;
2262	uint32_t idLbrToIpMsrLast;
2263	uint32_t idLbrInfoMsrFirst;
2264	uint32_t idLbrInfoMsrLast;
2265	uint32_t idLbrTosMsr;
2266	uint32_t idLbrSelectMsr;
2267	uint32_t idLerFromIpMsr;
2268	uint32_t idLerToIpMsr;
2269
2270	/*
2271	* Determine the LBR MSRs supported for this host CPU family and model.
2272	*
2273	* See Intel spec. 17.4.8 "LBR Stack".
2274	* See Intel "Model-Specific Registers" spec.
2275	*/
2276	uint32_t const uFamilyModel = (g_CpumHostFeatures.s.uFamily << 8)
2277	\| g_CpumHostFeatures.s.uModel;
2278	switch (uFamilyModel)
2279	{
2280	case 0x0f01: case 0x0f02:
2281	idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
2282	idLbrFromIpMsrLast = MSR_P4_LASTBRANCH_3;
2283	idLbrToIpMsrFirst = 0x0;
2284	idLbrToIpMsrLast = 0x0;
2285	idLbrInfoMsrFirst = 0x0;
2286	idLbrInfoMsrLast = 0x0;
2287	idLbrTosMsr = MSR_P4_LASTBRANCH_TOS;
2288	idLbrSelectMsr = 0x0;
2289	idLerFromIpMsr = 0x0;
2290	idLerToIpMsr = 0x0;
2291	break;
2292
2293	case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
2294	case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
2295	case 0x066a: case 0x066c: case 0x067d: case 0x067e:
2296	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2297	idLbrFromIpMsrLast = MSR_LASTBRANCH_31_FROM_IP;
2298	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2299	idLbrToIpMsrLast = MSR_LASTBRANCH_31_TO_IP;
2300	idLbrInfoMsrFirst = MSR_LASTBRANCH_0_INFO;
2301	idLbrInfoMsrLast = MSR_LASTBRANCH_31_INFO;
2302	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2303	idLbrSelectMsr = MSR_LASTBRANCH_SELECT;
2304	idLerFromIpMsr = MSR_LER_FROM_IP;
2305	idLerToIpMsr = MSR_LER_TO_IP;
2306	break;
2307
2308	case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
2309	case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
2310	case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
2311	case 0x062e: case 0x0625: case 0x062c: case 0x062f:
2312	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2313	idLbrFromIpMsrLast = MSR_LASTBRANCH_15_FROM_IP;
2314	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2315	idLbrToIpMsrLast = MSR_LASTBRANCH_15_TO_IP;
2316	idLbrInfoMsrFirst = MSR_LASTBRANCH_0_INFO;
2317	idLbrInfoMsrLast = MSR_LASTBRANCH_15_INFO;
2318	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2319	idLbrSelectMsr = MSR_LASTBRANCH_SELECT;
2320	idLerFromIpMsr = MSR_LER_FROM_IP;
2321	idLerToIpMsr = MSR_LER_TO_IP;
2322	break;
2323
2324	case 0x0617: case 0x061d: case 0x060f:
2325	idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
2326	idLbrFromIpMsrLast = MSR_CORE2_LASTBRANCH_3_FROM_IP;
2327	idLbrToIpMsrFirst = MSR_CORE2_LASTBRANCH_0_TO_IP;
2328	idLbrToIpMsrLast = MSR_CORE2_LASTBRANCH_3_TO_IP;
2329	idLbrInfoMsrFirst = 0x0;
2330	idLbrInfoMsrLast = 0x0;
2331	idLbrTosMsr = MSR_CORE2_LASTBRANCH_TOS;
2332	idLbrSelectMsr = 0x0;
2333	idLerFromIpMsr = 0x0;
2334	idLerToIpMsr = 0x0;
2335	break;
2336
2337	/* Atom and related microarchitectures we don't care about:
2338	case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
2339	case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
2340	case 0x0636: */
2341	/* All other CPUs: */
2342	default:
2343	{
2344	LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
2345	VMCC_GET_CPU_0(pVM)->nem.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
2346	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2347	}
2348	}
2349
2350	/*
2351	* Validate.
2352	*/
2353	uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
2354	PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
2355	AssertCompile( RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrFromIpMsr)
2356	== RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrToIpMsr));
2357	AssertCompile( RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrFromIpMsr)
2358	== RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrInfoMsr));
2359	if (cLbrStack > RT_ELEMENTS(pVCpu0->nem.s.vmx.VmcsInfo.au64LbrFromIpMsr))
2360	{
2361	LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
2362	VMCC_GET_CPU_0(pVM)->nem.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
2363	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2364	}
2365	NOREF(pVCpu0);
2366
2367	/*
2368	* Update the LBR info. to the VM struct. for use later.
2369	*/
2370	pVM->nem.s.idLbrTosMsr = idLbrTosMsr;
2371	pVM->nem.s.idLbrSelectMsr = idLbrSelectMsr;
2372
2373	pVM->nem.s.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
2374	pVM->nem.s.idLbrFromIpMsrLast = idLbrFromIpMsrLast;
2375
2376	pVM->nem.s.idLbrToIpMsrFirst = idLbrToIpMsrFirst;
2377	pVM->nem.s.idLbrToIpMsrLast = idLbrToIpMsrLast;
2378
2379	pVM->nem.s.idLbrInfoMsrFirst = idLbrInfoMsrFirst;
2380	pVM->nem.s.idLbrInfoMsrLast = idLbrInfoMsrLast;
2381
2382	pVM->nem.s.idLerFromIpMsr = idLerFromIpMsr;
2383	pVM->nem.s.idLerToIpMsr = idLerToIpMsr;
2384	return VINF_SUCCESS;
2385	}
2386
2387
2388	/**
2389	* Sets up pin-based VM-execution controls in the VMCS.
2390	*
2391	* @returns VBox status code.
2392	* @param pVCpu The cross context virtual CPU structure.
2393	* @param pVmcsInfo The VMCS info. object.
2394	*/
2395	static int nemR3DarwinVmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2396	{
2397	//PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2398	uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
2399	uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2400
2401	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2402	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2403
2404	#if 0 /** @todo Use preemption timer */
2405	/* Enable the VMX-preemption timer. */
2406	if (pVM->hmr0.s.vmx.fUsePreemptTimer)
2407	{
2408	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2409	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2410	}
2411
2412	/* Enable posted-interrupt processing. */
2413	if (pVM->hm.s.fPostedIntrs)
2414	{
2415	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2416	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2417	fVal \|= VMX_PIN_CTLS_POSTED_INT;
2418	}
2419	#endif
2420
2421	if ((fVal & fZap) != fVal)
2422	{
2423	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2424	g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
2425	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2426	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2427	}
2428
2429	/* Commit it to the VMCS and update our cache. */
2430	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2431	AssertRC(rc);
2432	pVmcsInfo->u32PinCtls = fVal;
2433
2434	return VINF_SUCCESS;
2435	}
2436
2437
2438	/**
2439	* Sets up secondary processor-based VM-execution controls in the VMCS.
2440	*
2441	* @returns VBox status code.
2442	* @param pVCpu The cross context virtual CPU structure.
2443	* @param pVmcsInfo The VMCS info. object.
2444	*/
2445	static int nemR3DarwinVmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2446	{
2447	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2448	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2449	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2450
2451	/* WBINVD causes a VM-exit. */
2452	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2453	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2454
2455	/* Enable the INVPCID instruction if we expose it to the guest and is supported
2456	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
2457	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
2458	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
2459	fVal \|= VMX_PROC_CTLS2_INVPCID;
2460
2461	#if 0 /** @todo */
2462	/* Enable VPID. */
2463	if (pVM->hmr0.s.vmx.fVpid)
2464	fVal \|= VMX_PROC_CTLS2_VPID;
2465
2466	if (pVM->hm.s.fVirtApicRegs)
2467	{
2468	/* Enable APIC-register virtualization. */
2469	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2470	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2471
2472	/* Enable virtual-interrupt delivery. */
2473	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2474	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2475	}
2476
2477	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
2478	where the TPR shadow resides. */
2479	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2480	* done dynamically. */
2481	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2482	{
2483	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2484	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
2485	}
2486	#endif
2487
2488	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
2489	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
2490	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
2491	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
2492	fVal \|= VMX_PROC_CTLS2_RDTSCP;
2493
2494	/* Enable Pause-Loop exiting. */
2495	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
2496	&& pVM->nem.s.cPleGapTicks
2497	&& pVM->nem.s.cPleWindowTicks)
2498	{
2499	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2500
2501	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PLE_GAP, pVM->nem.s.cPleGapTicks); AssertRC(rc);
2502	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PLE_WINDOW, pVM->nem.s.cPleWindowTicks); AssertRC(rc);
2503	}
2504
2505	if ((fVal & fZap) != fVal)
2506	{
2507	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2508	g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
2509	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2510	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2511	}
2512
2513	/* Commit it to the VMCS and update our cache. */
2514	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2515	AssertRC(rc);
2516	pVmcsInfo->u32ProcCtls2 = fVal;
2517
2518	return VINF_SUCCESS;
2519	}
2520
2521
2522	/**
2523	* Enables native access for the given MSR.
2524	*
2525	* @returns VBox status code.
2526	* @param pVCpu The cross context virtual CPU structure.
2527	* @param idMsr The MSR to enable native access for.
2528	*/
2529	static int nemR3DarwinMsrSetNative(PVMCPUCC pVCpu, uint32_t idMsr)
2530	{
2531	hv_return_t hrc = hv_vcpu_enable_native_msr(pVCpu->nem.s.hVCpuId, idMsr, true /enable/);
2532	if (hrc == HV_SUCCESS)
2533	return VINF_SUCCESS;
2534
2535	return nemR3DarwinHvSts2Rc(hrc);
2536	}
2537
2538
2539	/**
2540	* Sets the MSR to managed for the given vCPU allowing the guest to access it.
2541	*
2542	* @returns VBox status code.
2543	* @param pVCpu The cross context virtual CPU structure.
2544	* @param idMsr The MSR to enable managed access for.
2545	* @param fMsrPerm The MSR permissions flags.
2546	*/
2547	static int nemR3DarwinMsrSetManaged(PVMCPUCC pVCpu, uint32_t idMsr, hv_msr_flags_t fMsrPerm)
2548	{
2549	Assert(hv_vcpu_enable_managed_msr);
2550
2551	hv_return_t hrc = hv_vcpu_enable_managed_msr(pVCpu->nem.s.hVCpuId, idMsr, true /enable/);
2552	if (hrc == HV_SUCCESS)
2553	{
2554	hrc = hv_vcpu_set_msr_access(pVCpu->nem.s.hVCpuId, idMsr, fMsrPerm);
2555	if (hrc == HV_SUCCESS)
2556	return VINF_SUCCESS;
2557	}
2558
2559	return nemR3DarwinHvSts2Rc(hrc);
2560	}
2561
2562
2563	/**
2564	* Sets up the MSR permissions which don't change through the lifetime of the VM.
2565	*
2566	* @returns VBox status code.
2567	* @param pVCpu The cross context virtual CPU structure.
2568	* @param pVmcsInfo The VMCS info. object.
2569	*/
2570	static int nemR3DarwinSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2571	{
2572	RT_NOREF(pVmcsInfo);
2573
2574	/*
2575	* The guest can access the following MSRs (read, write) without causing
2576	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2577	*/
2578	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2579	int rc;
2580	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_CS); AssertRCReturn(rc, rc);
2581	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_ESP); AssertRCReturn(rc, rc);
2582	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_EIP); AssertRCReturn(rc, rc);
2583	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_GS_BASE); AssertRCReturn(rc, rc);
2584	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_FS_BASE); AssertRCReturn(rc, rc);
2585
2586	/*
2587	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2588	* associated with then. We never need to intercept access (writes need to be
2589	* executed without causing a VM-exit, reads will #GP fault anyway).
2590	*
2591	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2592	* read/write them. We swap the guest/host MSR value using the
2593	* auto-load/store MSR area.
2594	*/
2595	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2596	{
2597	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_PRED_CMD);
2598	AssertRCReturn(rc, rc);
2599	}
2600	#if 0 /* Doesn't work. */
2601	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2602	{
2603	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_FLUSH_CMD);
2604	AssertRCReturn(rc, rc);
2605	}
2606	#endif
2607	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2608	{
2609	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SPEC_CTRL);
2610	AssertRCReturn(rc, rc);
2611	}
2612
2613	/*
2614	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2615	* required for 64-bit guests.
2616	*/
2617	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_LSTAR); AssertRCReturn(rc, rc);
2618	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K6_STAR); AssertRCReturn(rc, rc);
2619	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_SF_MASK); AssertRCReturn(rc, rc);
2620	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_KERNEL_GS_BASE); AssertRCReturn(rc, rc);
2621
2622	/* Required for enabling the RDTSCP instruction. */
2623	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_TSC_AUX); AssertRCReturn(rc, rc);
2624
2625	/* Last Branch Record. */
2626	if (pVM->nem.s.fLbr)
2627	{
2628	uint32_t const idFromIpMsrStart = pVM->nem.s.idLbrFromIpMsrFirst;
2629	uint32_t const idToIpMsrStart = pVM->nem.s.idLbrToIpMsrFirst;
2630	uint32_t const idInfoMsrStart = pVM->nem.s.idLbrInfoMsrFirst;
2631	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
2632	Assert(cLbrStack <= 32);
2633	for (uint32_t i = 0; i < cLbrStack; i++)
2634	{
2635	rc = nemR3DarwinMsrSetManaged(pVCpu, idFromIpMsrStart + i, HV_MSR_READ \| HV_MSR_WRITE);
2636	AssertRCReturn(rc, rc);
2637
2638	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
2639	if (idToIpMsrStart != 0)
2640	{
2641	rc = nemR3DarwinMsrSetManaged(pVCpu, idToIpMsrStart + i, HV_MSR_READ \| HV_MSR_WRITE);
2642	AssertRCReturn(rc, rc);
2643	}
2644
2645	if (idInfoMsrStart != 0)
2646	{
2647	rc = nemR3DarwinMsrSetManaged(pVCpu, idInfoMsrStart + i, HV_MSR_READ \| HV_MSR_WRITE);
2648	AssertRCReturn(rc, rc);
2649	}
2650	}
2651
2652	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLbrTosMsr, HV_MSR_READ \| HV_MSR_WRITE);
2653	AssertRCReturn(rc, rc);
2654
2655	if (pVM->nem.s.idLerFromIpMsr)
2656	{
2657	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLerFromIpMsr, HV_MSR_READ \| HV_MSR_WRITE);
2658	AssertRCReturn(rc, rc);
2659	}
2660
2661	if (pVM->nem.s.idLerToIpMsr)
2662	{
2663	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLerToIpMsr, HV_MSR_READ \| HV_MSR_WRITE);
2664	AssertRCReturn(rc, rc);
2665	}
2666
2667	if (pVM->nem.s.idLbrSelectMsr)
2668	{
2669	rc = nemR3DarwinMsrSetManaged(pVCpu, pVM->nem.s.idLbrSelectMsr, HV_MSR_READ \| HV_MSR_WRITE);
2670	AssertRCReturn(rc, rc);
2671	}
2672	}
2673
2674	return VINF_SUCCESS;
2675	}
2676
2677
2678	/**
2679	* Sets up processor-based VM-execution controls in the VMCS.
2680	*
2681	* @returns VBox status code.
2682	* @param pVCpu The cross context virtual CPU structure.
2683	* @param pVmcsInfo The VMCS info. object.
2684	*/
2685	static int nemR3DarwinVmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2686	{
2687	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2688	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2689
2690	/** @todo The DRx handling is not quite correct breaking debugging inside the guest with gdb,
2691	* see @ticketref{21413} and @ticketref{21546}, so intercepting mov drX is disabled for now. See @bugref{10504}
2692	* as well. This will break the hypervisor debugger but only very few people use it and even less on macOS
2693	* using the NEM backend.
2694	*/
2695	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2696	// \| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2697	// \| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2698	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2699	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2700	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2701	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2702
2703	#ifdef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
2704	fVal \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
2705	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2706	#endif
2707
2708	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2709	if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2710	\|\| (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2711	{
2712	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2713	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2714	}
2715
2716	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2717	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2718	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2719
2720	if ((fVal & fZap) != fVal)
2721	{
2722	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2723	g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2724	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2725	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2726	}
2727
2728	/* Commit it to the VMCS and update our cache. */
2729	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2730	AssertRC(rc);
2731	pVmcsInfo->u32ProcCtls = fVal;
2732
2733	/* Set up MSR permissions that don't change through the lifetime of the VM. */
2734	rc = nemR3DarwinSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2735	AssertRCReturn(rc, rc);
2736
2737	/*
2738	* Set up secondary processor-based VM-execution controls
2739	* (we assume the CPU to always support it as we rely on unrestricted guest execution support).
2740	*/
2741	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
2742	return nemR3DarwinVmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2743	}
2744
2745
2746	/**
2747	* Sets up miscellaneous (everything other than Pin, Processor and secondary
2748	* Processor-based VM-execution) control fields in the VMCS.
2749	*
2750	* @returns VBox status code.
2751	* @param pVCpu The cross context virtual CPU structure.
2752	* @param pVmcsInfo The VMCS info. object.
2753	*/
2754	static int nemR3DarwinVmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2755	{
2756	int rc = VINF_SUCCESS;
2757	//rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo); TODO
2758	if (RT_SUCCESS(rc))
2759	{
2760	uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2761	uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2762
2763	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2764	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2765
2766	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2767	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2768
2769	if (pVCpu->CTX_SUFF(pVM)->nem.s.fLbr)
2770	{
2771	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2772	AssertRC(rc);
2773	}
2774	return VINF_SUCCESS;
2775	}
2776	else
2777	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2778	return rc;
2779	}
2780
2781
2782	/**
2783	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2784	*
2785	* We shall setup those exception intercepts that don't change during the
2786	* lifetime of the VM here. The rest are done dynamically while loading the
2787	* guest state.
2788	*
2789	* @param pVCpu The cross context virtual CPU structure.
2790	* @param pVmcsInfo The VMCS info. object.
2791	*/
2792	static void nemR3DarwinVmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2793	{
2794	/*
2795	* The following exceptions are always intercepted:
2796	*
2797	* #AC - To prevent the guest from hanging the CPU and for dealing with
2798	* split-lock detecting host configs.
2799	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2800	* recursive #DBs can cause a CPU hang.
2801	*/
2802	/** @todo The DRx handling is not quite correct breaking debugging inside the guest with gdb,
2803	* see @ticketref{21413} and @ticketref{21546}, so intercepting \#DB is disabled for now. See @bugref{10504}
2804	* as well. This will break the hypervisor debugger but only very few people use it and even less on macOS
2805	* using the NEM backend.
2806	*/
2807	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2808	/\| RT_BIT(X86_XCPT_DB)/;
2809
2810	/* Commit it to the VMCS. */
2811	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2812	AssertRC(rc);
2813
2814	/* Update our cache of the exception bitmap. */
2815	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2816	}
2817
2818
2819	/**
2820	* Initialize the VMCS information field for the given vCPU.
2821	*
2822	* @returns VBox status code.
2823	* @param pVCpu The cross context virtual CPU structure of the
2824	* calling EMT.
2825	*/
2826	static int nemR3DarwinInitVmcs(PVMCPU pVCpu)
2827	{
2828	int rc = nemR3DarwinVmxSetupVmcsPinCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2829	if (RT_SUCCESS(rc))
2830	{
2831	rc = nemR3DarwinVmxSetupVmcsProcCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2832	if (RT_SUCCESS(rc))
2833	{
2834	rc = nemR3DarwinVmxSetupVmcsMiscCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2835	if (RT_SUCCESS(rc))
2836	{
2837	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &pVCpu->nem.s.VmcsInfo.u32EntryCtls);
2838	if (RT_SUCCESS(rc))
2839	{
2840	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_EXIT, &pVCpu->nem.s.VmcsInfo.u32ExitCtls);
2841	if (RT_SUCCESS(rc))
2842	{
2843	nemR3DarwinVmxSetupVmcsXcptBitmap(pVCpu, &pVCpu->nem.s.VmcsInfo);
2844	return VINF_SUCCESS;
2845	}
2846	LogRelFunc(("Failed to read the exit controls. rc=%Rrc\n", rc));
2847	}
2848	else
2849	LogRelFunc(("Failed to read the entry controls. rc=%Rrc\n", rc));
2850	}
2851	else
2852	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
2853	}
2854	else
2855	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
2856	}
2857	else
2858	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
2859
2860	return rc;
2861	}
2862
2863
2864	/**
2865	* Registers statistics for the given vCPU.
2866	*
2867	* @returns VBox status code.
2868	* @param pVM The cross context VM structure.
2869	* @param idCpu The CPU ID.
2870	* @param pNemCpu The NEM CPU structure.
2871	*/
2872	static int nemR3DarwinStatisticsRegister(PVM pVM, VMCPUID idCpu, PNEMCPU pNemCpu)
2873	{
2874	#define NEM_REG_STAT(a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szNmFmt, a_szDesc) do { \
2875	int rc = STAMR3RegisterF(pVM, a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szDesc, a_szNmFmt, idCpu); \
2876	AssertRC(rc); \
2877	} while (0)
2878	#define NEM_REG_PROFILE(a_pVar, a_szNmFmt, a_szDesc) \
2879	NEM_REG_STAT(a_pVar, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, a_szNmFmt, a_szDesc)
2880	#define NEM_REG_COUNTER(a, b, desc) NEM_REG_STAT(a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, b, desc)
2881
2882	PVMXSTATISTICS const pVmxStats = pNemCpu->pVmxStats;
2883
2884	NEM_REG_COUNTER(&pVmxStats->StatExitCR0Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR0", "CR0 read.");
2885	NEM_REG_COUNTER(&pVmxStats->StatExitCR2Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR2", "CR2 read.");
2886	NEM_REG_COUNTER(&pVmxStats->StatExitCR3Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR3", "CR3 read.");
2887	NEM_REG_COUNTER(&pVmxStats->StatExitCR4Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR4", "CR4 read.");
2888	NEM_REG_COUNTER(&pVmxStats->StatExitCR8Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR8", "CR8 read.");
2889	NEM_REG_COUNTER(&pVmxStats->StatExitCR0Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR0", "CR0 write.");
2890	NEM_REG_COUNTER(&pVmxStats->StatExitCR2Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR2", "CR2 write.");
2891	NEM_REG_COUNTER(&pVmxStats->StatExitCR3Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR3", "CR3 write.");
2892	NEM_REG_COUNTER(&pVmxStats->StatExitCR4Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR4", "CR4 write.");
2893	NEM_REG_COUNTER(&pVmxStats->StatExitCR8Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR8", "CR8 write.");
2894
2895	NEM_REG_COUNTER(&pVmxStats->StatExitAll, "/NEM/CPU%u/Exit/All", "Total exits (including nested-guest exits).");
2896
2897	NEM_REG_COUNTER(&pVmxStats->StatImportGuestStateFallback, "/NEM/CPU%u/ImportGuestStateFallback", "Times vmxHCImportGuestState took the fallback code path.");
2898	NEM_REG_COUNTER(&pVmxStats->StatReadToTransientFallback, "/NEM/CPU%u/ReadToTransientFallback", "Times vmxHCReadToTransient took the fallback code path.");
2899
2900	#ifdef VBOX_WITH_STATISTICS
2901	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateImport, "/NEM/CPU%u/ImportGuestState", "Profiling of importing guest state from hardware after VM-exit.");
2902	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateExport, "/NEM/CPU%u/ExportGuestState", "Profiling of exporting guest state from hardware after VM-exit.");
2903
2904	for (int j = 0; j < MAX_EXITREASON_STAT; j++)
2905	{
2906	const char *pszExitName = HMGetVmxExitName(j);
2907	if (pszExitName)
2908	{
2909	int rc = STAMR3RegisterF(pVM, &pVmxStats->aStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
2910	STAMUNIT_OCCURENCES, pszExitName, "/NEM/CPU%u/Exit/Reason/%02x", idCpu, j);
2911	AssertRCReturn(rc, rc);
2912	}
2913	}
2914	#endif
2915
2916	return VINF_SUCCESS;
2917
2918	#undef NEM_REG_COUNTER
2919	#undef NEM_REG_PROFILE
2920	#undef NEM_REG_STAT
2921	}
2922
2923
2924	/**
2925	* Displays the HM Last-Branch-Record info. for the guest.
2926	*
2927	* @param pVM The cross context VM structure.
2928	* @param pHlp The info helper functions.
2929	* @param pszArgs Arguments, ignored.
2930	*/
2931	static DECLCALLBACK(void) nemR3DarwinInfoLbr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2932	{
2933	NOREF(pszArgs);
2934	PVMCPU pVCpu = VMMGetCpu(pVM);
2935	if (!pVCpu)
2936	pVCpu = pVM->apCpusR3[0];
2937
2938	Assert(pVM->nem.s.fLbr);
2939
2940	PCVMXVMCSINFOSHARED pVmcsInfoShared = &pVCpu->nem.s.vmx.VmcsInfo;
2941	uint32_t const cLbrStack = pVM->nem.s.idLbrFromIpMsrLast - pVM->nem.s.idLbrFromIpMsrFirst + 1;
2942
2943	/** @todo r=ramshankar: The index technically varies depending on the CPU, but
2944	* 0xf should cover everything we support thus far. Fix if necessary
2945	* later. */
2946	uint32_t const idxTopOfStack = pVmcsInfoShared->u64LbrTosMsr & 0xf;
2947	if (idxTopOfStack > cLbrStack)
2948	{
2949	pHlp->pfnPrintf(pHlp, "Top-of-stack LBR MSR seems corrupt (index=%u, msr=%#RX64) expected index < %u\n",
2950	idxTopOfStack, pVmcsInfoShared->u64LbrTosMsr, cLbrStack);
2951	return;
2952	}
2953
2954	/*
2955	* Dump the circular buffer of LBR records starting from the most recent record (contained in idxTopOfStack).
2956	*/
2957	pHlp->pfnPrintf(pHlp, "CPU[%u]: LBRs (most-recent first)\n", pVCpu->idCpu);
2958	if (pVM->nem.s.idLerFromIpMsr)
2959	pHlp->pfnPrintf(pHlp, "LER: From IP=%#016RX64 - To IP=%#016RX64\n",
2960	pVmcsInfoShared->u64LerFromIpMsr, pVmcsInfoShared->u64LerToIpMsr);
2961	uint32_t idxCurrent = idxTopOfStack;
2962	Assert(idxTopOfStack < cLbrStack);
2963	Assert(RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr) <= cLbrStack);
2964	Assert(RT_ELEMENTS(pVmcsInfoShared->au64LbrToIpMsr) <= cLbrStack);
2965	for (;;)
2966	{
2967	if (pVM->nem.s.idLbrToIpMsrFirst)
2968	pHlp->pfnPrintf(pHlp, " Branch (%2u): From IP=%#016RX64 - To IP=%#016RX64 (Info: %#016RX64)\n", idxCurrent,
2969	pVmcsInfoShared->au64LbrFromIpMsr[idxCurrent],
2970	pVmcsInfoShared->au64LbrToIpMsr[idxCurrent],
2971	pVmcsInfoShared->au64LbrInfoMsr[idxCurrent]);
2972	else
2973	pHlp->pfnPrintf(pHlp, " Branch (%2u): LBR=%#RX64\n", idxCurrent, pVmcsInfoShared->au64LbrFromIpMsr[idxCurrent]);
2974
2975	idxCurrent = (idxCurrent - 1) % cLbrStack;
2976	if (idxCurrent == idxTopOfStack)
2977	break;
2978	}
2979	}
2980
2981
2982	/**
2983	* Try initialize the native API.
2984	*
2985	* This may only do part of the job, more can be done in
2986	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
2987	*
2988	* @returns VBox status code.
2989	* @param pVM The cross context VM structure.
2990	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
2991	* the latter we'll fail if we cannot initialize.
2992	* @param fForced Whether the HMForced flag is set and we should
2993	* fail if we cannot initialize.
2994	*/
2995	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
2996	{
2997	AssertReturn(!pVM->nem.s.fCreatedVm, VERR_WRONG_ORDER);
2998
2999	/*
3000	* Some state init.
3001	*/
3002	PCFGMNODE pCfgNem = CFGMR3GetChild(CFGMR3GetRoot(pVM), "NEM/");
3003
3004	/** @cfgm{/NEM/VmxPleGap, uint32_t, 0}
3005	* The pause-filter exiting gap in TSC ticks. When the number of ticks between
3006	* two successive PAUSE instructions exceeds VmxPleGap, the CPU considers the
3007	* latest PAUSE instruction to be start of a new PAUSE loop.
3008	*/
3009	int rc = CFGMR3QueryU32Def(pCfgNem, "VmxPleGap", &pVM->nem.s.cPleGapTicks, 0);
3010	AssertRCReturn(rc, rc);
3011
3012	/** @cfgm{/NEM/VmxPleWindow, uint32_t, 0}
3013	* The pause-filter exiting window in TSC ticks. When the number of ticks
3014	* between the current PAUSE instruction and first PAUSE of a loop exceeds
3015	* VmxPleWindow, a VM-exit is triggered.
3016	*
3017	* Setting VmxPleGap and VmxPleGap to 0 disables pause-filter exiting.
3018	*/
3019	rc = CFGMR3QueryU32Def(pCfgNem, "VmxPleWindow", &pVM->nem.s.cPleWindowTicks, 0);
3020	AssertRCReturn(rc, rc);
3021
3022	/** @cfgm{/NEM/VmxLbr, bool, false}
3023	* Whether to enable LBR for the guest. This is disabled by default as it's only
3024	* useful while debugging and enabling it causes a noticeable performance hit. */
3025	rc = CFGMR3QueryBoolDef(pCfgNem, "VmxLbr", &pVM->nem.s.fLbr, false);
3026	AssertRCReturn(rc, rc);
3027
3028	/*
3029	* Error state.
3030	* The error message will be non-empty on failure and 'rc' will be set too.
3031	*/
3032	RTERRINFOSTATIC ErrInfo;
3033	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
3034	rc = nemR3DarwinLoadHv(fForced, pErrInfo);
3035	if (RT_SUCCESS(rc))
3036	{
3037	if ( !hv_vcpu_enable_managed_msr
3038	&& pVM->nem.s.fLbr)
3039	{
3040	LogRel(("NEM: LBR recording is disabled because the Hypervisor API misses hv_vcpu_enable_managed_msr/hv_vcpu_set_msr_access functionality\n"));
3041	pVM->nem.s.fLbr = false;
3042	}
3043
3044	/*
3045	* While hv_vcpu_run_until() is available starting with Catalina (10.15) it sometimes returns
3046	* an error there for no obvious reasons and there is no indication as to why this happens
3047	* and Apple doesn't document anything. Starting with BigSur (11.0) it appears to work correctly
3048	* so pretend that hv_vcpu_run_until() doesn't exist on Catalina which can be determined by checking
3049	* whether another method is available which was introduced with BigSur.
3050	*/
3051	if (!hv_vmx_get_msr_info) /* Not available means this runs on < 11.0 */
3052	hv_vcpu_run_until = NULL;
3053
3054	if (hv_vcpu_run_until)
3055	{
3056	struct mach_timebase_info TimeInfo;
3057
3058	if (mach_timebase_info(&TimeInfo) == KERN_SUCCESS)
3059	{
3060	pVM->nem.s.cMachTimePerNs = RT_MIN(1, (double)TimeInfo.denom / (double)TimeInfo.numer);
3061	LogRel(("NEM: cMachTimePerNs=%llu (TimeInfo.numer=%u TimeInfo.denom=%u)\n",
3062	pVM->nem.s.cMachTimePerNs, TimeInfo.numer, TimeInfo.denom));
3063	}
3064	else
3065	hv_vcpu_run_until = NULL; /* To avoid running forever (TM asserts when the guest runs for longer than 4 seconds). */
3066	}
3067
3068	hv_return_t hrc = hv_vm_create(HV_VM_DEFAULT);
3069	if (hrc == HV_SUCCESS)
3070	{
3071	if (hv_vm_space_create)
3072	{
3073	hrc = hv_vm_space_create(&pVM->nem.s.uVmAsid);
3074	if (hrc == HV_SUCCESS)
3075	{
3076	LogRel(("NEM: Successfully created ASID: %u\n", pVM->nem.s.uVmAsid));
3077	pVM->nem.s.fCreatedAsid = true;
3078	}
3079	else
3080	LogRel(("NEM: Failed to create ASID for VM (hrc=%#x), continuing...\n", pVM->nem.s.uVmAsid));
3081	}
3082	pVM->nem.s.fCreatedVm = true;
3083
3084	/* Register release statistics */
3085	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3086	{
3087	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
3088	PVMXSTATISTICS pVmxStats = (PVMXSTATISTICS)RTMemAllocZ(sizeof(*pVmxStats));
3089	if (RT_LIKELY(pVmxStats))
3090	{
3091	pNemCpu->pVmxStats = pVmxStats;
3092	rc = nemR3DarwinStatisticsRegister(pVM, idCpu, pNemCpu);
3093	AssertRC(rc);
3094	}
3095	else
3096	{
3097	rc = VERR_NO_MEMORY;
3098	break;
3099	}
3100	}
3101
3102	if (RT_SUCCESS(rc))
3103	{
3104	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
3105	Log(("NEM: Marked active!\n"));
3106	PGMR3EnableNemMode(pVM);
3107	}
3108	}
3109	else
3110	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
3111	"hv_vm_create() failed: %#x", hrc);
3112	}
3113
3114	/*
3115	* We only fail if in forced mode, otherwise just log the complaint and return.
3116	*/
3117	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
3118	if ( (fForced \|\| !fFallback)
3119	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
3120	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
3121
3122	if (pVM->nem.s.fLbr)
3123	{
3124	rc = DBGFR3InfoRegisterInternalEx(pVM, "lbr", "Dumps the NEM LBR info.", nemR3DarwinInfoLbr, DBGFINFO_FLAGS_ALL_EMTS);
3125	AssertRCReturn(rc, rc);
3126	}
3127
3128	if (RTErrInfoIsSet(pErrInfo))
3129	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
3130	return VINF_SUCCESS;
3131	}
3132
3133
3134	/**
3135	* Worker to create the vCPU handle on the EMT running it later on (as required by HV).
3136	*
3137	* @returns VBox status code
3138	* @param pVM The VM handle.
3139	* @param pVCpu The vCPU handle.
3140	* @param idCpu ID of the CPU to create.
3141	*/
3142	static DECLCALLBACK(int) nemR3DarwinNativeInitVCpuOnEmt(PVM pVM, PVMCPU pVCpu, VMCPUID idCpu)
3143	{
3144	hv_return_t hrc = hv_vcpu_create(&pVCpu->nem.s.hVCpuId, HV_VCPU_DEFAULT);
3145	if (hrc != HV_SUCCESS)
3146	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
3147	"Call to hv_vcpu_create failed on vCPU %u: %#x (%Rrc)", idCpu, hrc, nemR3DarwinHvSts2Rc(hrc));
3148
3149	if (idCpu == 0)
3150	{
3151	/* First call initializs the MSR structure holding the capabilities of the host CPU. */
3152	int rc = nemR3DarwinCapsInit();
3153	AssertRCReturn(rc, rc);
3154
3155	if (hv_vmx_vcpu_get_cap_write_vmcs)
3156	{
3157	/* Log the VMCS field write capabilities. */
3158	for (uint32_t i = 0; i < RT_ELEMENTS(g_aVmcsFieldsCap); i++)
3159	{
3160	uint64_t u64Allowed0 = 0;
3161	uint64_t u64Allowed1 = 0;
3162
3163	hrc = hv_vmx_vcpu_get_cap_write_vmcs(pVCpu->nem.s.hVCpuId, g_aVmcsFieldsCap[i].u32VmcsFieldId,
3164	&u64Allowed0, &u64Allowed1);
3165	if (hrc == HV_SUCCESS)
3166	{
3167	if (g_aVmcsFieldsCap[i].f64Bit)
3168	LogRel(("NEM: %s = (allowed_0=%#016RX64 allowed_1=%#016RX64)\n",
3169	g_aVmcsFieldsCap[i].pszVmcsField, u64Allowed0, u64Allowed1));
3170	else
3171	LogRel(("NEM: %s = (allowed_0=%#08RX32 allowed_1=%#08RX32)\n",
3172	g_aVmcsFieldsCap[i].pszVmcsField, (uint32_t)u64Allowed0, (uint32_t)u64Allowed1));
3173
3174	uint32_t cBits = g_aVmcsFieldsCap[i].f64Bit ? 64 : 32;
3175	for (uint32_t iBit = 0; iBit < cBits; iBit++)
3176	{
3177	bool fAllowed0 = RT_BOOL(u64Allowed0 & RT_BIT_64(iBit));
3178	bool fAllowed1 = RT_BOOL(u64Allowed1 & RT_BIT_64(iBit));
3179
3180	if (!fAllowed0 && !fAllowed1)
3181	LogRel(("NEM: Bit %02u = Must NOT be set\n", iBit));
3182	else if (!fAllowed0 && fAllowed1)
3183	LogRel(("NEM: Bit %02u = Can be set or not be set\n", iBit));
3184	else if (fAllowed0 && !fAllowed1)
3185	LogRel(("NEM: Bit %02u = UNDEFINED (AppleHV error)!\n", iBit));
3186	else if (fAllowed0 && fAllowed1)
3187	LogRel(("NEM: Bit %02u = MUST be set\n", iBit));
3188	else
3189	AssertFailed();
3190	}
3191	}
3192	else
3193	LogRel(("NEM: %s = failed to query (hrc=%d)\n", g_aVmcsFieldsCap[i].pszVmcsField, hrc));
3194	}
3195	}
3196	}
3197
3198	int rc = nemR3DarwinInitVmcs(pVCpu);
3199	AssertRCReturn(rc, rc);
3200
3201	if (pVM->nem.s.fCreatedAsid)
3202	{
3203	hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, pVM->nem.s.uVmAsid);
3204	AssertReturn(hrc == HV_SUCCESS, VERR_NEM_VM_CREATE_FAILED);
3205	}
3206
3207	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3208
3209	return VINF_SUCCESS;
3210	}
3211
3212
3213	/**
3214	* Worker to destroy the vCPU handle on the EMT running it later on (as required by HV).
3215	*
3216	* @returns VBox status code
3217	* @param pVCpu The vCPU handle.
3218	*/
3219	static DECLCALLBACK(int) nemR3DarwinNativeTermVCpuOnEmt(PVMCPU pVCpu)
3220	{
3221	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
3222	Assert(hrc == HV_SUCCESS);
3223
3224	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
3225	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
3226	return VINF_SUCCESS;
3227	}
3228
3229
3230	/**
3231	* Worker to setup the TPR shadowing feature if available on the CPU and the VM has an APIC enabled.
3232	*
3233	* @returns VBox status code
3234	* @param pVM The VM handle.
3235	* @param pVCpu The vCPU handle.
3236	*/
3237	static DECLCALLBACK(int) nemR3DarwinNativeInitTprShadowing(PVM pVM, PVMCPU pVCpu)
3238	{
3239	PVMXVMCSINFO pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
3240	uint32_t fVal = pVmcsInfo->u32ProcCtls;
3241
3242	/* Use TPR shadowing if supported by the CPU. */
3243	if ( PDMHasApic(pVM)
3244	&& (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
3245	{
3246	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3247	/* CR8 writes cause a VM-exit based on TPR threshold. */
3248	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3249	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3250	}
3251	else
3252	{
3253	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3254	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3255	}
3256
3257	/* Commit it to the VMCS and update our cache. */
3258	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3259	AssertRC(rc);
3260	pVmcsInfo->u32ProcCtls = fVal;
3261
3262	return VINF_SUCCESS;
3263	}
3264
3265
3266	/**
3267	* This is called after CPUMR3Init is done.
3268	*
3269	* @returns VBox status code.
3270	* @param pVM The VM handle..
3271	*/
3272	int nemR3NativeInitAfterCPUM(PVM pVM)
3273	{
3274	/*
3275	* Validate sanity.
3276	*/
3277	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
3278	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
3279
3280	if (pVM->nem.s.fLbr)
3281	{
3282	int rc = nemR3DarwinSetupLbrMsrRange(pVM);
3283	AssertRCReturn(rc, rc);
3284	}
3285
3286	/*
3287	* Setup the EMTs.
3288	*/
3289	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3290	{
3291	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
3292
3293	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitVCpuOnEmt, 3, pVM, pVCpu, idCpu);
3294	if (RT_FAILURE(rc))
3295	{
3296	/* Rollback. */
3297	while (idCpu--)
3298	VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeTermVCpuOnEmt, 1, pVCpu);
3299
3300	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to hv_vcpu_create failed: %Rrc", rc);
3301	}
3302	}
3303
3304	pVM->nem.s.fCreatedEmts = true;
3305	return VINF_SUCCESS;
3306	}
3307
3308
3309	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
3310	{
3311	if (enmWhat == VMINITCOMPLETED_RING3)
3312	{
3313	/* Now that PDM is initialized the APIC state is known in order to enable the TPR shadowing feature on all EMTs. */
3314	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3315	{
3316	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
3317
3318	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitTprShadowing, 2, pVM, pVCpu);
3319	if (RT_FAILURE(rc))
3320	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Setting up TPR shadowing failed: %Rrc", rc);
3321	}
3322	}
3323	return VINF_SUCCESS;
3324	}
3325
3326
3327	int nemR3NativeTerm(PVM pVM)
3328	{
3329	/*
3330	* Delete the VM.
3331	*/
3332
3333	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu--)
3334	{
3335	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
3336
3337	/*
3338	* Need to do this or hv_vm_space_destroy() fails later on (on 10.15 at least). Could've been documented in
3339	* API reference so I wouldn't have to decompile the kext to find this out but we are talking
3340	* about Apple here unfortunately, API documentation is not their strong suit...
3341	* Would have been of course even better to just automatically drop the address space reference when the vCPU
3342	* gets destroyed.
3343	*/
3344	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
3345	Assert(hrc == HV_SUCCESS);
3346
3347	/*
3348	* Apple's documentation states that the vCPU should be destroyed
3349	* on the thread running the vCPU but as all the other EMTs are gone
3350	* at this point, destroying the VM would hang.
3351	*
3352	* We seem to be at luck here though as destroying apparently works
3353	* from EMT(0) as well.
3354	*/
3355	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
3356	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
3357
3358	if (pVCpu->nem.s.pVmxStats)
3359	{
3360	RTMemFree(pVCpu->nem.s.pVmxStats);
3361	pVCpu->nem.s.pVmxStats = NULL;
3362	}
3363	}
3364
3365	pVM->nem.s.fCreatedEmts = false;
3366
3367	if (pVM->nem.s.fCreatedAsid)
3368	{
3369	hv_return_t hrc = hv_vm_space_destroy(pVM->nem.s.uVmAsid);
3370	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
3371	pVM->nem.s.fCreatedAsid = false;
3372	}
3373
3374	if (pVM->nem.s.fCreatedVm)
3375	{
3376	hv_return_t hrc = hv_vm_destroy();
3377	if (hrc != HV_SUCCESS)
3378	LogRel(("NEM: hv_vm_destroy() failed with %#x\n", hrc));
3379
3380	pVM->nem.s.fCreatedVm = false;
3381	}
3382	return VINF_SUCCESS;
3383	}
3384
3385
3386	/**
3387	* VM reset notification.
3388	*
3389	* @param pVM The cross context VM structure.
3390	*/
3391	void nemR3NativeReset(PVM pVM)
3392	{
3393	RT_NOREF(pVM);
3394	}
3395
3396
3397	/**
3398	* Reset CPU due to INIT IPI or hot (un)plugging.
3399	*
3400	* @param pVCpu The cross context virtual CPU structure of the CPU being
3401	* reset.
3402	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
3403	*/
3404	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
3405	{
3406	RT_NOREF(fInitIpi);
3407	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3408	}
3409
3410
3411	/**
3412	* Dumps the VMCS in response to a faild hv_vcpu_run{_until}() call.
3413	*
3414	* @param pVCpu The cross context virtual CPU structure.
3415	*/
3416	static void nemR3DarwinVmcsDump(PVMCPU pVCpu)
3417	{
3418	static const struct
3419	{
3420	uint32_t u32VmcsFieldId; /*< The VMCS field identifier. /
3421	const char pszVmcsField; /< The VMCS field name. /
3422	bool f64Bit;
3423	} s_aVmcsFieldsDump[] =
3424	{
3425	#define NEM_DARWIN_VMCSNW_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, true }
3426	#define NEM_DARWIN_VMCS64_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, true }
3427	#define NEM_DARWIN_VMCS32_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, false }
3428	#define NEM_DARWIN_VMCS16_FIELD_DUMP(a_u32VmcsFieldId) { (a_u32VmcsFieldId), #a_u32VmcsFieldId, false }
3429	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_VPID),
3430	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR),
3431	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_EPTP_INDEX),
3432	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_ES_SEL),
3433	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_CS_SEL),
3434	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_SS_SEL),
3435	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_DS_SEL),
3436	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_FS_SEL),
3437	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_GS_SEL),
3438	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_LDTR_SEL),
3439	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_TR_SEL),
3440	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_INTR_STATUS),
3441	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_GUEST_PML_INDEX),
3442	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_ES_SEL),
3443	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_CS_SEL),
3444	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_SS_SEL),
3445	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_DS_SEL),
3446	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_FS_SEL),
3447	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_GS_SEL),
3448	NEM_DARWIN_VMCS16_FIELD_DUMP(VMX_VMCS16_HOST_TR_SEL),
3449
3450	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL),
3451	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH),
3452	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL),
3453	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH),
3454	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_MSR_BITMAP_FULL),
3455	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_MSR_BITMAP_HIGH),
3456	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL),
3457	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH),
3458	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL),
3459	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH),
3460	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL),
3461	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH),
3462	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL),
3463	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH),
3464	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL),
3465	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH),
3466	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_OFFSET_FULL),
3467	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_OFFSET_HIGH),
3468	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL),
3469	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH),
3470	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL),
3471	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH),
3472	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL),
3473	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH),
3474	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL),
3475	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH),
3476	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_FULL),
3477	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_HIGH),
3478	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL),
3479	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH),
3480	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL),
3481	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH),
3482	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL),
3483	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH),
3484	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL),
3485	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH),
3486	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_LIST_FULL),
3487	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_EPTP_LIST_HIGH),
3488	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL),
3489	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH),
3490	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL),
3491	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH),
3492	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL),
3493	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH),
3494	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL),
3495	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH),
3496	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL),
3497	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH),
3498	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_SPPTP_FULL),
3499	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_SPPTP_HIGH),
3500	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL),
3501	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH),
3502	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_PROC_EXEC3_FULL),
3503	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_PROC_EXEC3_HIGH),
3504	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL),
3505	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH),
3506	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL),
3507	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH),
3508	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL),
3509	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH),
3510	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_DEBUGCTL_FULL),
3511	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_DEBUGCTL_HIGH),
3512	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PAT_FULL),
3513	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PAT_HIGH),
3514	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_EFER_FULL),
3515	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_EFER_HIGH),
3516	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL),
3517	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH),
3518	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE0_FULL),
3519	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE0_HIGH),
3520	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE1_FULL),
3521	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE1_HIGH),
3522	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE2_FULL),
3523	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE2_HIGH),
3524	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE3_FULL),
3525	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PDPTE3_HIGH),
3526	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_BNDCFGS_FULL),
3527	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_BNDCFGS_HIGH),
3528	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_RTIT_CTL_FULL),
3529	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_RTIT_CTL_HIGH),
3530	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PKRS_FULL),
3531	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_GUEST_PKRS_HIGH),
3532	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PAT_FULL),
3533	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PAT_HIGH),
3534	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_EFER_FULL),
3535	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_EFER_HIGH),
3536	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL),
3537	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH),
3538	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PKRS_FULL),
3539	NEM_DARWIN_VMCS64_FIELD_DUMP(VMX_VMCS64_HOST_PKRS_HIGH),
3540
3541	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PIN_EXEC),
3542	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PROC_EXEC),
3543	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXCEPTION_BITMAP),
3544	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK),
3545	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH),
3546	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_CR3_TARGET_COUNT),
3547	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXIT),
3548	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT),
3549	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT),
3550	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY),
3551	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT),
3552	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO),
3553	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE),
3554	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH),
3555	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_TPR_THRESHOLD),
3556	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PROC_EXEC2),
3557	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PLE_GAP),
3558	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_CTRL_PLE_WINDOW),
3559	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_VM_INSTR_ERROR),
3560	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_REASON),
3561	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO),
3562	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE),
3563	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_IDT_VECTORING_INFO),
3564	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE),
3565	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INSTR_LENGTH),
3566	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_RO_EXIT_INSTR_INFO),
3567	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_ES_LIMIT),
3568	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_CS_LIMIT),
3569	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SS_LIMIT),
3570	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_DS_LIMIT),
3571	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_FS_LIMIT),
3572	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_GS_LIMIT),
3573	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_LDTR_LIMIT),
3574	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_TR_LIMIT),
3575	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_GDTR_LIMIT),
3576	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_IDTR_LIMIT),
3577	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS),
3578	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS),
3579	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS),
3580	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS),
3581	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS),
3582	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS),
3583	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS),
3584	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS),
3585	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_INT_STATE),
3586	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_ACTIVITY_STATE),
3587	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SMBASE),
3588	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_GUEST_SYSENTER_CS),
3589	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_PREEMPT_TIMER_VALUE),
3590	NEM_DARWIN_VMCS32_FIELD_DUMP(VMX_VMCS32_HOST_SYSENTER_CS),
3591
3592	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR0_MASK),
3593	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR4_MASK),
3594	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR0_READ_SHADOW),
3595	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR4_READ_SHADOW),
3596	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL0),
3597	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL1),
3598	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL2),
3599	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_CTRL_CR3_TARGET_VAL3),
3600	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_EXIT_QUALIFICATION),
3601	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RCX),
3602	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RSI),
3603	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RDI),
3604	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_IO_RIP),
3605	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_RO_GUEST_LINEAR_ADDR),
3606	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CR0),
3607	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CR3),
3608	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CR4),
3609	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_ES_BASE),
3610	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_CS_BASE),
3611	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SS_BASE),
3612	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_DS_BASE),
3613	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_FS_BASE),
3614	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_GS_BASE),
3615	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_LDTR_BASE),
3616	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_TR_BASE),
3617	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_GDTR_BASE),
3618	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_IDTR_BASE),
3619	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_DR7),
3620	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_RSP),
3621	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_RIP),
3622	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_RFLAGS),
3623	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS),
3624	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SYSENTER_ESP),
3625	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SYSENTER_EIP),
3626	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_S_CET),
3627	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_SSP),
3628	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR),
3629	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_CR0),
3630	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_CR3),
3631	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_CR4),
3632	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_FS_BASE),
3633	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_GS_BASE),
3634	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_TR_BASE),
3635	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_GDTR_BASE),
3636	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_IDTR_BASE),
3637	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_SYSENTER_ESP),
3638	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_SYSENTER_EIP),
3639	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_RSP),
3640	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_RIP),
3641	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_S_CET),
3642	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_SSP),
3643	NEM_DARWIN_VMCSNW_FIELD_DUMP(VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR)
3644	#undef NEM_DARWIN_VMCSNW_FIELD_DUMP
3645	#undef NEM_DARWIN_VMCS64_FIELD_DUMP
3646	#undef NEM_DARWIN_VMCS32_FIELD_DUMP
3647	#undef NEM_DARWIN_VMCS16_FIELD_DUMP
3648	};
3649
3650	for (uint32_t i = 0; i < RT_ELEMENTS(s_aVmcsFieldsDump); i++)
3651	{
3652	if (s_aVmcsFieldsDump[i].f64Bit)
3653	{
3654	uint64_t u64Val;
3655	int rc = nemR3DarwinReadVmcs64(pVCpu, s_aVmcsFieldsDump[i].u32VmcsFieldId, &u64Val);
3656	if (RT_SUCCESS(rc))
3657	LogRel(("NEM/VMCS: %040s: 0x%016RX64\n", s_aVmcsFieldsDump[i].pszVmcsField, u64Val));
3658	else
3659	LogRel(("NEM/VMCS: %040s: rc=%Rrc\n", s_aVmcsFieldsDump[i].pszVmcsField, rc));
3660	}
3661	else
3662	{
3663	uint32_t u32Val;
3664	int rc = nemR3DarwinReadVmcs32(pVCpu, s_aVmcsFieldsDump[i].u32VmcsFieldId, &u32Val);
3665	if (RT_SUCCESS(rc))
3666	LogRel(("NEM/VMCS: %040s: 0x%08RX32\n", s_aVmcsFieldsDump[i].pszVmcsField, u32Val));
3667	else
3668	LogRel(("NEM/VMCS: %040s: rc=%Rrc\n", s_aVmcsFieldsDump[i].pszVmcsField, rc));
3669	}
3670	}
3671	}
3672
3673
3674	/**
3675	* Runs the guest once until an exit occurs.
3676	*
3677	* @returns HV status code.
3678	* @param pVM The cross context VM structure.
3679	* @param pVCpu The cross context virtual CPU structure.
3680	* @param pVmxTransient The transient VMX execution structure.
3681	*/
3682	static hv_return_t nemR3DarwinRunGuest(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
3683	{
3684	TMNotifyStartOfExecution(pVM, pVCpu);
3685
3686	Assert(!pVCpu->nem.s.fCtxChanged);
3687	hv_return_t hrc;
3688	if (hv_vcpu_run_until) /** @todo Configur the deadline dynamically based on when the next timer triggers. */
3689	hrc = hv_vcpu_run_until(pVCpu->nem.s.hVCpuId, mach_absolute_time() + 2 * RT_NS_1SEC_64 * pVM->nem.s.cMachTimePerNs);
3690	else
3691	hrc = hv_vcpu_run(pVCpu->nem.s.hVCpuId);
3692
3693	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
3694
3695	if (hrc != HV_SUCCESS)
3696	nemR3DarwinVmcsDump(pVCpu);
3697
3698	/*
3699	* Sync the TPR shadow with our APIC state.
3700	*/
3701	if ( !pVmxTransient->fIsNestedGuest
3702	&& (pVCpu->nem.s.VmcsInfo.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
3703	{
3704	uint64_t u64Tpr;
3705	hv_return_t hrc2 = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, HV_X86_TPR, &u64Tpr);
3706	Assert(hrc2 == HV_SUCCESS); RT_NOREF(hrc2);
3707
3708	if (pVmxTransient->u8GuestTpr != (uint8_t)u64Tpr)
3709	{
3710	int rc = APICSetTpr(pVCpu, (uint8_t)u64Tpr);
3711	AssertRC(rc);
3712	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
3713	}
3714	}
3715
3716	return hrc;
3717	}
3718
3719
3720	/**
3721	* Prepares the VM to run the guest.
3722	*
3723	* @returns Strict VBox status code.
3724	* @param pVM The cross context VM structure.
3725	* @param pVCpu The cross context virtual CPU structure.
3726	* @param pVmxTransient The VMX transient state.
3727	* @param fSingleStepping Flag whether we run in single stepping mode.
3728	*/
3729	static VBOXSTRICTRC nemR3DarwinPreRunGuest(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fSingleStepping)
3730	{
3731	/*
3732	* Check and process force flag actions, some of which might require us to go back to ring-3.
3733	*/
3734	VBOXSTRICTRC rcStrict = vmxHCCheckForceFlags(pVCpu, false /fIsNestedGuest/, fSingleStepping);
3735	if (rcStrict == VINF_SUCCESS)
3736	{ /likely / }
3737	else
3738	return rcStrict;
3739
3740	/*
3741	* Do not execute in HV if the A20 isn't enabled.
3742	*/
3743	if (PGMPhysIsA20Enabled(pVCpu))
3744	{ /* likely */ }
3745	else
3746	{
3747	LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
3748	return VINF_EM_RESCHEDULE_REM;
3749	}
3750
3751	/*
3752	* Evaluate events to be injected into the guest.
3753	*
3754	* Events in TRPM can be injected without inspecting the guest state.
3755	* If any new events (interrupts/NMI) are pending currently, we try to set up the
3756	* guest to cause a VM-exit the next time they are ready to receive the event.
3757	*/
3758	if (TRPMHasTrap(pVCpu))
3759	vmxHCTrpmTrapToPendingEvent(pVCpu);
3760
3761	uint32_t fIntrState;
3762	rcStrict = vmxHCEvaluatePendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, &fIntrState);
3763
3764	/*
3765	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
3766	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
3767	* also result in triple-faulting the VM.
3768	*
3769	* With nested-guests, the above does not apply since unrestricted guest execution is a
3770	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
3771	*/
3772	rcStrict = vmxHCInjectPendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, fIntrState, fSingleStepping);
3773	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3774	{ /* likely */ }
3775	else
3776	return rcStrict;
3777
3778	int rc = nemR3DarwinExportGuestState(pVM, pVCpu, pVmxTransient);
3779	AssertRCReturn(rc, rc);
3780
3781	LogFlowFunc(("Running vCPU\n"));
3782	pVCpu->nem.s.Event.fPending = false;
3783	return VINF_SUCCESS;
3784	}
3785
3786
3787	/**
3788	* The normal runloop (no debugging features enabled).
3789	*
3790	* @returns Strict VBox status code.
3791	* @param pVM The cross context VM structure.
3792	* @param pVCpu The cross context virtual CPU structure.
3793	*/
3794	static VBOXSTRICTRC nemR3DarwinRunGuestNormal(PVM pVM, PVMCPU pVCpu)
3795	{
3796	/*
3797	* The run loop.
3798	*
3799	* Current approach to state updating to use the sledgehammer and sync
3800	* everything every time. This will be optimized later.
3801	*/
3802	VMXTRANSIENT VmxTransient;
3803	RT_ZERO(VmxTransient);
3804	VmxTransient.pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
3805
3806	/*
3807	* Poll timers and run for a bit.
3808	*/
3809	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
3810	* the whole polling job when timers have changed... */
3811	uint64_t offDeltaIgnored;
3812	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
3813	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
3814	for (unsigned iLoop = 0;; iLoop++)
3815	{
3816	rcStrict = nemR3DarwinPreRunGuest(pVM, pVCpu, &VmxTransient, false /* fSingleStepping */);
3817	if (rcStrict != VINF_SUCCESS)
3818	break;
3819
3820	hv_return_t hrc = nemR3DarwinRunGuest(pVM, pVCpu, &VmxTransient);
3821	if (hrc == HV_SUCCESS)
3822	{
3823	/*
3824	* Deal with the message.
3825	*/
3826	rcStrict = nemR3DarwinHandleExit(pVM, pVCpu, &VmxTransient);
3827	if (rcStrict == VINF_SUCCESS)
3828	{ /* hopefully likely */ }
3829	else
3830	{
3831	LogFlow(("NEM/%u: breaking: nemR3DarwinHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
3832	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
3833	break;
3834	}
3835	}
3836	else
3837	{
3838	AssertLogRelMsgFailedReturn(("hv_vcpu_run()) failed for CPU #%u: %#x %u\n",
3839	pVCpu->idCpu, hrc, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
3840	VERR_NEM_IPE_0);
3841	}
3842	} /* the run loop */
3843
3844	return rcStrict;
3845	}
3846
3847
3848	/**
3849	* Checks if any expensive dtrace probes are enabled and we should go to the
3850	* debug loop.
3851	*
3852	* @returns true if we should use debug loop, false if not.
3853	*/
3854	static bool nemR3DarwinAnyExpensiveProbesEnabled(void)
3855	{
3856	/** @todo Check performance penalty when checking these over and over */
3857	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED() /* expensive too due to context */
3858	\| VBOXVMM_XCPT_DE_ENABLED()
3859	\| VBOXVMM_XCPT_DB_ENABLED()
3860	\| VBOXVMM_XCPT_BP_ENABLED()
3861	\| VBOXVMM_XCPT_OF_ENABLED()
3862	\| VBOXVMM_XCPT_BR_ENABLED()
3863	\| VBOXVMM_XCPT_UD_ENABLED()
3864	\| VBOXVMM_XCPT_NM_ENABLED()
3865	\| VBOXVMM_XCPT_DF_ENABLED()
3866	\| VBOXVMM_XCPT_TS_ENABLED()
3867	\| VBOXVMM_XCPT_NP_ENABLED()
3868	\| VBOXVMM_XCPT_SS_ENABLED()
3869	\| VBOXVMM_XCPT_GP_ENABLED()
3870	\| VBOXVMM_XCPT_PF_ENABLED()
3871	\| VBOXVMM_XCPT_MF_ENABLED()
3872	\| VBOXVMM_XCPT_AC_ENABLED()
3873	\| VBOXVMM_XCPT_XF_ENABLED()
3874	\| VBOXVMM_XCPT_VE_ENABLED()
3875	\| VBOXVMM_XCPT_SX_ENABLED()
3876	\| VBOXVMM_INT_SOFTWARE_ENABLED()
3877	/* not available in R3 \| VBOXVMM_INT_HARDWARE_ENABLED()*/
3878	) != 0
3879	\|\| ( VBOXVMM_INSTR_HALT_ENABLED()
3880	\| VBOXVMM_INSTR_MWAIT_ENABLED()
3881	\| VBOXVMM_INSTR_MONITOR_ENABLED()
3882	\| VBOXVMM_INSTR_CPUID_ENABLED()
3883	\| VBOXVMM_INSTR_INVD_ENABLED()
3884	\| VBOXVMM_INSTR_WBINVD_ENABLED()
3885	\| VBOXVMM_INSTR_INVLPG_ENABLED()
3886	\| VBOXVMM_INSTR_RDTSC_ENABLED()
3887	\| VBOXVMM_INSTR_RDTSCP_ENABLED()
3888	\| VBOXVMM_INSTR_RDPMC_ENABLED()
3889	\| VBOXVMM_INSTR_RDMSR_ENABLED()
3890	\| VBOXVMM_INSTR_WRMSR_ENABLED()
3891	\| VBOXVMM_INSTR_CRX_READ_ENABLED()
3892	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED()
3893	\| VBOXVMM_INSTR_DRX_READ_ENABLED()
3894	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED()
3895	\| VBOXVMM_INSTR_PAUSE_ENABLED()
3896	\| VBOXVMM_INSTR_XSETBV_ENABLED()
3897	\| VBOXVMM_INSTR_SIDT_ENABLED()
3898	\| VBOXVMM_INSTR_LIDT_ENABLED()
3899	\| VBOXVMM_INSTR_SGDT_ENABLED()
3900	\| VBOXVMM_INSTR_LGDT_ENABLED()
3901	\| VBOXVMM_INSTR_SLDT_ENABLED()
3902	\| VBOXVMM_INSTR_LLDT_ENABLED()
3903	\| VBOXVMM_INSTR_STR_ENABLED()
3904	\| VBOXVMM_INSTR_LTR_ENABLED()
3905	\| VBOXVMM_INSTR_GETSEC_ENABLED()
3906	\| VBOXVMM_INSTR_RSM_ENABLED()
3907	\| VBOXVMM_INSTR_RDRAND_ENABLED()
3908	\| VBOXVMM_INSTR_RDSEED_ENABLED()
3909	\| VBOXVMM_INSTR_XSAVES_ENABLED()
3910	\| VBOXVMM_INSTR_XRSTORS_ENABLED()
3911	\| VBOXVMM_INSTR_VMM_CALL_ENABLED()
3912	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED()
3913	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED()
3914	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED()
3915	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED()
3916	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED()
3917	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED()
3918	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED()
3919	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED()
3920	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED()
3921	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED()
3922	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED()
3923	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED()
3924	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED()
3925	) != 0
3926	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED()
3927	\| VBOXVMM_EXIT_HALT_ENABLED()
3928	\| VBOXVMM_EXIT_MWAIT_ENABLED()
3929	\| VBOXVMM_EXIT_MONITOR_ENABLED()
3930	\| VBOXVMM_EXIT_CPUID_ENABLED()
3931	\| VBOXVMM_EXIT_INVD_ENABLED()
3932	\| VBOXVMM_EXIT_WBINVD_ENABLED()
3933	\| VBOXVMM_EXIT_INVLPG_ENABLED()
3934	\| VBOXVMM_EXIT_RDTSC_ENABLED()
3935	\| VBOXVMM_EXIT_RDTSCP_ENABLED()
3936	\| VBOXVMM_EXIT_RDPMC_ENABLED()
3937	\| VBOXVMM_EXIT_RDMSR_ENABLED()
3938	\| VBOXVMM_EXIT_WRMSR_ENABLED()
3939	\| VBOXVMM_EXIT_CRX_READ_ENABLED()
3940	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED()
3941	\| VBOXVMM_EXIT_DRX_READ_ENABLED()
3942	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED()
3943	\| VBOXVMM_EXIT_PAUSE_ENABLED()
3944	\| VBOXVMM_EXIT_XSETBV_ENABLED()
3945	\| VBOXVMM_EXIT_SIDT_ENABLED()
3946	\| VBOXVMM_EXIT_LIDT_ENABLED()
3947	\| VBOXVMM_EXIT_SGDT_ENABLED()
3948	\| VBOXVMM_EXIT_LGDT_ENABLED()
3949	\| VBOXVMM_EXIT_SLDT_ENABLED()
3950	\| VBOXVMM_EXIT_LLDT_ENABLED()
3951	\| VBOXVMM_EXIT_STR_ENABLED()
3952	\| VBOXVMM_EXIT_LTR_ENABLED()
3953	\| VBOXVMM_EXIT_GETSEC_ENABLED()
3954	\| VBOXVMM_EXIT_RSM_ENABLED()
3955	\| VBOXVMM_EXIT_RDRAND_ENABLED()
3956	\| VBOXVMM_EXIT_RDSEED_ENABLED()
3957	\| VBOXVMM_EXIT_XSAVES_ENABLED()
3958	\| VBOXVMM_EXIT_XRSTORS_ENABLED()
3959	\| VBOXVMM_EXIT_VMM_CALL_ENABLED()
3960	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED()
3961	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED()
3962	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED()
3963	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED()
3964	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED()
3965	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED()
3966	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED()
3967	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED()
3968	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED()
3969	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED()
3970	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED()
3971	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED()
3972	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED()
3973	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED()
3974	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED()
3975	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED()
3976	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED()
3977	) != 0;
3978	}
3979
3980
3981	/**
3982	* The debug runloop.
3983	*
3984	* @returns Strict VBox status code.
3985	* @param pVM The cross context VM structure.
3986	* @param pVCpu The cross context virtual CPU structure.
3987	*/
3988	static VBOXSTRICTRC nemR3DarwinRunGuestDebug(PVM pVM, PVMCPU pVCpu)
3989	{
3990	/*
3991	* The run loop.
3992	*
3993	* Current approach to state updating to use the sledgehammer and sync
3994	* everything every time. This will be optimized later.
3995	*/
3996	VMXTRANSIENT VmxTransient;
3997	RT_ZERO(VmxTransient);
3998	VmxTransient.pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
3999
4000	bool const fSavedSingleInstruction = pVCpu->nem.s.fSingleInstruction;
4001	pVCpu->nem.s.fSingleInstruction = pVCpu->nem.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
4002	pVCpu->nem.s.fDebugWantRdTscExit = false;
4003	pVCpu->nem.s.fUsingDebugLoop = true;
4004
4005	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
4006	VMXRUNDBGSTATE DbgState;
4007	vmxHCRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
4008	vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
4009
4010	/*
4011	* Poll timers and run for a bit.
4012	*/
4013	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
4014	* the whole polling job when timers have changed... */
4015	uint64_t offDeltaIgnored;
4016	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
4017	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4018	for (unsigned iLoop = 0;; iLoop++)
4019	{
4020	bool fStepping = pVCpu->nem.s.fSingleInstruction;
4021
4022	/* Set up VM-execution controls the next two can respond to. */
4023	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
4024
4025	rcStrict = nemR3DarwinPreRunGuest(pVM, pVCpu, &VmxTransient, fStepping);
4026	if (rcStrict != VINF_SUCCESS)
4027	break;
4028
4029	/* Override any obnoxious code in the above call. */
4030	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
4031
4032	hv_return_t hrc = nemR3DarwinRunGuest(pVM, pVCpu, &VmxTransient);
4033	if (hrc == HV_SUCCESS)
4034	{
4035	/*
4036	* Deal with the message.
4037	*/
4038	rcStrict = nemR3DarwinHandleExitDebug(pVM, pVCpu, &VmxTransient, &DbgState);
4039	if (rcStrict == VINF_SUCCESS)
4040	{ /* hopefully likely */ }
4041	else
4042	{
4043	LogFlow(("NEM/%u: breaking: nemR3DarwinHandleExitDebug -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4044	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4045	break;
4046	}
4047
4048	/*
4049	* Stepping: Did the RIP change, if so, consider it a single step.
4050	* Otherwise, make sure one of the TFs gets set.
4051	*/
4052	if (fStepping)
4053	{
4054	int rc = vmxHCImportGuestStateEx(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4055	AssertRC(rc);
4056	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
4057	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
4058	{
4059	rcStrict = VINF_EM_DBG_STEPPED;
4060	break;
4061	}
4062	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
4063	}
4064	}
4065	else
4066	{
4067	AssertLogRelMsgFailedReturn(("hv_vcpu_run()) failed for CPU #%u: %#x %u\n",
4068	pVCpu->idCpu, hrc, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
4069	VERR_NEM_IPE_0);
4070	}
4071	} /* the run loop */
4072
4073	/*
4074	* Clear the X86_EFL_TF if necessary.
4075	*/
4076	if (pVCpu->nem.s.fClearTrapFlag)
4077	{
4078	int rc = vmxHCImportGuestStateEx(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
4079	AssertRC(rc);
4080	pVCpu->nem.s.fClearTrapFlag = false;
4081	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
4082	}
4083
4084	pVCpu->nem.s.fUsingDebugLoop = false;
4085	pVCpu->nem.s.fDebugWantRdTscExit = false;
4086	pVCpu->nem.s.fSingleInstruction = fSavedSingleInstruction;
4087
4088	/* Restore all controls applied by vmxHCPreRunGuestDebugStateApply above. */
4089	return vmxHCRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
4090	}
4091
4092
4093	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
4094	{
4095	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u));
4096	#ifdef LOG_ENABLED
4097	if (LogIs3Enabled())
4098	nemR3DarwinLogState(pVM, pVCpu);
4099	#endif
4100
4101	AssertReturn(NEMR3CanExecuteGuest(pVM, pVCpu), VERR_NEM_IPE_9);
4102
4103	/*
4104	* Try switch to NEM runloop state.
4105	*/
4106	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
4107	{ /* likely */ }
4108	else
4109	{
4110	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4111	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
4112	return VINF_SUCCESS;
4113	}
4114
4115	VBOXSTRICTRC rcStrict;
4116	if ( !pVCpu->nem.s.fUseDebugLoop
4117	&& !nemR3DarwinAnyExpensiveProbesEnabled()
4118	&& !DBGFIsStepping(pVCpu)
4119	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4120	rcStrict = nemR3DarwinRunGuestNormal(pVM, pVCpu);
4121	else
4122	rcStrict = nemR3DarwinRunGuestDebug(pVM, pVCpu);
4123
4124	if (rcStrict == VINF_EM_RAW_TO_R3)
4125	rcStrict = VINF_SUCCESS;
4126
4127	/*
4128	* Convert any pending HM events back to TRPM due to premature exits.
4129	*
4130	* This is because execution may continue from IEM and we would need to inject
4131	* the event from there (hence place it back in TRPM).
4132	*/
4133	if (pVCpu->nem.s.Event.fPending)
4134	{
4135	vmxHCPendingEventToTrpmTrap(pVCpu);
4136	Assert(!pVCpu->nem.s.Event.fPending);
4137
4138	/* Clear the events from the VMCS. */
4139	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
4140	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
4141	}
4142
4143
4144	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
4145	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4146
4147	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL))
4148	{
4149	/* Try anticipate what we might need. */
4150	uint64_t fImport = NEM_DARWIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
4151	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
4152	\|\| RT_FAILURE(rcStrict))
4153	fImport = CPUMCTX_EXTRN_ALL;
4154	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
4155	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4156	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
4157
4158	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
4159	{
4160	/* Only import what is external currently. */
4161	int rc2 = nemR3DarwinCopyStateFromHv(pVM, pVCpu, fImport);
4162	if (RT_SUCCESS(rc2))
4163	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4164	else if (RT_SUCCESS(rcStrict))
4165	rcStrict = rc2;
4166	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
4167	{
4168	pVCpu->cpum.GstCtx.fExtrn = 0;
4169	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4170	}
4171	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
4172	}
4173	else
4174	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4175	}
4176	else
4177	{
4178	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4179	pVCpu->cpum.GstCtx.fExtrn = 0;
4180	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4181	}
4182
4183	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
4184	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, VBOXSTRICTRC_VAL(rcStrict) ));
4185	return rcStrict;
4186	}
4187
4188
4189	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
4190	{
4191	NOREF(pVM);
4192	return PGMPhysIsA20Enabled(pVCpu);
4193	}
4194
4195
4196	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
4197	{
4198	VMCPU_ASSERT_EMT(pVCpu);
4199	bool fOld = pVCpu->nem.s.fSingleInstruction;
4200	pVCpu->nem.s.fSingleInstruction = fEnable;
4201	pVCpu->nem.s.fUseDebugLoop = fEnable \|\| pVM->nem.s.fUseDebugLoop;
4202	return fOld;
4203	}
4204
4205
4206	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
4207	{
4208	LogFlowFunc(("pVM=%p pVCpu=%p fFlags=%#x\n", pVM, pVCpu, fFlags));
4209
4210	RT_NOREF(pVM, fFlags);
4211
4212	hv_return_t hrc = hv_vcpu_interrupt(&pVCpu->nem.s.hVCpuId, 1);
4213	if (hrc != HV_SUCCESS)
4214	LogRel(("NEM: hv_vcpu_interrupt(%u, 1) failed with %#x\n", pVCpu->nem.s.hVCpuId, hrc));
4215	}
4216
4217
4218	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
4219	{
4220	for (DBGFEVENTTYPE enmEvent = DBGFEVENT_EXIT_VMX_FIRST;
4221	!fUseDebugLoop && enmEvent <= DBGFEVENT_EXIT_VMX_LAST;
4222	enmEvent = (DBGFEVENTTYPE)(enmEvent + 1))
4223	fUseDebugLoop = DBGF_IS_EVENT_ENABLED(pVM, enmEvent);
4224
4225	return fUseDebugLoop;
4226	}
4227
4228
4229	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
4230	{
4231	RT_NOREF(pVM, pVCpu);
4232	return fUseDebugLoop;
4233	}
4234
4235
4236	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
4237	uint8_t pu2State, uint32_t puNemRange)
4238	{
4239	RT_NOREF(pVM, puNemRange);
4240
4241	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p\n", GCPhys, cb, pvR3));
4242	#if defined(VBOX_WITH_PGM_NEM_MODE)
4243	if (pvR3)
4244	{
4245	int rc = nemR3DarwinMap(pVM, GCPhys, pvR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE, pu2State);
4246	if (RT_FAILURE(rc))
4247	{
4248	LogRel(("NEMR3NotifyPhysRamRegister: GCPhys=%RGp LB %RGp pvR3=%p rc=%Rrc\n", GCPhys, cb, pvR3, rc));
4249	return VERR_NEM_MAP_PAGES_FAILED;
4250	}
4251	}
4252	return VINF_SUCCESS;
4253	#else
4254	RT_NOREF(pVM, GCPhys, cb, pvR3);
4255	return VERR_NEM_MAP_PAGES_FAILED;
4256	#endif
4257	}
4258
4259
4260	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
4261	{
4262	RT_NOREF(pVM);
4263	return false;
4264	}
4265
4266
4267	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
4268	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
4269	{
4270	RT_NOREF(pVM, puNemRange, pvRam, fFlags);
4271
4272	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d)\n",
4273	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, *pu2State));
4274
4275	#if defined(VBOX_WITH_PGM_NEM_MODE)
4276	/*
4277	* Unmap the RAM we're replacing.
4278	*/
4279	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
4280	{
4281	int rc = nemR3DarwinUnmap(pVM, GCPhys, cb, pu2State);
4282	if (RT_SUCCESS(rc))
4283	{ /* likely */ }
4284	else if (pvMmio2)
4285	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rc(ignored)\n",
4286	GCPhys, cb, fFlags, rc));
4287	else
4288	{
4289	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
4290	GCPhys, cb, fFlags, rc));
4291	return VERR_NEM_UNMAP_PAGES_FAILED;
4292	}
4293	}
4294
4295	/*
4296	* Map MMIO2 if any.
4297	*/
4298	if (pvMmio2)
4299	{
4300	Assert(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2);
4301	int rc = nemR3DarwinMap(pVM, GCPhys, pvMmio2, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE, pu2State);
4302	if (RT_FAILURE(rc))
4303	{
4304	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x pvMmio2=%p: Map -> rc=%Rrc\n",
4305	GCPhys, cb, fFlags, pvMmio2, rc));
4306	return VERR_NEM_MAP_PAGES_FAILED;
4307	}
4308	}
4309	else
4310	Assert(!(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2));
4311
4312	#else
4313	RT_NOREF(pVM, GCPhys, cb, pvRam, pvMmio2);
4314	*pu2State = (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE) ? UINT8_MAX : NEM_DARWIN_PAGE_STATE_UNMAPPED;
4315	#endif
4316	return VINF_SUCCESS;
4317	}
4318
4319
4320	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
4321	void pvRam, void pvMmio2, uint32_t *puNemRange)
4322	{
4323	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
4324	return VINF_SUCCESS;
4325	}
4326
4327
4328	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
4329	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
4330	{
4331	RT_NOREF(pVM, puNemRange);
4332
4333	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p uNemRange=%#x (%#x)\n",
4334	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
4335
4336	int rc = VINF_SUCCESS;
4337	#if defined(VBOX_WITH_PGM_NEM_MODE)
4338	/*
4339	* Unmap the MMIO2 pages.
4340	*/
4341	/** @todo If we implement aliasing (MMIO2 page aliased into MMIO range),
4342	* we may have more stuff to unmap even in case of pure MMIO... */
4343	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
4344	{
4345	rc = nemR3DarwinUnmap(pVM, GCPhys, cb, pu2State);
4346	if (RT_FAILURE(rc))
4347	{
4348	LogRel(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
4349	GCPhys, cb, fFlags, rc));
4350	return VERR_NEM_UNMAP_PAGES_FAILED;
4351	}
4352	}
4353
4354	/* Ensure the page is masked as unmapped if relevant. */
4355	Assert(!pu2State \|\| *pu2State == NEM_DARWIN_PAGE_STATE_UNMAPPED);
4356
4357	/*
4358	* Restore the RAM we replaced.
4359	*/
4360	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
4361	{
4362	AssertPtr(pvRam);
4363	rc = nemR3DarwinMap(pVM, GCPhys, pvRam, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE, pu2State);
4364	if (RT_SUCCESS(rc))
4365	{ /* likely */ }
4366	else
4367	{
4368	LogRel(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp pvMmio2=%p rc=%Rrc\n", GCPhys, cb, pvMmio2, rc));
4369	rc = VERR_NEM_MAP_PAGES_FAILED;
4370	}
4371	}
4372
4373	RT_NOREF(pvMmio2);
4374	#else
4375	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
4376	if (pu2State)
4377	*pu2State = UINT8_MAX;
4378	rc = VERR_NEM_UNMAP_PAGES_FAILED;
4379	#endif
4380	return rc;
4381	}
4382
4383
4384	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
4385	void *pvBitmap, size_t cbBitmap)
4386	{
4387	RT_NOREF(pVM, GCPhys, cb, uNemRange, pvBitmap, cbBitmap);
4388	AssertFailed();
4389	return VERR_NOT_IMPLEMENTED;
4390	}
4391
4392
4393	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
4394	uint8_t pu2State, uint32_t puNemRange)
4395	{
4396	RT_NOREF(pvPages);
4397
4398	Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
4399	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
4400	if (fFlags & NEM_NOTIFY_PHYS_ROM_F_REPLACE)
4401	{
4402	int rc = nemR3DarwinUnmap(pVM, GCPhys, cb, pu2State);
4403	if (RT_FAILURE(rc))
4404	{
4405	LogRel(("NEMR3NotifyPhysRomRegisterLate: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
4406	GCPhys, cb, fFlags, rc));
4407	return VERR_NEM_UNMAP_PAGES_FAILED;
4408	}
4409	}
4410
4411	*puNemRange = 0;
4412	return VINF_SUCCESS;
4413	}
4414
4415
4416	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
4417	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
4418	{
4419	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
4420	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
4421	*pu2State = UINT8_MAX;
4422	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
4423	return VINF_SUCCESS;
4424	}
4425
4426
4427	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
4428	RTR3PTR pvMemR3, uint8_t *pu2State)
4429	{
4430	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
4431	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
4432	*pu2State = UINT8_MAX;
4433	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
4434	}
4435
4436
4437	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
4438	{
4439	Log(("NEMR3NotifySetA20: fEnabled=%RTbool\n", fEnabled));
4440	RT_NOREF(pVCpu, fEnabled);
4441	}
4442
4443
4444	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
4445	{
4446	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
4447	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
4448	}
4449
4450
4451	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
4452	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
4453	{
4454	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
4455	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
4456	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
4457	}
4458
4459
4460	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
4461	PGMPAGETYPE enmType, uint8_t *pu2State)
4462	{
4463	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4464	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
4465	RT_NOREF(HCPhys, fPageProt, enmType);
4466
4467	return nemR3DarwinUnmap(pVM, GCPhys, X86_PAGE_SIZE, pu2State);
4468	}
4469
4470
4471	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
4472	PGMPAGETYPE enmType, uint8_t *pu2State)
4473	{
4474	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp pvR3=%p fPageProt=%#x enmType=%d *pu2State=%d\n",
4475	GCPhys, HCPhys, pvR3, fPageProt, enmType, *pu2State));
4476	RT_NOREF(HCPhys, pvR3, fPageProt, enmType)
4477
4478	uint8_t u2StateOld = *pu2State;
4479	/* Can return early if this is an unmap request and the page is not mapped. */
4480	if ( fPageProt == NEM_PAGE_PROT_NONE
4481	&& u2StateOld == NEM_DARWIN_PAGE_STATE_UNMAPPED)
4482	{
4483	Assert(!pvR3);
4484	return;
4485	}
4486
4487	int rc;
4488	if (u2StateOld == NEM_DARWIN_PAGE_STATE_UNMAPPED)
4489	{
4490	AssertPtr(pvR3);
4491	rc = nemR3DarwinMap(pVM, GCPhys, pvR3, X86_PAGE_SIZE, fPageProt, pu2State);
4492	}
4493	else
4494	rc = nemR3DarwinProtect(pVM, GCPhys, X86_PAGE_SIZE, fPageProt, pu2State);
4495	AssertLogRelMsgRC(rc, ("NEMHCNotifyPhysPageProtChanged: nemR3DarwinMap/nemR3DarwinProtect(,%p,%RGp,%RGp,) u2StateOld=%u -> %Rrc\n",
4496	pvR3, GCPhys, X86_PAGE_SIZE, u2StateOld, rc));
4497	}
4498
4499
4500	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
4501	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
4502	{
4503	Log5(("NEMHCNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4504	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
4505	RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType);
4506
4507	int rc = nemR3DarwinUnmap(pVM, GCPhys, X86_PAGE_SIZE, pu2State);
4508	if (RT_SUCCESS(rc))
4509	{
4510	rc = nemR3DarwinMap(pVM, GCPhys, pvNewR3, X86_PAGE_SIZE, fPageProt, pu2State);
4511	AssertLogRelMsgRC(rc, ("NEMHCNotifyPhysPageChanged: nemR3DarwinMap(,%p,%RGp,%RGp,) -> %Rrc\n",
4512	pvNewR3, GCPhys, X86_PAGE_SIZE, rc));
4513	}
4514	else
4515	AssertReleaseFailed();
4516	}
4517
4518
4519	/**
4520	* Interface for importing state on demand (used by IEM).
4521	*
4522	* @returns VBox status code.
4523	* @param pVCpu The cross context CPU structure.
4524	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
4525	*/
4526	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
4527	{
4528	LogFlowFunc(("pVCpu=%p fWhat=%RX64\n", pVCpu, fWhat));
4529	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
4530
4531	return nemR3DarwinCopyStateFromHv(pVCpu->pVMR3, pVCpu, fWhat);
4532	}
4533
4534
4535	/**
4536	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
4537	*
4538	* @returns VBox status code.
4539	* @param pVCpu The cross context CPU structure.
4540	* @param pcTicks Where to return the CPU tick count.
4541	* @param puAux Where to return the TSC_AUX register value.
4542	*/
4543	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
4544	{
4545	LogFlowFunc(("pVCpu=%p pcTicks=%RX64 puAux=%RX32\n", pVCpu, pcTicks, puAux));
4546	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
4547
4548	int rc = nemR3DarwinMsrRead(pVCpu, MSR_IA32_TSC, pcTicks);
4549	if ( RT_SUCCESS(rc)
4550	&& puAux)
4551	{
4552	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX)
4553	{
4554	uint64_t u64Aux;
4555	rc = nemR3DarwinMsrRead(pVCpu, MSR_K8_TSC_AUX, &u64Aux);
4556	if (RT_SUCCESS(rc))
4557	*puAux = (uint32_t)u64Aux;
4558	}
4559	else
4560	*puAux = CPUMGetGuestTscAux(pVCpu);
4561	}
4562
4563	return rc;
4564	}
4565
4566
4567	/**
4568	* Resumes CPU clock (TSC) on all virtual CPUs.
4569	*
4570	* This is called by TM when the VM is started, restored, resumed or similar.
4571	*
4572	* @returns VBox status code.
4573	* @param pVM The cross context VM structure.
4574	* @param pVCpu The cross context CPU structure of the calling EMT.
4575	* @param uPausedTscValue The TSC value at the time of pausing.
4576	*/
4577	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
4578	{
4579	LogFlowFunc(("pVM=%p pVCpu=%p uPausedTscValue=%RX64\n", pVCpu, uPausedTscValue));
4580	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
4581	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
4582
4583	hv_return_t hrc = hv_vm_sync_tsc(uPausedTscValue);
4584	if (RT_LIKELY(hrc == HV_SUCCESS))
4585	{
4586	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_TSC_AUX);
4587	return VINF_SUCCESS;
4588	}
4589
4590	return nemR3DarwinHvSts2Rc(hrc);
4591	}
4592
4593
4594	/**
4595	* Returns features supported by the NEM backend.
4596	*
4597	* @returns Flags of features supported by the native NEM backend.
4598	* @param pVM The cross context VM structure.
4599	*/
4600	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
4601	{
4602	RT_NOREF(pVM);
4603	/*
4604	* Apple's Hypervisor.framework is not supported if the CPU doesn't support nested paging
4605	* and unrestricted guest execution support so we can safely return these flags here always.
4606	*/
4607	return NEM_FEAT_F_NESTED_PAGING \| NEM_FEAT_F_FULL_GST_EXEC \| NEM_FEAT_F_XSAVE_XRSTOR;
4608	}
4609
4610
4611	/** @page pg_nem_darwin NEM/darwin - Native Execution Manager, macOS.
4612	*
4613	* @todo Add notes as the implementation progresses...
4614	*/
4615

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-darwin.cpp@ 103583

Download in other formats: