NEMR3Native-darwin.cpp@ 93115

Last change on this file since 93115 was 93115, checked in by vboxsync, 3 years ago
scm --update-copyright-year
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 129.1 KB

Line
1	/* $Id: NEMR3Native-darwin.cpp 93115 2022-01-01 11:31:46Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 macOS backend using Hypervisor.framework.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	*/
9
10	/*
11	* Copyright (C) 2020-2022 Oracle Corporation
12	*
13	* This file is part of VirtualBox Open Source Edition (OSE), as
14	* available from http://www.virtualbox.org. This file is free software;
15	* you can redistribute it and/or modify it under the terms of the GNU
16	* General Public License (GPL) as published by the Free Software
17	* Foundation, in version 2 as it comes in the "COPYING" file of the
18	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20	*/
21
22
23	/*********************************************************************************************************************************
24	* Header Files *
25	*********************************************************************************************************************************/
26	#define LOG_GROUP LOG_GROUP_NEM
27	#define VMCPU_INCL_CPUM_GST_CTX
28	#include <VBox/vmm/nem.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/em.h>
31	#include <VBox/vmm/apic.h>
32	#include <VBox/vmm/pdm.h>
33	#include <VBox/vmm/hm.h>
34	#include <VBox/vmm/hm_vmx.h>
35	#include <VBox/vmm/dbgftrace.h>
36	#include "VMXInternal.h"
37	#include "NEMInternal.h"
38	#include <VBox/vmm/vmcc.h>
39	#include "dtrace/VBoxVMM.h"
40
41	#include <iprt/asm.h>
42	#include <iprt/ldr.h>
43	#include <iprt/mem.h>
44	#include <iprt/path.h>
45	#include <iprt/string.h>
46	#include <iprt/system.h>
47	#include <iprt/utf16.h>
48
49
50	/*********************************************************************************************************************************
51	* Defined Constants And Macros *
52	*********************************************************************************************************************************/
53	/* No nested hwvirt (for now). */
54	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
55	# undef VBOX_WITH_NESTED_HWVIRT_VMX
56	#endif
57
58
59	/** @name HV return codes.
60	* @{ */
61	/** Operation was successful. */
62	#define HV_SUCCESS 0
63	/** An error occurred during operation. */
64	#define HV_ERROR 0xfae94001
65	/** The operation could not be completed right now, try again. */
66	#define HV_BUSY 0xfae94002
67	/** One of the parameters passed wis invalid. */
68	#define HV_BAD_ARGUMENT 0xfae94003
69	/** Not enough resources left to fulfill the operation. */
70	#define HV_NO_RESOURCES 0xfae94005
71	/** The device could not be found. */
72	#define HV_NO_DEVICE 0xfae94006
73	/** The operation is not supportd on this platform with this configuration. */
74	#define HV_UNSUPPORTED 0xfae94007
75	/** @} */
76
77
78	/** @name HV memory protection flags.
79	* @{ */
80	/** Memory is readable. */
81	#define HV_MEMORY_READ RT_BIT_64(0)
82	/** Memory is writeable. */
83	#define HV_MEMORY_WRITE RT_BIT_64(1)
84	/** Memory is executable. */
85	#define HV_MEMORY_EXEC RT_BIT_64(2)
86	/** @} */
87
88
89	/** @name HV shadow VMCS protection flags.
90	* @{ */
91	/** Shadow VMCS field is not accessible. */
92	#define HV_SHADOW_VMCS_NONE 0
93	/** Shadow VMCS fild is readable. */
94	#define HV_SHADOW_VMCS_READ RT_BIT_64(0)
95	/** Shadow VMCS field is writeable. */
96	#define HV_SHADOW_VMCS_WRITE RT_BIT_64(1)
97	/** @} */
98
99
100	/** Default VM creation flags. */
101	#define HV_VM_DEFAULT 0
102	/** Default guest address space creation flags. */
103	#define HV_VM_SPACE_DEFAULT 0
104	/** Default vCPU creation flags. */
105	#define HV_VCPU_DEFAULT 0
106
107	#define HV_DEADLINE_FOREVER UINT64_MAX
108
109
110	/*********************************************************************************************************************************
111	* Structures and Typedefs *
112	*********************************************************************************************************************************/
113
114	/** HV return code type. */
115	typedef uint32_t hv_return_t;
116	/** HV capability bitmask. */
117	typedef uint64_t hv_capability_t;
118	/** Option bitmask type when creating a VM. */
119	typedef uint64_t hv_vm_options_t;
120	/** Option bitmask when creating a vCPU. */
121	typedef uint64_t hv_vcpu_options_t;
122	/** HV memory protection flags type. */
123	typedef uint64_t hv_memory_flags_t;
124	/** Shadow VMCS protection flags. */
125	typedef uint64_t hv_shadow_flags_t;
126	/** Guest physical address type. */
127	typedef uint64_t hv_gpaddr_t;
128
129
130	/**
131	* VMX Capability enumeration.
132	*/
133	typedef enum
134	{
135	HV_VMX_CAP_PINBASED = 0,
136	HV_VMX_CAP_PROCBASED,
137	HV_VMX_CAP_PROCBASED2,
138	HV_VMX_CAP_ENTRY,
139	HV_VMX_CAP_EXIT,
140	HV_VMX_CAP_BASIC, /* Since 11.0 */
141	HV_VMX_CAP_TRUE_PINBASED, /* Since 11.0 */
142	HV_VMX_CAP_TRUE_PROCBASED, /* Since 11.0 */
143	HV_VMX_CAP_TRUE_ENTRY, /* Since 11.0 */
144	HV_VMX_CAP_TRUE_EXIT, /* Since 11.0 */
145	HV_VMX_CAP_MISC, /* Since 11.0 */
146	HV_VMX_CAP_CR0_FIXED0, /* Since 11.0 */
147	HV_VMX_CAP_CR0_FIXED1, /* Since 11.0 */
148	HV_VMX_CAP_CR4_FIXED0, /* Since 11.0 */
149	HV_VMX_CAP_CR4_FIXED1, /* Since 11.0 */
150	HV_VMX_CAP_VMCS_ENUM, /* Since 11.0 */
151	HV_VMX_CAP_EPT_VPID_CAP, /* Since 11.0 */
152	HV_VMX_CAP_PREEMPTION_TIMER = 32
153	} hv_vmx_capability_t;
154
155
156	/**
157	* HV x86 register enumeration.
158	*/
159	typedef enum
160	{
161	HV_X86_RIP = 0,
162	HV_X86_RFLAGS,
163	HV_X86_RAX,
164	HV_X86_RCX,
165	HV_X86_RDX,
166	HV_X86_RBX,
167	HV_X86_RSI,
168	HV_X86_RDI,
169	HV_X86_RSP,
170	HV_X86_RBP,
171	HV_X86_R8,
172	HV_X86_R9,
173	HV_X86_R10,
174	HV_X86_R11,
175	HV_X86_R12,
176	HV_X86_R13,
177	HV_X86_R14,
178	HV_X86_R15,
179	HV_X86_CS,
180	HV_X86_SS,
181	HV_X86_DS,
182	HV_X86_ES,
183	HV_X86_FS,
184	HV_X86_GS,
185	HV_X86_IDT_BASE,
186	HV_X86_IDT_LIMIT,
187	HV_X86_GDT_BASE,
188	HV_X86_GDT_LIMIT,
189	HV_X86_LDTR,
190	HV_X86_LDT_BASE,
191	HV_X86_LDT_LIMIT,
192	HV_X86_LDT_AR,
193	HV_X86_TR,
194	HV_X86_TSS_BASE,
195	HV_X86_TSS_LIMIT,
196	HV_X86_TSS_AR,
197	HV_X86_CR0,
198	HV_X86_CR1,
199	HV_X86_CR2,
200	HV_X86_CR3,
201	HV_X86_CR4,
202	HV_X86_DR0,
203	HV_X86_DR1,
204	HV_X86_DR2,
205	HV_X86_DR3,
206	HV_X86_DR4,
207	HV_X86_DR5,
208	HV_X86_DR6,
209	HV_X86_DR7,
210	HV_X86_TPR,
211	HV_X86_XCR0,
212	HV_X86_REGISTERS_MAX
213	} hv_x86_reg_t;
214
215
216	typedef hv_return_t FN_HV_CAPABILITY(hv_capability_t capability, uint64_t *valu);
217	typedef hv_return_t FN_HV_VM_CREATE(hv_vm_options_t flags);
218	typedef hv_return_t FN_HV_VM_DESTROY(void);
219	typedef hv_return_t FN_HV_VM_SPACE_CREATE(hv_vm_space_t *asid);
220	typedef hv_return_t FN_HV_VM_SPACE_DESTROY(hv_vm_space_t asid);
221	typedef hv_return_t FN_HV_VM_MAP(const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
222	typedef hv_return_t FN_HV_VM_UNMAP(hv_gpaddr_t gpa, size_t size);
223	typedef hv_return_t FN_HV_VM_PROTECT(hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
224	typedef hv_return_t FN_HV_VM_MAP_SPACE(hv_vm_space_t asid, const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
225	typedef hv_return_t FN_HV_VM_UNMAP_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size);
226	typedef hv_return_t FN_HV_VM_PROTECT_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
227	typedef hv_return_t FN_HV_VM_SYNC_TSC(uint64_t tsc);
228
229	typedef hv_return_t FN_HV_VCPU_CREATE(hv_vcpuid_t *vcpu, hv_vcpu_options_t flags);
230	typedef hv_return_t FN_HV_VCPU_DESTROY(hv_vcpuid_t vcpu);
231	typedef hv_return_t FN_HV_VCPU_SET_SPACE(hv_vcpuid_t vcpu, hv_vm_space_t asid);
232	typedef hv_return_t FN_HV_VCPU_READ_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t *value);
233	typedef hv_return_t FN_HV_VCPU_WRITE_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t value);
234	typedef hv_return_t FN_HV_VCPU_READ_FPSTATE(hv_vcpuid_t vcpu, void *buffer, size_t size);
235	typedef hv_return_t FN_HV_VCPU_WRITE_FPSTATE(hv_vcpuid_t vcpu, const void *buffer, size_t size);
236	typedef hv_return_t FN_HV_VCPU_ENABLE_NATIVE_MSR(hv_vcpuid_t vcpu, uint32_t msr, bool enable);
237	typedef hv_return_t FN_HV_VCPU_READ_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t *value);
238	typedef hv_return_t FN_HV_VCPU_WRITE_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t value);
239	typedef hv_return_t FN_HV_VCPU_FLUSH(hv_vcpuid_t vcpu);
240	typedef hv_return_t FN_HV_VCPU_INVALIDATE_TLB(hv_vcpuid_t vcpu);
241	typedef hv_return_t FN_HV_VCPU_RUN(hv_vcpuid_t vcpu);
242	typedef hv_return_t FN_HV_VCPU_RUN_UNTIL(hv_vcpuid_t vcpu, uint64_t deadline);
243	typedef hv_return_t FN_HV_VCPU_INTERRUPT(hv_vcpuid_t *vcpus, unsigned int vcpu_count);
244	typedef hv_return_t FN_HV_VCPU_GET_EXEC_TIME(hv_vcpuid_t vcpus, uint64_t time);
245
246	typedef hv_return_t FN_HV_VMX_VCPU_READ_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
247	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
248
249	typedef hv_return_t FN_HV_VMX_VCPU_READ_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
250	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
251	typedef hv_return_t FN_HV_VMX_VCPU_SET_SHADOW_ACCESS(hv_vcpuid_t vcpu, uint32_t field, hv_shadow_flags_t flags);
252
253	typedef hv_return_t FN_HV_VMX_READ_CAPABILITY(hv_vmx_capability_t field, uint64_t *value);
254	typedef hv_return_t FN_HV_VMX_VCPU_SET_APIC_ADDRESS(hv_vcpuid_t vcpu, hv_gpaddr_t gpa);
255
256
257	/*********************************************************************************************************************************
258	* Global Variables *
259	*********************************************************************************************************************************/
260	/** NEM_DARWIN_PAGE_STATE_XXX names. */
261	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
262	/** MSRs. */
263	static SUPHWVIRTMSRS g_HmMsrs;
264	/** VMX: Set if swapping EFER is supported. */
265	static bool g_fHmVmxSupportsVmcsEfer = false;
266	/** @name APIs imported from Hypervisor.framework.
267	* @{ */
268	static FN_HV_CAPABILITY g_pfnHvCapability = NULL; / Since 10.15 */
269	static FN_HV_VM_CREATE g_pfnHvVmCreate = NULL; / Since 10.10 */
270	static FN_HV_VM_DESTROY g_pfnHvVmDestroy = NULL; / Since 10.10 */
271	static FN_HV_VM_SPACE_CREATE g_pfnHvVmSpaceCreate = NULL; / Since 10.15 */
272	static FN_HV_VM_SPACE_DESTROY g_pfnHvVmSpaceDestroy = NULL; / Since 10.15 */
273	static FN_HV_VM_MAP g_pfnHvVmMap = NULL; / Since 10.10 */
274	static FN_HV_VM_UNMAP g_pfnHvVmUnmap = NULL; / Since 10.10 */
275	static FN_HV_VM_PROTECT g_pfnHvVmProtect = NULL; / Since 10.10 */
276	static FN_HV_VM_MAP_SPACE g_pfnHvVmMapSpace = NULL; / Since 10.15 */
277	static FN_HV_VM_UNMAP_SPACE g_pfnHvVmUnmapSpace = NULL; / Since 10.15 */
278	static FN_HV_VM_PROTECT_SPACE g_pfnHvVmProtectSpace = NULL; / Since 10.15 */
279	static FN_HV_VM_SYNC_TSC g_pfnHvVmSyncTsc = NULL; / Since 10.10 */
280
281	static FN_HV_VCPU_CREATE g_pfnHvVCpuCreate = NULL; / Since 10.10 */
282	static FN_HV_VCPU_DESTROY g_pfnHvVCpuDestroy = NULL; / Since 10.10 */
283	static FN_HV_VCPU_SET_SPACE g_pfnHvVCpuSetSpace = NULL; / Since 10.15 */
284	static FN_HV_VCPU_READ_REGISTER g_pfnHvVCpuReadRegister = NULL; / Since 10.10 */
285	static FN_HV_VCPU_WRITE_REGISTER g_pfnHvVCpuWriteRegister = NULL; / Since 10.10 */
286	static FN_HV_VCPU_READ_FPSTATE g_pfnHvVCpuReadFpState = NULL; / Since 10.10 */
287	static FN_HV_VCPU_WRITE_FPSTATE g_pfnHvVCpuWriteFpState = NULL; / Since 10.10 */
288	static FN_HV_VCPU_ENABLE_NATIVE_MSR g_pfnHvVCpuEnableNativeMsr = NULL; / Since 10.10 */
289	static FN_HV_VCPU_READ_MSR g_pfnHvVCpuReadMsr = NULL; / Since 10.10 */
290	static FN_HV_VCPU_WRITE_MSR g_pfnHvVCpuWriteMsr = NULL; / Since 10.10 */
291	static FN_HV_VCPU_FLUSH g_pfnHvVCpuFlush = NULL; / Since 10.10 */
292	static FN_HV_VCPU_INVALIDATE_TLB g_pfnHvVCpuInvalidateTlb = NULL; / Since 10.10 */
293	static FN_HV_VCPU_RUN g_pfnHvVCpuRun = NULL; / Since 10.10 */
294	static FN_HV_VCPU_RUN_UNTIL g_pfnHvVCpuRunUntil = NULL; / Since 10.15 */
295	static FN_HV_VCPU_INTERRUPT g_pfnHvVCpuInterrupt = NULL; / Since 10.10 */
296	static FN_HV_VCPU_GET_EXEC_TIME g_pfnHvVCpuGetExecTime = NULL; / Since 10.10 */
297
298	static FN_HV_VMX_READ_CAPABILITY g_pfnHvVmxReadCapability = NULL; / Since 10.10 */
299	static FN_HV_VMX_VCPU_READ_VMCS g_pfnHvVmxVCpuReadVmcs = NULL; / Since 10.10 */
300	static FN_HV_VMX_VCPU_WRITE_VMCS g_pfnHvVmxVCpuWriteVmcs = NULL; / Since 10.10 */
301	static FN_HV_VMX_VCPU_READ_SHADOW_VMCS g_pfnHvVmxVCpuReadShadowVmcs = NULL; / Since 10.15 */
302	static FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS g_pfnHvVmxVCpuWriteShadowVmcs = NULL; / Since 10.15 */
303	static FN_HV_VMX_VCPU_SET_SHADOW_ACCESS g_pfnHvVmxVCpuSetShadowAccess = NULL; / Since 10.15 */
304	static FN_HV_VMX_VCPU_SET_APIC_ADDRESS g_pfnHvVmxVCpuSetApicAddress = NULL; / Since 10.10 */
305	/** @} */
306
307
308	/**
309	* Import instructions.
310	*/
311	static const struct
312	{
313	bool fOptional; /*< Set if import is optional. /
314	void ppfn; /< The function pointer variable. */
315	const char pszName; /< The function name. /
316	} g_aImports[] =
317	{
318	#define NEM_DARWIN_IMPORT(a_fOptional, a_Pfn, a_Name) { (a_fOptional), (void **)&(a_Pfn), #a_Name }
319	NEM_DARWIN_IMPORT(true, g_pfnHvCapability, hv_capability),
320	NEM_DARWIN_IMPORT(false, g_pfnHvVmCreate, hv_vm_create),
321	NEM_DARWIN_IMPORT(false, g_pfnHvVmDestroy, hv_vm_destroy),
322	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceCreate, hv_vm_space_create),
323	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceDestroy, hv_vm_space_destroy),
324	NEM_DARWIN_IMPORT(false, g_pfnHvVmMap, hv_vm_map),
325	NEM_DARWIN_IMPORT(false, g_pfnHvVmUnmap, hv_vm_unmap),
326	NEM_DARWIN_IMPORT(false, g_pfnHvVmProtect, hv_vm_protect),
327	NEM_DARWIN_IMPORT(true, g_pfnHvVmMapSpace, hv_vm_map_space),
328	NEM_DARWIN_IMPORT(true, g_pfnHvVmUnmapSpace, hv_vm_unmap_space),
329	NEM_DARWIN_IMPORT(true, g_pfnHvVmProtectSpace, hv_vm_protect_space),
330	NEM_DARWIN_IMPORT(false, g_pfnHvVmSyncTsc, hv_vm_sync_tsc),
331
332	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuCreate, hv_vcpu_create),
333	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuDestroy, hv_vcpu_destroy),
334	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuSetSpace, hv_vcpu_set_space),
335	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadRegister, hv_vcpu_read_register),
336	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteRegister, hv_vcpu_write_register),
337	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadFpState, hv_vcpu_read_fpstate),
338	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteFpState, hv_vcpu_write_fpstate),
339	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuEnableNativeMsr, hv_vcpu_enable_native_msr),
340	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadMsr, hv_vcpu_read_msr),
341	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteMsr, hv_vcpu_write_msr),
342	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuFlush, hv_vcpu_flush),
343	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInvalidateTlb, hv_vcpu_invalidate_tlb),
344	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuRun, hv_vcpu_run),
345	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuRunUntil, hv_vcpu_run_until),
346	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInterrupt, hv_vcpu_interrupt),
347	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuGetExecTime, hv_vcpu_get_exec_time),
348	NEM_DARWIN_IMPORT(false, g_pfnHvVmxReadCapability, hv_vmx_read_capability),
349	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuReadVmcs, hv_vmx_vcpu_read_vmcs),
350	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuWriteVmcs, hv_vmx_vcpu_write_vmcs),
351	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuReadShadowVmcs, hv_vmx_vcpu_read_shadow_vmcs),
352	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuWriteShadowVmcs, hv_vmx_vcpu_write_shadow_vmcs),
353	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuSetShadowAccess, hv_vmx_vcpu_set_shadow_access),
354	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuSetApicAddress, hv_vmx_vcpu_set_apic_address),
355	#undef NEM_DARWIN_IMPORT
356	};
357
358
359	/*
360	* Let the preprocessor alias the APIs to import variables for better autocompletion.
361	*/
362	#ifndef IN_SLICKEDIT
363	# define hv_capability g_pfnHvCapability
364	# define hv_vm_create g_pfnHvVmCreate
365	# define hv_vm_destroy g_pfnHvVmDestroy
366	# define hv_vm_space_create g_pfnHvVmSpaceCreate
367	# define hv_vm_space_destroy g_pfnHvVmSpaceDestroy
368	# define hv_vm_map g_pfnHvVmMap
369	# define hv_vm_unmap g_pfnHvVmUnmap
370	# define hv_vm_protect g_pfnHvVmProtect
371	# define hv_vm_map_space g_pfnHvVmMapSpace
372	# define hv_vm_unmap_space g_pfnHvVmUnmapSpace
373	# define hv_vm_protect_space g_pfnHvVmProtectSpace
374	# define hv_vm_sync_tsc g_pfnHvVmSyncTsc
375
376	# define hv_vcpu_create g_pfnHvVCpuCreate
377	# define hv_vcpu_destroy g_pfnHvVCpuDestroy
378	# define hv_vcpu_set_space g_pfnHvVCpuSetSpace
379	# define hv_vcpu_read_register g_pfnHvVCpuReadRegister
380	# define hv_vcpu_write_register g_pfnHvVCpuWriteRegister
381	# define hv_vcpu_read_fpstate g_pfnHvVCpuReadFpState
382	# define hv_vcpu_write_fpstate g_pfnHvVCpuWriteFpState
383	# define hv_vcpu_enable_native_msr g_pfnHvVCpuEnableNativeMsr
384	# define hv_vcpu_read_msr g_pfnHvVCpuReadMsr
385	# define hv_vcpu_write_msr g_pfnHvVCpuWriteMsr
386	# define hv_vcpu_flush g_pfnHvVCpuFlush
387	# define hv_vcpu_invalidate_tlb g_pfnHvVCpuInvalidateTlb
388	# define hv_vcpu_run g_pfnHvVCpuRun
389	# define hv_vcpu_run_until g_pfnHvVCpuRunUntil
390	# define hv_vcpu_interrupt g_pfnHvVCpuInterrupt
391	# define hv_vcpu_get_exec_time g_pfnHvVCpuGetExecTime
392
393	# define hv_vmx_read_capability g_pfnHvVmxReadCapability
394	# define hv_vmx_vcpu_read_vmcs g_pfnHvVmxVCpuReadVmcs
395	# define hv_vmx_vcpu_write_vmcs g_pfnHvVmxVCpuWriteVmcs
396	# define hv_vmx_vcpu_read_shadow_vmcs g_pfnHvVmxVCpuReadShadowVmcs
397	# define hv_vmx_vcpu_write_shadow_vmcs g_pfnHvVmxVCpuWriteShadowVmcs
398	# define hv_vmx_vcpu_set_shadow_access g_pfnHvVmxVCpuSetShadowAccess
399	# define hv_vmx_vcpu_set_apic_address g_pfnHvVmxVCpuSetApicAddress
400	#endif
401
402
403	/*********************************************************************************************************************************
404	* Internal Functions *
405	*********************************************************************************************************************************/
406	static void vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo);
407
408	/**
409	* Converts a HV return code to a VBox status code.
410	*
411	* @returns VBox status code.
412	* @param hrc The HV return code to convert.
413	*/
414	DECLINLINE(int) nemR3DarwinHvSts2Rc(hv_return_t hrc)
415	{
416	if (hrc == HV_SUCCESS)
417	return VINF_SUCCESS;
418
419	switch (hrc)
420	{
421	case HV_ERROR: return VERR_INVALID_STATE;
422	case HV_BUSY: return VERR_RESOURCE_BUSY;
423	case HV_BAD_ARGUMENT: return VERR_INVALID_PARAMETER;
424	case HV_NO_RESOURCES: return VERR_OUT_OF_RESOURCES;
425	case HV_NO_DEVICE: return VERR_NOT_FOUND;
426	case HV_UNSUPPORTED: return VERR_NOT_SUPPORTED;
427	}
428
429	return VERR_IPE_UNEXPECTED_STATUS;
430	}
431
432
433	/**
434	* Unmaps the given guest physical address range (page aligned).
435	*
436	* @returns VBox status code.
437	* @param pVM The cross context VM structure.
438	* @param GCPhys The guest physical address to start unmapping at.
439	* @param cb The size of the range to unmap in bytes.
440	*/
441	DECLINLINE(int) nemR3DarwinUnmap(PVM pVM, RTGCPHYS GCPhys, size_t cb)
442	{
443	LogFlowFunc(("Unmapping %RGp LB %zu\n", GCPhys, cb));
444	hv_return_t hrc;
445	if (pVM->nem.s.fCreatedAsid)
446	hrc = hv_vm_unmap_space(pVM->nem.s.uVmAsid, GCPhys, cb);
447	else
448	hrc = hv_vm_unmap(GCPhys, cb);
449	return nemR3DarwinHvSts2Rc(hrc);
450	}
451
452
453	/**
454	* Maps a given guest physical address range backed by the given memory with the given
455	* protection flags.
456	*
457	* @returns VBox status code.
458	* @param pVM The cross context VM structure.
459	* @param GCPhys The guest physical address to start mapping.
460	* @param pvRam The R3 pointer of the memory to back the range with.
461	* @param cb The size of the range, page aligned.
462	* @param fPageProt The page protection flags to use for this range, combination of NEM_PAGE_PROT_XXX
463	*/
464	DECLINLINE(int) nemR3DarwinMap(PVM pVM, RTGCPHYS GCPhys, void *pvRam, size_t cb, uint32_t fPageProt)
465	{
466	LogFlowFunc(("Mapping %RGp LB %zu fProt=%#x\n", GCPhys, cb, fPageProt));
467
468	hv_memory_flags_t fHvMemProt = 0;
469	if (fPageProt & NEM_PAGE_PROT_READ)
470	fHvMemProt \|= HV_MEMORY_READ;
471	if (fPageProt & NEM_PAGE_PROT_WRITE)
472	fHvMemProt \|= HV_MEMORY_WRITE;
473	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
474	fHvMemProt \|= HV_MEMORY_EXEC;
475
476	hv_return_t hrc;
477	if (pVM->nem.s.fCreatedAsid)
478	hrc = hv_vm_map_space(pVM->nem.s.uVmAsid, pvRam, GCPhys, cb, fHvMemProt);
479	else
480	hrc = hv_vm_map(pvRam, GCPhys, cb, fHvMemProt);
481	return nemR3DarwinHvSts2Rc(hrc);
482	}
483
484
485	#if 0 /* unused */
486	DECLINLINE(int) nemR3DarwinProtectPage(PVM pVM, RTGCPHYS GCPhys, size_t cb, uint32_t fPageProt)
487	{
488	hv_memory_flags_t fHvMemProt = 0;
489	if (fPageProt & NEM_PAGE_PROT_READ)
490	fHvMemProt \|= HV_MEMORY_READ;
491	if (fPageProt & NEM_PAGE_PROT_WRITE)
492	fHvMemProt \|= HV_MEMORY_WRITE;
493	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
494	fHvMemProt \|= HV_MEMORY_EXEC;
495
496	if (pVM->nem.s.fCreatedAsid)
497	hrc = hv_vm_protect_space(pVM->nem.s.uVmAsid, GCPhys, cb, fHvMemProt);
498	else
499	hrc = hv_vm_protect(GCPhys, cb, fHvMemProt);
500
501	return nemR3DarwinHvSts2Rc(hrc);
502	}
503	#endif
504
505
506	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
507	{
508	PGMPAGEMAPLOCK Lock;
509	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
510	if (RT_SUCCESS(rc))
511	PGMPhysReleasePageMappingLock(pVM, &Lock);
512	return rc;
513	}
514
515
516	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
517	{
518	PGMPAGEMAPLOCK Lock;
519	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
520	if (RT_SUCCESS(rc))
521	PGMPhysReleasePageMappingLock(pVM, &Lock);
522	return rc;
523	}
524
525
526	/**
527	* Worker that maps pages into Hyper-V.
528	*
529	* This is used by the PGM physical page notifications as well as the memory
530	* access VMEXIT handlers.
531	*
532	* @returns VBox status code.
533	* @param pVM The cross context VM structure.
534	* @param pVCpu The cross context virtual CPU structure of the
535	* calling EMT.
536	* @param GCPhysSrc The source page address.
537	* @param GCPhysDst The hyper-V destination page. This may differ from
538	* GCPhysSrc when A20 is disabled.
539	* @param fPageProt NEM_PAGE_PROT_XXX.
540	* @param pu2State Our page state (input/output).
541	* @param fBackingChanged Set if the page backing is being changed.
542	* @thread EMT(pVCpu)
543	*/
544	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
545	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
546	{
547	/*
548	* Looks like we need to unmap a page before we can change the backing
549	* or even modify the protection. This is going to be REALLY efficient.
550	* PGM lends us two bits to keep track of the state here.
551	*/
552	RT_NOREF(pVCpu);
553	uint8_t const u2OldState = *pu2State;
554	uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_DARWIN_PAGE_STATE_WRITABLE
555	: fPageProt & NEM_PAGE_PROT_READ ? NEM_DARWIN_PAGE_STATE_READABLE : NEM_DARWIN_PAGE_STATE_UNMAPPED;
556	if ( fBackingChanged
557	\|\| u2NewState != u2OldState)
558	{
559	if (u2OldState > NEM_DARWIN_PAGE_STATE_UNMAPPED)
560	{
561	int rc = nemR3DarwinUnmap(pVM, GCPhysDst, X86_PAGE_SIZE);
562	if (RT_SUCCESS(rc))
563	{
564	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
565	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
566	if (u2NewState == NEM_DARWIN_PAGE_STATE_UNMAPPED)
567	{
568	Log5(("NEM GPA unmapped/set: %RGp (was %s)\n", GCPhysDst, g_apszPageStates[u2OldState]));
569	return VINF_SUCCESS;
570	}
571	}
572	else
573	{
574	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
575	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
576	return VERR_NEM_INIT_FAILED;
577	}
578	}
579	}
580
581	/*
582	* Writeable mapping?
583	*/
584	if (fPageProt & NEM_PAGE_PROT_WRITE)
585	{
586	void *pvPage;
587	int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
588	if (RT_SUCCESS(rc))
589	{
590	rc = nemR3DarwinMap(pVM, GCPhysDst, pvPage, X86_PAGE_SIZE, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
591	if (RT_SUCCESS(rc))
592	{
593	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
594	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
595	Log5(("NEM GPA mapped/set: %RGp %s (was %s)\n", GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState]));
596	return VINF_SUCCESS;
597	}
598	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
599	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst));
600	return VERR_NEM_INIT_FAILED;
601	}
602	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
603	return rc;
604	}
605
606	if (fPageProt & NEM_PAGE_PROT_READ)
607	{
608	const void *pvPage;
609	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
610	if (RT_SUCCESS(rc))
611	{
612	rc = nemR3DarwinMap(pVM, GCPhysDst, (void *)pvPage, X86_PAGE_SIZE, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE);
613	if (RT_SUCCESS(rc))
614	{
615	*pu2State = NEM_DARWIN_PAGE_STATE_READABLE;
616	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
617	Log5(("NEM GPA mapped/set: %RGp %s (was %s)\n", GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState]));
618	return VINF_SUCCESS;
619	}
620	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
621	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
622	return VERR_NEM_INIT_FAILED;
623	}
624	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
625	return rc;
626	}
627
628	/* We already unmapped it above. */
629	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
630	return VINF_SUCCESS;
631	}
632
633
634	#ifdef LOG_ENABLED
635	/**
636	* Logs the current CPU state.
637	*/
638	static void nemR3DarwinLogState(PVMCC pVM, PVMCPUCC pVCpu)
639	{
640	if (LogIs3Enabled())
641	{
642	#if 0
643	char szRegs[4096];
644	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
645	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
646	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
647	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
648	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
649	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
650	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
651	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
652	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
653	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
654	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
655	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
656	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
657	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
658	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
659	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
660	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
661	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
662	" efer=%016VR{efer}\n"
663	" pat=%016VR{pat}\n"
664	" sf_mask=%016VR{sf_mask}\n"
665	"krnl_gs_base=%016VR{krnl_gs_base}\n"
666	" lstar=%016VR{lstar}\n"
667	" star=%016VR{star} cstar=%016VR{cstar}\n"
668	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
669	);
670
671	char szInstr[256];
672	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
673	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
674	szInstr, sizeof(szInstr), NULL);
675	Log3(("%s%s\n", szRegs, szInstr));
676	#else
677	RT_NOREF(pVM, pVCpu);
678	#endif
679	}
680	}
681	#endif /* LOG_ENABLED */
682
683
684	DECLINLINE(int) nemR3DarwinReadVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t *pData)
685	{
686	uint64_t u64Data;
687	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
688	if (RT_LIKELY(hrc == HV_SUCCESS))
689	{
690	*pData = (uint16_t)u64Data;
691	return VINF_SUCCESS;
692	}
693
694	return nemR3DarwinHvSts2Rc(hrc);
695	}
696
697
698	DECLINLINE(int) nemR3DarwinReadVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t *pData)
699	{
700	uint64_t u64Data;
701	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
702	if (RT_LIKELY(hrc == HV_SUCCESS))
703	{
704	*pData = (uint32_t)u64Data;
705	return VINF_SUCCESS;
706	}
707
708	return nemR3DarwinHvSts2Rc(hrc);
709	}
710
711
712	DECLINLINE(int) nemR3DarwinReadVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t *pData)
713	{
714	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, pData);
715	if (RT_LIKELY(hrc == HV_SUCCESS))
716	return VINF_SUCCESS;
717
718	return nemR3DarwinHvSts2Rc(hrc);
719	}
720
721
722	DECLINLINE(int) nemR3DarwinWriteVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t u16Val)
723	{
724	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u16Val);
725	if (RT_LIKELY(hrc == HV_SUCCESS))
726	return VINF_SUCCESS;
727
728	return nemR3DarwinHvSts2Rc(hrc);
729	}
730
731
732	DECLINLINE(int) nemR3DarwinWriteVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t u32Val)
733	{
734	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u32Val);
735	if (RT_LIKELY(hrc == HV_SUCCESS))
736	return VINF_SUCCESS;
737
738	return nemR3DarwinHvSts2Rc(hrc);
739	}
740
741
742	DECLINLINE(int) nemR3DarwinWriteVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t u64Val)
743	{
744	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u64Val);
745	if (RT_LIKELY(hrc == HV_SUCCESS))
746	return VINF_SUCCESS;
747
748	return nemR3DarwinHvSts2Rc(hrc);
749	}
750
751	DECLINLINE(int) nemR3DarwinMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Val)
752	{
753	hv_return_t hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, idMsr, pu64Val);
754	if (RT_LIKELY(hrc == HV_SUCCESS))
755	return VINF_SUCCESS;
756
757	return nemR3DarwinHvSts2Rc(hrc);
758	}
759
760	#if 0 /unused/
761	DECLINLINE(int) nemR3DarwinMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Val)
762	{
763	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, idMsr, u64Val);
764	if (RT_LIKELY(hrc == HV_SUCCESS))
765	return VINF_SUCCESS;
766
767	return nemR3DarwinHvSts2Rc(hrc);
768	}
769	#endif
770
771	static int nemR3DarwinCopyStateFromHv(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
772	{
773	#define READ_GREG(a_GReg, a_Value) \
774	do \
775	{ \
776	hrc = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, (a_GReg), &(a_Value)); \
777	if (RT_LIKELY(hrc == HV_SUCCESS)) \
778	{ /* likely */ } \
779	else \
780	return VERR_INTERNAL_ERROR; \
781	} while(0)
782	#define READ_VMCS_FIELD(a_Field, a_Value) \
783	do \
784	{ \
785	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &(a_Value)); \
786	if (RT_LIKELY(hrc == HV_SUCCESS)) \
787	{ /* likely */ } \
788	else \
789	return VERR_INTERNAL_ERROR; \
790	} while(0)
791	#define READ_VMCS16_FIELD(a_Field, a_Value) \
792	do \
793	{ \
794	uint64_t u64Data; \
795	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
796	if (RT_LIKELY(hrc == HV_SUCCESS)) \
797	{ (a_Value) = (uint16_t)u64Data; } \
798	else \
799	return VERR_INTERNAL_ERROR; \
800	} while(0)
801	#define READ_VMCS32_FIELD(a_Field, a_Value) \
802	do \
803	{ \
804	uint64_t u64Data; \
805	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
806	if (RT_LIKELY(hrc == HV_SUCCESS)) \
807	{ (a_Value) = (uint32_t)u64Data; } \
808	else \
809	return VERR_INTERNAL_ERROR; \
810	} while(0)
811	#define READ_MSR(a_Msr, a_Value) \
812	do \
813	{ \
814	hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, (a_Msr), &(a_Value)); \
815	if (RT_LIKELY(hrc == HV_SUCCESS)) \
816	{ /* likely */ } \
817	else \
818	AssertFailedReturn(VERR_INTERNAL_ERROR); \
819	} while(0)
820
821	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateImport, x);
822
823	RT_NOREF(pVM);
824	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
825
826	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
827	vmxHCImportGuestIntrState(pVCpu, &pVCpu->nem.s.VmcsInfo);
828
829	/* GPRs */
830	hv_return_t hrc;
831	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
832	{
833	if (fWhat & CPUMCTX_EXTRN_RAX)
834	READ_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
835	if (fWhat & CPUMCTX_EXTRN_RCX)
836	READ_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
837	if (fWhat & CPUMCTX_EXTRN_RDX)
838	READ_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
839	if (fWhat & CPUMCTX_EXTRN_RBX)
840	READ_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
841	if (fWhat & CPUMCTX_EXTRN_RSP)
842	READ_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
843	if (fWhat & CPUMCTX_EXTRN_RBP)
844	READ_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
845	if (fWhat & CPUMCTX_EXTRN_RSI)
846	READ_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
847	if (fWhat & CPUMCTX_EXTRN_RDI)
848	READ_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
849	if (fWhat & CPUMCTX_EXTRN_R8_R15)
850	{
851	READ_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
852	READ_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
853	READ_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
854	READ_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
855	READ_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
856	READ_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
857	READ_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
858	READ_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
859	}
860	}
861
862	/* RIP & Flags */
863	if (fWhat & CPUMCTX_EXTRN_RIP)
864	READ_GREG(HV_X86_RIP, pVCpu->cpum.GstCtx.rip);
865	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
866	READ_GREG(HV_X86_RFLAGS, pVCpu->cpum.GstCtx.rflags.u);
867
868	/* Segments */
869	#define READ_SEG(a_SReg, a_enmName) \
870	do { \
871	READ_VMCS16_FIELD(VMX_VMCS16_GUEST_ ## a_enmName ## _SEL, (a_SReg).Sel); \
872	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _LIMIT, (a_SReg).u32Limit); \
873	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _ACCESS_RIGHTS, (a_SReg).Attr.u); \
874	READ_VMCS_FIELD(VMX_VMCS_GUEST_ ## a_enmName ## _BASE, (a_SReg).u64Base); \
875	(a_SReg).ValidSel = (a_SReg).Sel; \
876	} while (0)
877	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
878	{
879	if (fWhat & CPUMCTX_EXTRN_ES)
880	READ_SEG(pVCpu->cpum.GstCtx.es, ES);
881	if (fWhat & CPUMCTX_EXTRN_CS)
882	READ_SEG(pVCpu->cpum.GstCtx.cs, CS);
883	if (fWhat & CPUMCTX_EXTRN_SS)
884	READ_SEG(pVCpu->cpum.GstCtx.ss, SS);
885	if (fWhat & CPUMCTX_EXTRN_DS)
886	READ_SEG(pVCpu->cpum.GstCtx.ds, DS);
887	if (fWhat & CPUMCTX_EXTRN_FS)
888	READ_SEG(pVCpu->cpum.GstCtx.fs, FS);
889	if (fWhat & CPUMCTX_EXTRN_GS)
890	READ_SEG(pVCpu->cpum.GstCtx.gs, GS);
891	}
892
893	/* Descriptor tables and the task segment. */
894	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
895	{
896	if (fWhat & CPUMCTX_EXTRN_LDTR)
897	READ_SEG(pVCpu->cpum.GstCtx.ldtr, LDTR);
898
899	if (fWhat & CPUMCTX_EXTRN_TR)
900	{
901	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
902	avoid to trigger sanity assertions around the code, always fix this. */
903	READ_SEG(pVCpu->cpum.GstCtx.tr, TR);
904	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
905	{
906	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
907	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
908	break;
909	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
910	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
911	break;
912	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
913	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
914	break;
915	}
916	}
917	if (fWhat & CPUMCTX_EXTRN_IDTR)
918	{
919	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_IDTR_LIMIT, pVCpu->cpum.GstCtx.idtr.cbIdt);
920	READ_VMCS_FIELD(VMX_VMCS_GUEST_IDTR_BASE, pVCpu->cpum.GstCtx.idtr.pIdt);
921	}
922	if (fWhat & CPUMCTX_EXTRN_GDTR)
923	{
924	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_GDTR_LIMIT, pVCpu->cpum.GstCtx.gdtr.cbGdt);
925	READ_VMCS_FIELD(VMX_VMCS_GUEST_GDTR_BASE, pVCpu->cpum.GstCtx.gdtr.pGdt);
926	}
927	}
928
929	/* Control registers. */
930	bool fMaybeChangedMode = false;
931	bool fUpdateCr3 = false;
932	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
933	{
934	uint64_t u64CrTmp = 0;
935
936	if (fWhat & CPUMCTX_EXTRN_CR0)
937	{
938	READ_GREG(HV_X86_CR0, u64CrTmp);
939	if (pVCpu->cpum.GstCtx.cr0 != u64CrTmp)
940	{
941	CPUMSetGuestCR0(pVCpu, u64CrTmp);
942	fMaybeChangedMode = true;
943	}
944	}
945	if (fWhat & CPUMCTX_EXTRN_CR2)
946	READ_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
947	if (fWhat & CPUMCTX_EXTRN_CR3)
948	{
949	READ_GREG(HV_X86_CR3, u64CrTmp);
950	if (pVCpu->cpum.GstCtx.cr3 != u64CrTmp)
951	{
952	CPUMSetGuestCR3(pVCpu, u64CrTmp);
953	fUpdateCr3 = true;
954	}
955	}
956	if (fWhat & CPUMCTX_EXTRN_CR4)
957	{
958	READ_GREG(HV_X86_CR4, u64CrTmp);
959	u64CrTmp &= ~VMX_V_CR4_FIXED0;
960
961	if (pVCpu->cpum.GstCtx.cr4 != u64CrTmp)
962	{
963	CPUMSetGuestCR4(pVCpu, u64CrTmp);
964	fMaybeChangedMode = true;
965	}
966	}
967	}
968
969	#if 0 /* Always done. */
970	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
971	{
972	uint64_t u64Cr8 = 0;
973
974	READ_GREG(HV_X86_TPR, u64Cr8);
975	APICSetTpr(pVCpu, u64Cr8 << 4);
976	}
977	#endif
978
979	if (fWhat & CPUMCTX_EXTRN_XCRx)
980	READ_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
981
982	/* Debug registers. */
983	if (fWhat & CPUMCTX_EXTRN_DR7)
984	{
985	uint64_t u64Dr7;
986	READ_GREG(HV_X86_DR7, u64Dr7);
987	if (pVCpu->cpum.GstCtx.dr[7] != u64Dr7)
988	CPUMSetGuestDR7(pVCpu, u64Dr7);
989	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
990	}
991	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
992	{
993	uint64_t u64DrTmp;
994
995	READ_GREG(HV_X86_DR0, u64DrTmp);
996	if (pVCpu->cpum.GstCtx.dr[0] != u64DrTmp)
997	CPUMSetGuestDR0(pVCpu, u64DrTmp);
998	READ_GREG(HV_X86_DR1, u64DrTmp);
999	if (pVCpu->cpum.GstCtx.dr[1] != u64DrTmp)
1000	CPUMSetGuestDR1(pVCpu, u64DrTmp);
1001	READ_GREG(HV_X86_DR2, u64DrTmp);
1002	if (pVCpu->cpum.GstCtx.dr[2] != u64DrTmp)
1003	CPUMSetGuestDR2(pVCpu, u64DrTmp);
1004	READ_GREG(HV_X86_DR3, u64DrTmp);
1005	if (pVCpu->cpum.GstCtx.dr[3] != u64DrTmp)
1006	CPUMSetGuestDR3(pVCpu, u64DrTmp);
1007	}
1008	if (fWhat & CPUMCTX_EXTRN_DR6)
1009	{
1010	uint64_t u64Dr6;
1011	READ_GREG(HV_X86_DR6, u64Dr6);
1012	if (pVCpu->cpum.GstCtx.dr[6] != u64Dr6)
1013	CPUMSetGuestDR6(pVCpu, u64Dr6);
1014	}
1015
1016	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
1017	{
1018	hrc = hv_vcpu_read_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1019	if (hrc == HV_SUCCESS)
1020	{ /* likely */ }
1021	else
1022	{
1023	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1024	return nemR3DarwinHvSts2Rc(hrc);
1025	}
1026	}
1027
1028	/* MSRs */
1029	if (fWhat & CPUMCTX_EXTRN_EFER)
1030	{
1031	uint64_t u64Efer;
1032
1033	READ_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, u64Efer);
1034	if (u64Efer != pVCpu->cpum.GstCtx.msrEFER)
1035	{
1036	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, u64Efer));
1037	if ((u64Efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1038	PGMNotifyNxeChanged(pVCpu, RT_BOOL(u64Efer & MSR_K6_EFER_NXE));
1039	pVCpu->cpum.GstCtx.msrEFER = u64Efer;
1040	fMaybeChangedMode = true;
1041	}
1042	}
1043
1044	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1045	READ_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1046	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1047	{
1048	uint64_t u64Tmp;
1049	READ_MSR(MSR_IA32_SYSENTER_EIP, u64Tmp);
1050	pVCpu->cpum.GstCtx.SysEnter.eip = u64Tmp;
1051	READ_MSR(MSR_IA32_SYSENTER_ESP, u64Tmp);
1052	pVCpu->cpum.GstCtx.SysEnter.esp = u64Tmp;
1053	READ_MSR(MSR_IA32_SYSENTER_CS, u64Tmp);
1054	pVCpu->cpum.GstCtx.SysEnter.cs = u64Tmp;
1055	}
1056	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1057	{
1058	READ_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1059	READ_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1060	READ_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1061	READ_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1062	}
1063	#if 0
1064	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1065	{
1066	Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
1067	const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
1068	if (aValues[iReg].Reg64 != uOldBase)
1069	{
1070	Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
1071	pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
1072	int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
1073	AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
1074	}
1075	iReg++;
1076
1077	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
1078	#if 0 /def LOG_ENABLED/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
1079	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
1080	#endif
1081	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1082	GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType, WHvX64RegisterMsrMtrrDefType, "MSR MTRR_DEF_TYPE");
1083	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
1084	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
1085	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
1086	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000, WHvX64RegisterMsrMtrrFix4kC0000, "MSR MTRR_FIX_4K_C0000");
1087	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000, WHvX64RegisterMsrMtrrFix4kC8000, "MSR MTRR_FIX_4K_C8000");
1088	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000, WHvX64RegisterMsrMtrrFix4kD0000, "MSR MTRR_FIX_4K_D0000");
1089	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000, WHvX64RegisterMsrMtrrFix4kD8000, "MSR MTRR_FIX_4K_D8000");
1090	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000, WHvX64RegisterMsrMtrrFix4kE0000, "MSR MTRR_FIX_4K_E0000");
1091	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000, WHvX64RegisterMsrMtrrFix4kE8000, "MSR MTRR_FIX_4K_E8000");
1092	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000, WHvX64RegisterMsrMtrrFix4kF0000, "MSR MTRR_FIX_4K_F0000");
1093	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000, WHvX64RegisterMsrMtrrFix4kF8000, "MSR MTRR_FIX_4K_F8000");
1094	GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX");
1095	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
1096	}
1097	#endif
1098
1099	/* Almost done, just update extrn flags and maybe change PGM mode. */
1100	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1101	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1102	pVCpu->cpum.GstCtx.fExtrn = 0;
1103
1104	#ifdef LOG_ENABLED
1105	nemR3DarwinLogState(pVM, pVCpu);
1106	#endif
1107
1108	/* Typical. */
1109	if (!fMaybeChangedMode && !fUpdateCr3)
1110	{
1111	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1112	return VINF_SUCCESS;
1113	}
1114
1115	/*
1116	* Slow.
1117	*/
1118	if (fMaybeChangedMode)
1119	{
1120	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
1121	false /* fForce */);
1122	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1123	}
1124
1125	if (fUpdateCr3)
1126	{
1127	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
1128	if (rc == VINF_SUCCESS)
1129	{ /* likely */ }
1130	else
1131	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1132	}
1133
1134	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1135
1136	return VINF_SUCCESS;
1137	#undef READ_GREG
1138	#undef READ_VMCS_FIELD
1139	#undef READ_VMCS32_FIELD
1140	#undef READ_SEG
1141	#undef READ_MSR
1142	}
1143
1144
1145	/**
1146	* State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
1147	* and nemHCWinHandleMemoryAccessPageCheckerCallback.
1148	*/
1149	typedef struct NEMHCDARWINHMACPCCSTATE
1150	{
1151	/** Input: Write access. */
1152	bool fWriteAccess;
1153	/** Output: Set if we did something. */
1154	bool fDidSomething;
1155	/** Output: Set it we should resume. */
1156	bool fCanResume;
1157	} NEMHCDARWINHMACPCCSTATE;
1158
1159	/**
1160	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1161	* Worker for nemR3WinHandleMemoryAccess; pvUser points to a
1162	* NEMHCDARWINHMACPCCSTATE structure. }
1163	*/
1164	static DECLCALLBACK(int)
1165	nemR3DarwinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1166	{
1167	NEMHCDARWINHMACPCCSTATE pState = (NEMHCDARWINHMACPCCSTATE )pvUser;
1168	pState->fDidSomething = false;
1169	pState->fCanResume = false;
1170
1171	uint8_t u2State = pInfo->u2NemState;
1172
1173	/*
1174	* Consolidate current page state with actual page protection and access type.
1175	* We don't really consider downgrades here, as they shouldn't happen.
1176	*/
1177	int rc;
1178	switch (u2State)
1179	{
1180	case NEM_DARWIN_PAGE_STATE_UNMAPPED:
1181	case NEM_DARWIN_PAGE_STATE_NOT_SET:
1182	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1183	{
1184	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1185	return VINF_SUCCESS;
1186	}
1187
1188	/* Don't bother remapping it if it's a write request to a non-writable page. */
1189	if ( pState->fWriteAccess
1190	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1191	{
1192	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1193	return VINF_SUCCESS;
1194	}
1195
1196	/* Map the page. */
1197	rc = nemHCNativeSetPhysPage(pVM,
1198	pVCpu,
1199	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1200	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1201	pInfo->fNemProt,
1202	&u2State,
1203	true /fBackingState/);
1204	pInfo->u2NemState = u2State;
1205	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1206	GCPhys, g_apszPageStates[u2State], rc));
1207	pState->fDidSomething = true;
1208	pState->fCanResume = true;
1209	return rc;
1210
1211	case NEM_DARWIN_PAGE_STATE_READABLE:
1212	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1213	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1214	{
1215	pState->fCanResume = true;
1216	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1217	return VINF_SUCCESS;
1218	}
1219	break;
1220
1221	case NEM_DARWIN_PAGE_STATE_WRITABLE:
1222	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1223	{
1224	/* We get spurious EPT exit violations when everything is fine (#3a case) but can resume without issues here... */
1225	pState->fCanResume = true;
1226	if (pInfo->u2OldNemState == NEM_DARWIN_PAGE_STATE_WRITABLE)
1227	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
1228	else
1229	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
1230	GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
1231	return VINF_SUCCESS;
1232	}
1233
1234	break;
1235
1236	default:
1237	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1238	}
1239
1240	/*
1241	* Unmap and restart the instruction.
1242	* If this fails, which it does every so often, just unmap everything for now.
1243	*/
1244	rc = nemR3DarwinUnmap(pVM, GCPhys, X86_PAGE_SIZE);
1245	if (RT_SUCCESS(rc))
1246	{
1247	pState->fDidSomething = true;
1248	pState->fCanResume = true;
1249	pInfo->u2NemState = NEM_DARWIN_PAGE_STATE_UNMAPPED;
1250	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
1251	Log5(("NEM GPA unmapped/exit: %RGp (was %s)\n", GCPhys, g_apszPageStates[u2State]));
1252	return VINF_SUCCESS;
1253	}
1254	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
1255	LogRel(("nemR3DarwinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s rc=%Rrc\n",
1256	GCPhys, g_apszPageStates[u2State], rc));
1257	return VERR_NEM_UNMAP_PAGES_FAILED;
1258	}
1259
1260
1261	DECL_FORCE_INLINE(bool) vmxHCShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1262	{
1263	RT_NOREF(pVCpu, pVmxTransient);
1264	return true;
1265	}
1266
1267
1268	DECL_FORCE_INLINE(bool) nemR3DarwinIsUnrestrictedGuest(PCVMCC pVM)
1269	{
1270	RT_NOREF(pVM);
1271	return true;
1272	}
1273
1274
1275	DECL_FORCE_INLINE(bool) nemR3DarwinIsNestedPaging(PCVMCC pVM)
1276	{
1277	RT_NOREF(pVM);
1278	return true;
1279	}
1280
1281
1282	DECL_FORCE_INLINE(bool) nemR3DarwinIsPreemptTimerUsed(PCVMCC pVM)
1283	{
1284	RT_NOREF(pVM);
1285	return false;
1286	}
1287
1288
1289	#if 0 /* unused */
1290	DECL_FORCE_INLINE(bool) nemR3DarwinIsVmxLbr(PCVMCC pVM)
1291	{
1292	RT_NOREF(pVM);
1293	return false;
1294	}
1295	#endif
1296
1297
1298	/*
1299	* Instantiate the code we share with ring-0.
1300	*/
1301	#define IN_NEM_DARWIN
1302	//#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
1303	//#define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
1304	#define VCPU_2_VMXSTATE(a_pVCpu) (a_pVCpu)->nem.s
1305	#define VCPU_2_VMXSTATS(a_pVCpu) (*(a_pVCpu)->nem.s.pVmxStats)
1306
1307	#define VM_IS_VMX_UNRESTRICTED_GUEST(a_pVM) nemR3DarwinIsUnrestrictedGuest((a_pVM))
1308	#define VM_IS_VMX_NESTED_PAGING(a_pVM) nemR3DarwinIsNestedPaging((a_pVM))
1309	#define VM_IS_VMX_PREEMPT_TIMER_USED(a_pVM) nemR3DarwinIsPreemptTimerUsed((a_pVM))
1310	#define VM_IS_VMX_LBR(a_pVM) nemR3DarwinIsVmxLbr((a_pVM))
1311
1312	#define VMX_VMCS_WRITE_16(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs16((a_pVCpu), (a_FieldEnc), (a_Val))
1313	#define VMX_VMCS_WRITE_32(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs32((a_pVCpu), (a_FieldEnc), (a_Val))
1314	#define VMX_VMCS_WRITE_64(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1315	#define VMX_VMCS_WRITE_NW(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1316
1317	#define VMX_VMCS_READ_16(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs16((a_pVCpu), (a_FieldEnc), (a_pVal))
1318	#define VMX_VMCS_READ_32(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs32((a_pVCpu), (a_FieldEnc), (a_pVal))
1319	#define VMX_VMCS_READ_64(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1320	#define VMX_VMCS_READ_NW(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1321
1322	#include "../VMMAll/VMXAllTemplate.cpp.h"
1323
1324	#undef VMX_VMCS_WRITE_16
1325	#undef VMX_VMCS_WRITE_32
1326	#undef VMX_VMCS_WRITE_64
1327	#undef VMX_VMCS_WRITE_NW
1328
1329	#undef VMX_VMCS_READ_16
1330	#undef VMX_VMCS_READ_32
1331	#undef VMX_VMCS_READ_64
1332	#undef VMX_VMCS_READ_NW
1333
1334	#undef VM_IS_VMX_PREEMPT_TIMER_USED
1335	#undef VM_IS_VMX_NESTED_PAGING
1336	#undef VM_IS_VMX_UNRESTRICTED_GUEST
1337	#undef VCPU_2_VMXSTATS
1338	#undef VCPU_2_VMXSTATE
1339
1340
1341	/**
1342	* Exports the guest GP registers to HV for execution.
1343	*
1344	* @returns VBox status code.
1345	* @param pVCpu The cross context virtual CPU structure of the
1346	* calling EMT.
1347	*/
1348	static int nemR3DarwinExportGuestGprs(PVMCPUCC pVCpu)
1349	{
1350	#define WRITE_GREG(a_GReg, a_Value) \
1351	do \
1352	{ \
1353	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1354	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1355	{ /* likely */ } \
1356	else \
1357	return VERR_INTERNAL_ERROR; \
1358	} while(0)
1359
1360	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->nem.s.fCtxChanged);
1361	if (fCtxChanged & HM_CHANGED_GUEST_GPRS_MASK)
1362	{
1363	if (fCtxChanged & HM_CHANGED_GUEST_RAX)
1364	WRITE_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
1365	if (fCtxChanged & HM_CHANGED_GUEST_RCX)
1366	WRITE_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
1367	if (fCtxChanged & HM_CHANGED_GUEST_RDX)
1368	WRITE_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
1369	if (fCtxChanged & HM_CHANGED_GUEST_RBX)
1370	WRITE_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
1371	if (fCtxChanged & HM_CHANGED_GUEST_RSP)
1372	WRITE_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
1373	if (fCtxChanged & HM_CHANGED_GUEST_RBP)
1374	WRITE_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
1375	if (fCtxChanged & HM_CHANGED_GUEST_RSI)
1376	WRITE_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
1377	if (fCtxChanged & HM_CHANGED_GUEST_RDI)
1378	WRITE_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
1379	if (fCtxChanged & HM_CHANGED_GUEST_R8_R15)
1380	{
1381	WRITE_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
1382	WRITE_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
1383	WRITE_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
1384	WRITE_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
1385	WRITE_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
1386	WRITE_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
1387	WRITE_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
1388	WRITE_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
1389	}
1390
1391	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_GPRS_MASK);
1392	}
1393
1394	if (fCtxChanged & HM_CHANGED_GUEST_CR2)
1395	{
1396	WRITE_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
1397	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_CR2);
1398	}
1399
1400	return VINF_SUCCESS;
1401	#undef WRITE_GREG
1402	}
1403
1404
1405	/**
1406	* Converts the given CPUM externalized bitmask to the appropriate HM changed bitmask.
1407	*
1408	* @returns Bitmask of HM changed flags.
1409	* @param fCpumExtrn The CPUM extern bitmask.
1410	*/
1411	static uint64_t nemR3DarwinCpumExtrnToHmChanged(uint64_t fCpumExtrn)
1412	{
1413	uint64_t fHmChanged = 0;
1414
1415	/* Invert to gt a mask of things which are kept in CPUM. */
1416	uint64_t fCpumIntern = ~fCpumExtrn;
1417
1418	if (fCpumIntern & CPUMCTX_EXTRN_GPRS_MASK)
1419	{
1420	if (fCpumIntern & CPUMCTX_EXTRN_RAX)
1421	fHmChanged \|= HM_CHANGED_GUEST_RAX;
1422	if (fCpumIntern & CPUMCTX_EXTRN_RCX)
1423	fHmChanged \|= HM_CHANGED_GUEST_RCX;
1424	if (fCpumIntern & CPUMCTX_EXTRN_RDX)
1425	fHmChanged \|= HM_CHANGED_GUEST_RDX;
1426	if (fCpumIntern & CPUMCTX_EXTRN_RBX)
1427	fHmChanged \|= HM_CHANGED_GUEST_RBX;
1428	if (fCpumIntern & CPUMCTX_EXTRN_RSP)
1429	fHmChanged \|= HM_CHANGED_GUEST_RSP;
1430	if (fCpumIntern & CPUMCTX_EXTRN_RBP)
1431	fHmChanged \|= HM_CHANGED_GUEST_RBP;
1432	if (fCpumIntern & CPUMCTX_EXTRN_RSI)
1433	fHmChanged \|= HM_CHANGED_GUEST_RSI;
1434	if (fCpumIntern & CPUMCTX_EXTRN_RDI)
1435	fHmChanged \|= HM_CHANGED_GUEST_RDI;
1436	if (fCpumIntern & CPUMCTX_EXTRN_R8_R15)
1437	fHmChanged \|= HM_CHANGED_GUEST_R8_R15;
1438	}
1439
1440	/* RIP & Flags */
1441	if (fCpumIntern & CPUMCTX_EXTRN_RIP)
1442	fHmChanged \|= HM_CHANGED_GUEST_RIP;
1443	if (fCpumIntern & CPUMCTX_EXTRN_RFLAGS)
1444	fHmChanged \|= HM_CHANGED_GUEST_RFLAGS;
1445
1446	/* Segments */
1447	if (fCpumIntern & CPUMCTX_EXTRN_SREG_MASK)
1448	{
1449	if (fCpumIntern & CPUMCTX_EXTRN_ES)
1450	fHmChanged \|= HM_CHANGED_GUEST_ES;
1451	if (fCpumIntern & CPUMCTX_EXTRN_CS)
1452	fHmChanged \|= HM_CHANGED_GUEST_CS;
1453	if (fCpumIntern & CPUMCTX_EXTRN_SS)
1454	fHmChanged \|= HM_CHANGED_GUEST_SS;
1455	if (fCpumIntern & CPUMCTX_EXTRN_DS)
1456	fHmChanged \|= HM_CHANGED_GUEST_DS;
1457	if (fCpumIntern & CPUMCTX_EXTRN_FS)
1458	fHmChanged \|= HM_CHANGED_GUEST_FS;
1459	if (fCpumIntern & CPUMCTX_EXTRN_GS)
1460	fHmChanged \|= HM_CHANGED_GUEST_GS;
1461	}
1462
1463	/* Descriptor tables & task segment. */
1464	if (fCpumIntern & CPUMCTX_EXTRN_TABLE_MASK)
1465	{
1466	if (fCpumIntern & CPUMCTX_EXTRN_LDTR)
1467	fHmChanged \|= HM_CHANGED_GUEST_LDTR;
1468	if (fCpumIntern & CPUMCTX_EXTRN_TR)
1469	fHmChanged \|= HM_CHANGED_GUEST_TR;
1470	if (fCpumIntern & CPUMCTX_EXTRN_IDTR)
1471	fHmChanged \|= HM_CHANGED_GUEST_IDTR;
1472	if (fCpumIntern & CPUMCTX_EXTRN_GDTR)
1473	fHmChanged \|= HM_CHANGED_GUEST_GDTR;
1474	}
1475
1476	/* Control registers. */
1477	if (fCpumIntern & CPUMCTX_EXTRN_CR_MASK)
1478	{
1479	if (fCpumIntern & CPUMCTX_EXTRN_CR0)
1480	fHmChanged \|= HM_CHANGED_GUEST_CR0;
1481	if (fCpumIntern & CPUMCTX_EXTRN_CR2)
1482	fHmChanged \|= HM_CHANGED_GUEST_CR2;
1483	if (fCpumIntern & CPUMCTX_EXTRN_CR3)
1484	fHmChanged \|= HM_CHANGED_GUEST_CR3;
1485	if (fCpumIntern & CPUMCTX_EXTRN_CR4)
1486	fHmChanged \|= HM_CHANGED_GUEST_CR4;
1487	}
1488	if (fCpumIntern & CPUMCTX_EXTRN_APIC_TPR)
1489	fHmChanged \|= HM_CHANGED_GUEST_APIC_TPR;
1490
1491	/* Debug registers. */
1492	if (fCpumIntern & CPUMCTX_EXTRN_DR0_DR3)
1493	fHmChanged \|= HM_CHANGED_GUEST_DR0_DR3;
1494	if (fCpumIntern & CPUMCTX_EXTRN_DR6)
1495	fHmChanged \|= HM_CHANGED_GUEST_DR6;
1496	if (fCpumIntern & CPUMCTX_EXTRN_DR7)
1497	fHmChanged \|= HM_CHANGED_GUEST_DR7;
1498
1499	/* Floating point state. */
1500	if (fCpumIntern & CPUMCTX_EXTRN_X87)
1501	fHmChanged \|= HM_CHANGED_GUEST_X87;
1502	if (fCpumIntern & CPUMCTX_EXTRN_SSE_AVX)
1503	fHmChanged \|= HM_CHANGED_GUEST_SSE_AVX;
1504	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_XSAVE)
1505	fHmChanged \|= HM_CHANGED_GUEST_OTHER_XSAVE;
1506	if (fCpumIntern & CPUMCTX_EXTRN_XCRx)
1507	fHmChanged \|= HM_CHANGED_GUEST_XCRx;
1508
1509	/* MSRs */
1510	if (fCpumIntern & CPUMCTX_EXTRN_EFER)
1511	fHmChanged \|= HM_CHANGED_GUEST_EFER_MSR;
1512	if (fCpumIntern & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1513	fHmChanged \|= HM_CHANGED_GUEST_KERNEL_GS_BASE;
1514	if (fCpumIntern & CPUMCTX_EXTRN_SYSENTER_MSRS)
1515	fHmChanged \|= HM_CHANGED_GUEST_SYSENTER_MSR_MASK;
1516	if (fCpumIntern & CPUMCTX_EXTRN_SYSCALL_MSRS)
1517	fHmChanged \|= HM_CHANGED_GUEST_SYSCALL_MSRS;
1518	if (fCpumIntern & CPUMCTX_EXTRN_TSC_AUX)
1519	fHmChanged \|= HM_CHANGED_GUEST_TSC_AUX;
1520	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_MSRS)
1521	fHmChanged \|= HM_CHANGED_GUEST_OTHER_MSRS;
1522
1523	return fHmChanged;
1524	}
1525
1526
1527	/**
1528	* Exports the guest state to HV for execution.
1529	*
1530	* @returns VBox status code.
1531	* @param pVM The cross context VM structure.
1532	* @param pVCpu The cross context virtual CPU structure of the
1533	* calling EMT.
1534	* @param pVmxTransient The transient VMX structure.
1535	*/
1536	static int nemR3DarwinExportGuestState(PVMCC pVM, PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1537	{
1538	#define WRITE_GREG(a_GReg, a_Value) \
1539	do \
1540	{ \
1541	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1542	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1543	{ /* likely */ } \
1544	else \
1545	return VERR_INTERNAL_ERROR; \
1546	} while(0)
1547	#define WRITE_VMCS_FIELD(a_Field, a_Value) \
1548	do \
1549	{ \
1550	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), (a_Value)); \
1551	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1552	{ /* likely */ } \
1553	else \
1554	return VERR_INTERNAL_ERROR; \
1555	} while(0)
1556	#define WRITE_MSR(a_Msr, a_Value) \
1557	do \
1558	{ \
1559	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, (a_Msr), (a_Value)); \
1560	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1561	{ /* likely */ } \
1562	else \
1563	AssertFailedReturn(VERR_INTERNAL_ERROR); \
1564	} while(0)
1565
1566	RT_NOREF(pVM);
1567
1568	#ifdef LOG_ENABLED
1569	nemR3DarwinLogState(pVM, pVCpu);
1570	#endif
1571
1572	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateExport, x);
1573
1574	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL;
1575	if (!fWhat)
1576	return VINF_SUCCESS;
1577
1578	pVCpu->nem.s.fCtxChanged \|= nemR3DarwinCpumExtrnToHmChanged(pVCpu->cpum.GstCtx.fExtrn);
1579
1580	int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
1581	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1582
1583	rc = nemR3DarwinExportGuestGprs(pVCpu);
1584	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1585
1586	rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
1587	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1588
1589	VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
1590	if (rcStrict == VINF_SUCCESS)
1591	{ /* likely */ }
1592	else
1593	{
1594	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
1595	return VBOXSTRICTRC_VAL(rcStrict);
1596	}
1597
1598	vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
1599	vmxHCExportGuestRip(pVCpu);
1600	//vmxHCExportGuestRsp(pVCpu);
1601	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
1602
1603	rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
1604	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1605
1606	if (fWhat & CPUMCTX_EXTRN_XCRx)
1607	{
1608	WRITE_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
1609	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_XCRx);
1610	}
1611
1612	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1613	{
1614	Assert(pVCpu->nem.s.fCtxChanged & HM_CHANGED_GUEST_APIC_TPR);
1615	vmxHCExportGuestApicTpr(pVCpu, pVmxTransient);
1616
1617	rc = APICGetTpr(pVCpu, &pVmxTransient->u8GuestTpr, NULL /pfPending/, NULL /pu8PendingIntr/);
1618	AssertRC(rc);
1619
1620	WRITE_GREG(HV_X86_TPR, pVmxTransient->u8GuestTpr);
1621	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1622	}
1623
1624	/* Debug registers. */
1625	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1626	{
1627	WRITE_GREG(HV_X86_DR0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
1628	WRITE_GREG(HV_X86_DR1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
1629	WRITE_GREG(HV_X86_DR2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
1630	WRITE_GREG(HV_X86_DR3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
1631	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR0_DR3);
1632	}
1633	if (fWhat & CPUMCTX_EXTRN_DR6)
1634	{
1635	WRITE_GREG(HV_X86_DR6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
1636	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR6);
1637	}
1638	if (fWhat & CPUMCTX_EXTRN_DR7)
1639	{
1640	WRITE_GREG(HV_X86_DR7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
1641	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR7);
1642	}
1643
1644	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1645	{
1646	hv_return_t hrc = hv_vcpu_write_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1647	if (hrc == HV_SUCCESS)
1648	{ /* likely */ }
1649	else
1650	return nemR3DarwinHvSts2Rc(hrc);
1651
1652	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~(HM_CHANGED_GUEST_X87 \| HM_CHANGED_GUEST_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE));
1653	}
1654
1655	/* MSRs */
1656	if (fWhat & CPUMCTX_EXTRN_EFER)
1657	{
1658	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, pVCpu->cpum.GstCtx.msrEFER);
1659	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
1660	}
1661	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1662	{
1663	WRITE_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1664	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_KERNEL_GS_BASE);
1665	}
1666	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1667	{
1668	WRITE_MSR(MSR_IA32_SYSENTER_CS, pVCpu->cpum.GstCtx.SysEnter.cs);
1669	WRITE_MSR(MSR_IA32_SYSENTER_EIP, pVCpu->cpum.GstCtx.SysEnter.eip);
1670	WRITE_MSR(MSR_IA32_SYSENTER_ESP, pVCpu->cpum.GstCtx.SysEnter.esp);
1671	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
1672	}
1673	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1674	{
1675	WRITE_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1676	WRITE_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1677	WRITE_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1678	WRITE_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1679	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSCALL_MSRS);
1680	}
1681	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1682	{
1683	#if 0
1684	hv_return_t hrc = hv_vmx_vcpu_set_apic_address(pVCpu->nem.s.hVCpuId, APICGetBaseMsrNoCheck(pVCpu) & PAGE_BASE_GC_MASK);
1685	if (RT_UNLIKELY(hrc != HV_SUCCESS))
1686	return nemR3DarwinHvSts2Rc(hrc);
1687	#endif
1688
1689	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
1690
1691	#if 0
1692	ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
1693	#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
1694	ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
1695	#endif
1696	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1697	ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
1698	ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
1699	ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
1700	ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
1701	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000, pCtxMsrs->msr.MtrrFix4K_C0000);
1702	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000, pCtxMsrs->msr.MtrrFix4K_C8000);
1703	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000, pCtxMsrs->msr.MtrrFix4K_D0000);
1704	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000, pCtxMsrs->msr.MtrrFix4K_D8000);
1705	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000, pCtxMsrs->msr.MtrrFix4K_E0000);
1706	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000, pCtxMsrs->msr.MtrrFix4K_E8000);
1707	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000, pCtxMsrs->msr.MtrrFix4K_F0000);
1708	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000, pCtxMsrs->msr.MtrrFix4K_F8000);
1709	ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
1710	#if 0 /** @todo these registers aren't available? Might explain something.. .*/
1711	const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
1712	if (enmCpuVendor != CPUMCPUVENDOR_AMD)
1713	{
1714	ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
1715	ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
1716	}
1717	#endif
1718	#endif
1719	}
1720
1721	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0 /MSR_IA32_DEBUGCTL_LBR/);
1722
1723	hv_vcpu_invalidate_tlb(pVCpu->nem.s.hVCpuId);
1724	hv_vcpu_flush(pVCpu->nem.s.hVCpuId);
1725
1726	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_NEM;
1727
1728	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
1729	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~(
1730	HM_CHANGED_GUEST_TSC_AUX
1731	\| HM_CHANGED_GUEST_HWVIRT
1732	\| HM_CHANGED_VMX_GUEST_AUTO_MSRS
1733	\| HM_CHANGED_VMX_GUEST_LAZY_MSRS
1734	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
1735
1736	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateExport, x);
1737	return VINF_SUCCESS;
1738	#undef WRITE_GREG
1739	#undef WRITE_VMCS_FIELD
1740	}
1741
1742
1743	/**
1744	* Handles an exit from hv_vcpu_run().
1745	*
1746	* @returns VBox strict status code.
1747	* @param pVM The cross context VM structure.
1748	* @param pVCpu The cross context virtual CPU structure of the
1749	* calling EMT.
1750	* @param pVmxTransient The transient VMX structure.
1751	*/
1752	static VBOXSTRICTRC nemR3DarwinHandleExit(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1753	{
1754	uint32_t uExitReason;
1755	int rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
1756	AssertRC(rc);
1757	pVmxTransient->fVmcsFieldsRead = 0;
1758	pVmxTransient->fIsNestedGuest = false;
1759	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
1760	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
1761
1762	if (RT_UNLIKELY(pVmxTransient->fVMEntryFailed))
1763	AssertLogRelMsgFailedReturn(("Running guest failed for CPU #%u: %#x %u\n",
1764	pVCpu->idCpu, pVmxTransient->uExitReason, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
1765	VERR_NEM_IPE_0);
1766
1767	/** @todo Only copy the state on demand (the R0 VT-x code saves some stuff unconditionally and the VMX template assumes that
1768	* when handling exits). */
1769	rc = nemR3DarwinCopyStateFromHv(pVM, pVCpu, CPUMCTX_EXTRN_ALL);
1770	AssertRCReturn(rc, rc);
1771
1772	STAM_COUNTER_INC(&pVCpu->nem.s.pVmxStats->aStatExitReason[pVmxTransient->uExitReason & MASK_EXITREASON_STAT]);
1773	STAM_REL_COUNTER_INC(&pVCpu->nem.s.pVmxStats->StatExitAll);
1774
1775	#ifndef HMVMX_USE_FUNCTION_TABLE
1776	return vmxHCHandleExit(pVCpu, pVmxTransient);
1777	#else
1778	return g_aVMExitHandlers[pVmxTransient->uExitReason].pfn(pVCpu, pVmxTransient);
1779	#endif
1780	}
1781
1782
1783	/**
1784	* Worker for nemR3NativeInit that loads the Hypervisor.framework shared library.
1785	*
1786	* @returns VBox status code.
1787	* @param fForced Whether the HMForced flag is set and we should
1788	* fail if we cannot initialize.
1789	* @param pErrInfo Where to always return error info.
1790	*/
1791	static int nemR3DarwinLoadHv(bool fForced, PRTERRINFO pErrInfo)
1792	{
1793	RTLDRMOD hMod = NIL_RTLDRMOD;
1794	static const char *s_pszHvPath = "/System/Library/Frameworks/Hypervisor.framework/Hypervisor";
1795
1796	int rc = RTLdrLoadEx(s_pszHvPath, &hMod, RTLDRLOAD_FLAGS_NO_UNLOAD \| RTLDRLOAD_FLAGS_NO_SUFFIX, pErrInfo);
1797	if (RT_SUCCESS(rc))
1798	{
1799	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
1800	{
1801	int rc2 = RTLdrGetSymbol(hMod, g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
1802	if (RT_SUCCESS(rc2))
1803	{
1804	if (g_aImports[i].fOptional)
1805	LogRel(("NEM: info: Found optional import Hypervisor!%s.\n",
1806	g_aImports[i].pszName));
1807	}
1808	else
1809	{
1810	*g_aImports[i].ppfn = NULL;
1811
1812	LogRel(("NEM: %s: Failed to import Hypervisor!%s: %Rrc\n",
1813	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
1814	g_aImports[i].pszName, rc2));
1815	if (!g_aImports[i].fOptional)
1816	{
1817	if (RTErrInfoIsSet(pErrInfo))
1818	RTErrInfoAddF(pErrInfo, rc2, ", Hypervisor!%s", g_aImports[i].pszName);
1819	else
1820	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: Hypervisor!%s", g_aImports[i].pszName);
1821	Assert(RT_FAILURE(rc));
1822	}
1823	}
1824	}
1825	if (RT_SUCCESS(rc))
1826	{
1827	Assert(!RTErrInfoIsSet(pErrInfo));
1828	}
1829
1830	RTLdrClose(hMod);
1831	}
1832	else
1833	{
1834	RTErrInfoAddF(pErrInfo, rc, "Failed to load Hypervisor.framwork: %s: %Rrc", s_pszHvPath, rc);
1835	rc = VERR_NEM_INIT_FAILED;
1836	}
1837
1838	return rc;
1839	}
1840
1841
1842	/**
1843	* Read and initialize the global capabilities supported by this CPU.
1844	*
1845	* @returns VBox status code.
1846	*/
1847	static int nemR3DarwinCapsInit(void)
1848	{
1849	RT_ZERO(g_HmMsrs);
1850
1851	hv_return_t hrc = hv_vmx_read_capability(HV_VMX_CAP_PINBASED, &g_HmMsrs.u.vmx.PinCtls.u);
1852	if (hrc == HV_SUCCESS)
1853	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED, &g_HmMsrs.u.vmx.ProcCtls.u);
1854	if (hrc == HV_SUCCESS)
1855	hrc = hv_vmx_read_capability(HV_VMX_CAP_ENTRY, &g_HmMsrs.u.vmx.EntryCtls.u);
1856	if (hrc == HV_SUCCESS)
1857	hrc = hv_vmx_read_capability(HV_VMX_CAP_EXIT, &g_HmMsrs.u.vmx.ExitCtls.u);
1858	if (hrc == HV_SUCCESS)
1859	{
1860	hrc = hv_vmx_read_capability(HV_VMX_CAP_BASIC, &g_HmMsrs.u.vmx.u64Basic);
1861	if (hrc == HV_SUCCESS)
1862	{
1863	if (hrc == HV_SUCCESS)
1864	hrc = hv_vmx_read_capability(HV_VMX_CAP_MISC, &g_HmMsrs.u.vmx.u64Misc);
1865	if (hrc == HV_SUCCESS)
1866	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED0, &g_HmMsrs.u.vmx.u64Cr0Fixed0);
1867	if (hrc == HV_SUCCESS)
1868	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED1, &g_HmMsrs.u.vmx.u64Cr0Fixed1);
1869	if (hrc == HV_SUCCESS)
1870	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED0, &g_HmMsrs.u.vmx.u64Cr4Fixed0);
1871	if (hrc == HV_SUCCESS)
1872	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED1, &g_HmMsrs.u.vmx.u64Cr4Fixed1);
1873	if (hrc == HV_SUCCESS)
1874	hrc = hv_vmx_read_capability(HV_VMX_CAP_VMCS_ENUM, &g_HmMsrs.u.vmx.u64VmcsEnum);
1875	if ( hrc == HV_SUCCESS
1876	&& RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_TRUE_CTLS))
1877	{
1878	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PINBASED, &g_HmMsrs.u.vmx.TruePinCtls.u);
1879	if (hrc == HV_SUCCESS)
1880	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PROCBASED, &g_HmMsrs.u.vmx.TrueProcCtls.u);
1881	if (hrc == HV_SUCCESS)
1882	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_ENTRY, &g_HmMsrs.u.vmx.TrueEntryCtls.u);
1883	if (hrc == HV_SUCCESS)
1884	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_EXIT, &g_HmMsrs.u.vmx.TrueExitCtls.u);
1885	}
1886	}
1887	else
1888	{
1889	/* Likely running on anything < 11.0 (BigSur) so provide some sensible defaults. */
1890	g_HmMsrs.u.vmx.u64Cr0Fixed0 = 0x80000021;
1891	g_HmMsrs.u.vmx.u64Cr0Fixed1 = 0xffffffff;
1892	g_HmMsrs.u.vmx.u64Cr4Fixed0 = 0x2000;
1893	g_HmMsrs.u.vmx.u64Cr4Fixed1 = 0x1767ff;
1894	hrc = HV_SUCCESS;
1895	}
1896	}
1897
1898	if ( hrc == HV_SUCCESS
1899	&& g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1900	{
1901	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED2, &g_HmMsrs.u.vmx.ProcCtls2.u);
1902
1903	if ( hrc == HV_SUCCESS
1904	&& g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & (VMX_PROC_CTLS2_EPT \| VMX_PROC_CTLS2_VPID))
1905	{
1906	hrc = hv_vmx_read_capability(HV_VMX_CAP_EPT_VPID_CAP, &g_HmMsrs.u.vmx.u64EptVpidCaps);
1907	if (hrc != HV_SUCCESS)
1908	hrc = HV_SUCCESS; /* Probably just outdated OS. */
1909	}
1910
1911	g_HmMsrs.u.vmx.u64VmFunc = 0; /* No way to read that on macOS. */
1912	}
1913
1914	if (hrc == HV_SUCCESS)
1915	{
1916	/*
1917	* Check for EFER swapping support.
1918	*/
1919	g_fHmVmxSupportsVmcsEfer = true; //(g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
1920	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1921	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
1922	}
1923
1924	return nemR3DarwinHvSts2Rc(hrc);
1925	}
1926
1927
1928	/**
1929	* Sets up pin-based VM-execution controls in the VMCS.
1930	*
1931	* @returns VBox status code.
1932	* @param pVCpu The cross context virtual CPU structure.
1933	* @param pVmcsInfo The VMCS info. object.
1934	*/
1935	static int nemR3DarwinVmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1936	{
1937	//PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1938	uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
1939	uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1940
1941	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
1942	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
1943
1944	#if 0 /** @todo Use preemption timer */
1945	/* Enable the VMX-preemption timer. */
1946	if (pVM->hmr0.s.vmx.fUsePreemptTimer)
1947	{
1948	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
1949	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
1950	}
1951
1952	/* Enable posted-interrupt processing. */
1953	if (pVM->hm.s.fPostedIntrs)
1954	{
1955	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
1956	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
1957	fVal \|= VMX_PIN_CTLS_POSTED_INT;
1958	}
1959	#endif
1960
1961	if ((fVal & fZap) != fVal)
1962	{
1963	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1964	g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
1965	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1966	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1967	}
1968
1969	/* Commit it to the VMCS and update our cache. */
1970	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, fVal);
1971	AssertRC(rc);
1972	pVmcsInfo->u32PinCtls = fVal;
1973
1974	return VINF_SUCCESS;
1975	}
1976
1977
1978	/**
1979	* Sets up secondary processor-based VM-execution controls in the VMCS.
1980	*
1981	* @returns VBox status code.
1982	* @param pVCpu The cross context virtual CPU structure.
1983	* @param pVmcsInfo The VMCS info. object.
1984	*/
1985	static int nemR3DarwinVmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1986	{
1987	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1988	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
1989	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1990
1991	/* WBINVD causes a VM-exit. */
1992	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
1993	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
1994
1995	/* Enable the INVPCID instruction if we expose it to the guest and is supported
1996	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
1997	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
1998	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
1999	fVal \|= VMX_PROC_CTLS2_INVPCID;
2000
2001	#if 0 /** @todo */
2002	/* Enable VPID. */
2003	if (pVM->hmr0.s.vmx.fVpid)
2004	fVal \|= VMX_PROC_CTLS2_VPID;
2005
2006	if (pVM->hm.s.fVirtApicRegs)
2007	{
2008	/* Enable APIC-register virtualization. */
2009	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2010	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2011
2012	/* Enable virtual-interrupt delivery. */
2013	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2014	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2015	}
2016
2017	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
2018	where the TPR shadow resides. */
2019	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2020	* done dynamically. */
2021	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2022	{
2023	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2024	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
2025	}
2026	#endif
2027
2028	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
2029	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
2030	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
2031	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
2032	fVal \|= VMX_PROC_CTLS2_RDTSCP;
2033
2034	#if 0
2035	/* Enable Pause-Loop exiting. */
2036	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
2037	&& pVM->hm.s.vmx.cPleGapTicks
2038	&& pVM->hm.s.vmx.cPleWindowTicks)
2039	{
2040	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2041
2042	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
2043	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
2044	}
2045	#endif
2046
2047	if ((fVal & fZap) != fVal)
2048	{
2049	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2050	g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
2051	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2052	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2053	}
2054
2055	/* Commit it to the VMCS and update our cache. */
2056	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2057	AssertRC(rc);
2058	pVmcsInfo->u32ProcCtls2 = fVal;
2059
2060	return VINF_SUCCESS;
2061	}
2062
2063
2064	/**
2065	* Enables native access for the given MSR.
2066	*
2067	* @returns VBox status code.
2068	* @param pVCpu The cross context virtual CPU structure.
2069	* @param idMsr The MSR to enable native access for.
2070	*/
2071	static int nemR3DarwinMsrSetNative(PVMCPUCC pVCpu, uint32_t idMsr)
2072	{
2073	hv_return_t hrc = hv_vcpu_enable_native_msr(pVCpu->nem.s.hVCpuId, idMsr, true /enable/);
2074	if (hrc == HV_SUCCESS)
2075	return VINF_SUCCESS;
2076
2077	return nemR3DarwinHvSts2Rc(hrc);
2078	}
2079
2080
2081	/**
2082	* Sets up the MSR permissions which don't change through the lifetime of the VM.
2083	*
2084	* @returns VBox status code.
2085	* @param pVCpu The cross context virtual CPU structure.
2086	* @param pVmcsInfo The VMCS info. object.
2087	*/
2088	static int nemR3DarwinSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2089	{
2090	RT_NOREF(pVmcsInfo);
2091
2092	/*
2093	* The guest can access the following MSRs (read, write) without causing
2094	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2095	*/
2096	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2097	int rc;
2098	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_CS); AssertRCReturn(rc, rc);
2099	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_ESP); AssertRCReturn(rc, rc);
2100	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_EIP); AssertRCReturn(rc, rc);
2101	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_GS_BASE); AssertRCReturn(rc, rc);
2102	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_FS_BASE); AssertRCReturn(rc, rc);
2103
2104	/*
2105	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2106	* associated with then. We never need to intercept access (writes need to be
2107	* executed without causing a VM-exit, reads will #GP fault anyway).
2108	*
2109	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2110	* read/write them. We swap the guest/host MSR value using the
2111	* auto-load/store MSR area.
2112	*/
2113	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2114	{
2115	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_PRED_CMD);
2116	AssertRCReturn(rc, rc);
2117	}
2118	#if 0 /* Doesn't work. */
2119	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2120	{
2121	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_FLUSH_CMD);
2122	AssertRCReturn(rc, rc);
2123	}
2124	#endif
2125	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2126	{
2127	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SPEC_CTRL);
2128	AssertRCReturn(rc, rc);
2129	}
2130
2131	/*
2132	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2133	* required for 64-bit guests.
2134	*/
2135	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_LSTAR); AssertRCReturn(rc, rc);
2136	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K6_STAR); AssertRCReturn(rc, rc);
2137	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_SF_MASK); AssertRCReturn(rc, rc);
2138	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_KERNEL_GS_BASE); AssertRCReturn(rc, rc);
2139
2140	/* Required for enabling the RDTSCP instruction. */
2141	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_TSC_AUX); AssertRCReturn(rc, rc);
2142
2143	return VINF_SUCCESS;
2144	}
2145
2146
2147	/**
2148	* Sets up processor-based VM-execution controls in the VMCS.
2149	*
2150	* @returns VBox status code.
2151	* @param pVCpu The cross context virtual CPU structure.
2152	* @param pVmcsInfo The VMCS info. object.
2153	*/
2154	static int nemR3DarwinVmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2155	{
2156	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2157	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2158
2159	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2160	// \| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2161	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2162	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2163	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2164	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2165	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2166
2167	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2168	if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2169	\|\| (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2170	{
2171	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2172	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2173	}
2174
2175	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2176	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2177	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2178
2179	if ((fVal & fZap) != fVal)
2180	{
2181	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2182	g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2183	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2184	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2185	}
2186
2187	/* Commit it to the VMCS and update our cache. */
2188	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2189	AssertRC(rc);
2190	pVmcsInfo->u32ProcCtls = fVal;
2191
2192	/* Set up MSR permissions that don't change through the lifetime of the VM. */
2193	rc = nemR3DarwinSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2194	AssertRCReturn(rc, rc);
2195
2196	/*
2197	* Set up secondary processor-based VM-execution controls
2198	* (we assume the CPU to always support it as we rely on unrestricted guest execution support).
2199	*/
2200	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
2201	return nemR3DarwinVmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2202	}
2203
2204
2205	/**
2206	* Sets up miscellaneous (everything other than Pin, Processor and secondary
2207	* Processor-based VM-execution) control fields in the VMCS.
2208	*
2209	* @returns VBox status code.
2210	* @param pVCpu The cross context virtual CPU structure.
2211	* @param pVmcsInfo The VMCS info. object.
2212	*/
2213	static int nemR3DarwinVmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2214	{
2215	int rc = VINF_SUCCESS;
2216	//rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo); TODO
2217	if (RT_SUCCESS(rc))
2218	{
2219	uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2220	uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2221
2222	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2223	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2224
2225	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2226	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2227
2228	#if 0 /** @todo */
2229	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fLbr)
2230	{
2231	rc = VMXWriteVmcsNw(VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2232	AssertRC(rc);
2233	}
2234	#endif
2235	return VINF_SUCCESS;
2236	}
2237	else
2238	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2239	return rc;
2240	}
2241
2242
2243	/**
2244	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2245	*
2246	* We shall setup those exception intercepts that don't change during the
2247	* lifetime of the VM here. The rest are done dynamically while loading the
2248	* guest state.
2249	*
2250	* @param pVCpu The cross context virtual CPU structure.
2251	* @param pVmcsInfo The VMCS info. object.
2252	*/
2253	static void nemR3DarwinVmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2254	{
2255	/*
2256	* The following exceptions are always intercepted:
2257	*
2258	* #AC - To prevent the guest from hanging the CPU and for dealing with
2259	* split-lock detecting host configs.
2260	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2261	* recursive #DBs can cause a CPU hang.
2262	*/
2263	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2264	\| RT_BIT(X86_XCPT_DB);
2265
2266	/* Commit it to the VMCS. */
2267	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2268	AssertRC(rc);
2269
2270	/* Update our cache of the exception bitmap. */
2271	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2272	}
2273
2274
2275	/**
2276	* Initialize the VMCS information field for the given vCPU.
2277	*
2278	* @returns VBox status code.
2279	* @param pVCpu The cross context virtual CPU structure of the
2280	* calling EMT.
2281	*/
2282	static int nemR3DarwinInitVmcs(PVMCPU pVCpu)
2283	{
2284	int rc = nemR3DarwinVmxSetupVmcsPinCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2285	if (RT_SUCCESS(rc))
2286	{
2287	rc = nemR3DarwinVmxSetupVmcsProcCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2288	if (RT_SUCCESS(rc))
2289	{
2290	rc = nemR3DarwinVmxSetupVmcsMiscCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2291	if (RT_SUCCESS(rc))
2292	{
2293	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &pVCpu->nem.s.VmcsInfo.u32EntryCtls);
2294	if (RT_SUCCESS(rc))
2295	{
2296	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_EXIT, &pVCpu->nem.s.VmcsInfo.u32ExitCtls);
2297	if (RT_SUCCESS(rc))
2298	{
2299	nemR3DarwinVmxSetupVmcsXcptBitmap(pVCpu, &pVCpu->nem.s.VmcsInfo);
2300	return VINF_SUCCESS;
2301	}
2302	else
2303	LogRelFunc(("Failed to read the exit controls. rc=%Rrc\n", rc));
2304	}
2305	else
2306	LogRelFunc(("Failed to read the entry controls. rc=%Rrc\n", rc));
2307	}
2308	else
2309	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
2310	}
2311	else
2312	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
2313	}
2314	else
2315	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
2316
2317	return rc;
2318	}
2319
2320
2321	/**
2322	* Registers statistics for the given vCPU.
2323	*
2324	* @returns VBox status code.
2325	* @param pVM The cross context VM structure.
2326	* @param idCpu The CPU ID.
2327	* @param pNemCpu The NEM CPU structure.
2328	*/
2329	static int nemR3DarwinStatisticsRegister(PVM pVM, VMCPUID idCpu, PNEMCPU pNemCpu)
2330	{
2331	#define NEM_REG_STAT(a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szNmFmt, a_szDesc) do { \
2332	int rc = STAMR3RegisterF(pVM, a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szDesc, a_szNmFmt, idCpu); \
2333	AssertRC(rc); \
2334	} while (0)
2335	#define NEM_REG_PROFILE(a_pVar, a_szNmFmt, a_szDesc) \
2336	NEM_REG_STAT(a_pVar, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, a_szNmFmt, a_szDesc)
2337	#define NEM_REG_COUNTER(a, b, desc) NEM_REG_STAT(a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, b, desc)
2338
2339	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR0Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR0", "CR0 read.");
2340	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR2Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR2", "CR2 read.");
2341	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR3Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR3", "CR3 read.");
2342	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR4Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR4", "CR4 read.");
2343	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR8Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR8", "CR8 read.");
2344	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR0Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR0", "CR0 write.");
2345	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR2Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR2", "CR2 write.");
2346	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR3Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR3", "CR3 write.");
2347	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR4Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR4", "CR4 write.");
2348	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR8Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR8", "CR8 write.");
2349
2350	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitAll, "/NEM/CPU%u/Exit/All", "Total exits (including nested-guest exits).");
2351
2352	#ifdef VBOX_WITH_STATISTICS
2353	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateImport, "/NEM/CPU%u/ImportGuestState", "Profiling of importing guest state from hardware after VM-exit.");
2354	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateExport, "/NEM/CPU%u/ExportGuestState", "Profiling of exporting guest state from hardware after VM-exit.");
2355
2356	for (int j = 0; j < MAX_EXITREASON_STAT; j++)
2357	{
2358	const char *pszExitName = HMGetVmxExitName(j);
2359	if (pszExitName)
2360	{
2361	int rc = STAMR3RegisterF(pVM, &pNemCpu->pVmxStats->aStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
2362	STAMUNIT_OCCURENCES, pszExitName, "/NEM/CPU%u/Exit/Reason/%02x", idCpu, j);
2363	AssertRCReturn(rc, rc);
2364	}
2365	}
2366	#endif
2367
2368	return VINF_SUCCESS;
2369
2370	#undef NEM_REG_COUNTER
2371	#undef NEM_REG_PROFILE
2372	#undef NEM_REG_STAT
2373	}
2374
2375
2376	/**
2377	* Try initialize the native API.
2378	*
2379	* This may only do part of the job, more can be done in
2380	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
2381	*
2382	* @returns VBox status code.
2383	* @param pVM The cross context VM structure.
2384	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
2385	* the latter we'll fail if we cannot initialize.
2386	* @param fForced Whether the HMForced flag is set and we should
2387	* fail if we cannot initialize.
2388	*/
2389	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
2390	{
2391	AssertReturn(!pVM->nem.s.fCreatedVm, VERR_WRONG_ORDER);
2392
2393	/*
2394	* Some state init.
2395	*/
2396
2397	/*
2398	* Error state.
2399	* The error message will be non-empty on failure and 'rc' will be set too.
2400	*/
2401	RTERRINFOSTATIC ErrInfo;
2402	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
2403	int rc = nemR3DarwinLoadHv(fForced, pErrInfo);
2404	if (RT_SUCCESS(rc))
2405	{
2406	hv_return_t hrc = hv_vm_create(HV_VM_DEFAULT);
2407	if (hrc == HV_SUCCESS)
2408	{
2409	if (hv_vm_space_create)
2410	{
2411	hrc = hv_vm_space_create(&pVM->nem.s.uVmAsid);
2412	if (hrc == HV_SUCCESS)
2413	{
2414	LogRel(("NEM: Successfully created ASID: %u\n", pVM->nem.s.uVmAsid));
2415	pVM->nem.s.fCreatedAsid = true;
2416	}
2417	else
2418	LogRel(("NEM: Failed to create ASID for VM (hrc=%#x), continuing...\n", pVM->nem.s.uVmAsid));
2419	}
2420	pVM->nem.s.fCreatedVm = true;
2421
2422	/* Register release statistics */
2423	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2424	{
2425	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
2426	PVMXSTATISTICS pVmxStats = (PVMXSTATISTICS)RTMemAllocZ(sizeof(*pVmxStats));
2427	if (RT_LIKELY(pVmxStats))
2428	{
2429	pNemCpu->pVmxStats = pVmxStats;
2430	rc = nemR3DarwinStatisticsRegister(pVM, idCpu, pNemCpu);
2431	AssertRC(rc);
2432	}
2433	else
2434	{
2435	rc = VERR_NO_MEMORY;
2436	break;
2437	}
2438	}
2439
2440	if (RT_SUCCESS(rc))
2441	{
2442	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
2443	Log(("NEM: Marked active!\n"));
2444	PGMR3EnableNemMode(pVM);
2445	}
2446	}
2447	else
2448	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
2449	"hv_vm_create() failed: %#x", hrc);
2450	}
2451
2452	/*
2453	* We only fail if in forced mode, otherwise just log the complaint and return.
2454	*/
2455	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
2456	if ( (fForced \|\| !fFallback)
2457	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
2458	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
2459
2460	if (RTErrInfoIsSet(pErrInfo))
2461	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
2462	return VINF_SUCCESS;
2463	}
2464
2465
2466	/**
2467	* Worker to create the vCPU handle on the EMT running it later on (as required by HV).
2468	*
2469	* @returns VBox status code
2470	* @param pVM The VM handle.
2471	* @param pVCpu The vCPU handle.
2472	* @param idCpu ID of the CPU to create.
2473	*/
2474	static DECLCALLBACK(int) nemR3DarwinNativeInitVCpuOnEmt(PVM pVM, PVMCPU pVCpu, VMCPUID idCpu)
2475	{
2476	hv_return_t hrc = hv_vcpu_create(&pVCpu->nem.s.hVCpuId, HV_VCPU_DEFAULT);
2477	if (hrc != HV_SUCCESS)
2478	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
2479	"Call to hv_vcpu_create failed on vCPU %u: %#x (%Rrc)", idCpu, hrc, nemR3DarwinHvSts2Rc(hrc));
2480
2481	if (idCpu == 0)
2482	{
2483	/* First call initializs the MSR structure holding the capabilities of the host CPU. */
2484	int rc = nemR3DarwinCapsInit();
2485	AssertRCReturn(rc, rc);
2486	}
2487
2488	int rc = nemR3DarwinInitVmcs(pVCpu);
2489	AssertRCReturn(rc, rc);
2490
2491	if (pVM->nem.s.fCreatedAsid)
2492	{
2493	hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, pVM->nem.s.uVmAsid);
2494	AssertReturn(hrc == HV_SUCCESS, VERR_NEM_VM_CREATE_FAILED);
2495	}
2496
2497	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2498
2499	return VINF_SUCCESS;
2500	}
2501
2502
2503	/**
2504	* Worker to destroy the vCPU handle on the EMT running it later on (as required by HV).
2505	*
2506	* @returns VBox status code
2507	* @param pVCpu The vCPU handle.
2508	*/
2509	static DECLCALLBACK(int) nemR3DarwinNativeTermVCpuOnEmt(PVMCPU pVCpu)
2510	{
2511	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
2512	Assert(hrc == HV_SUCCESS);
2513
2514	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
2515	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
2516	return VINF_SUCCESS;
2517	}
2518
2519
2520	/**
2521	* Worker to setup the TPR shadowing feature if available on the CPU and the VM has an APIC enabled.
2522	*
2523	* @returns VBox status code
2524	* @param pVM The VM handle.
2525	* @param pVCpu The vCPU handle.
2526	*/
2527	static DECLCALLBACK(int) nemR3DarwinNativeInitTprShadowing(PVM pVM, PVMCPU pVCpu)
2528	{
2529	PVMXVMCSINFO pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
2530	uint32_t fVal = pVmcsInfo->u32ProcCtls;
2531
2532	/* Use TPR shadowing if supported by the CPU. */
2533	if ( PDMHasApic(pVM)
2534	&& (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
2535	{
2536	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2537	/* CR8 writes cause a VM-exit based on TPR threshold. */
2538	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2539	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2540	}
2541	else
2542	{
2543	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2544	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2545	}
2546
2547	/* Commit it to the VMCS and update our cache. */
2548	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2549	AssertRC(rc);
2550	pVmcsInfo->u32ProcCtls = fVal;
2551
2552	return VINF_SUCCESS;
2553	}
2554
2555
2556	/**
2557	* This is called after CPUMR3Init is done.
2558	*
2559	* @returns VBox status code.
2560	* @param pVM The VM handle..
2561	*/
2562	int nemR3NativeInitAfterCPUM(PVM pVM)
2563	{
2564	/*
2565	* Validate sanity.
2566	*/
2567	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
2568	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
2569
2570	/*
2571	* Setup the EMTs.
2572	*/
2573	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2574	{
2575	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2576
2577	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitVCpuOnEmt, 3, pVM, pVCpu, idCpu);
2578	if (RT_FAILURE(rc))
2579	{
2580	/* Rollback. */
2581	while (idCpu--)
2582	VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeTermVCpuOnEmt, 1, pVCpu);
2583
2584	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to hv_vcpu_create failed: %Rrc", rc);
2585	}
2586	}
2587
2588	pVM->nem.s.fCreatedEmts = true;
2589	return VINF_SUCCESS;
2590	}
2591
2592
2593	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
2594	{
2595	if (enmWhat == VMINITCOMPLETED_RING3)
2596	{
2597	/* Now that PDM is initialized the APIC state is known in order to enable the TPR shadowing feature on all EMTs. */
2598	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2599	{
2600	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2601
2602	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitTprShadowing, 2, pVM, pVCpu);
2603	if (RT_FAILURE(rc))
2604	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Setting up TPR shadowing failed: %Rrc", rc);
2605	}
2606	}
2607	return VINF_SUCCESS;
2608	}
2609
2610
2611	int nemR3NativeTerm(PVM pVM)
2612	{
2613	/*
2614	* Delete the VM.
2615	*/
2616
2617	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu--)
2618	{
2619	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2620
2621	/*
2622	* Need to do this or hv_vm_space_destroy() fails later on (on 10.15 at least). Could've been documented in
2623	* API reference so I wouldn't have to decompile the kext to find this out but we are talking
2624	* about Apple here unfortunately, API documentation is not their strong suit...
2625	* Would have been of course even better to just automatically drop the address space reference when the vCPU
2626	* gets destroyed.
2627	*/
2628	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
2629	Assert(hrc == HV_SUCCESS);
2630
2631	/*
2632	* Apple's documentation states that the vCPU should be destroyed
2633	* on the thread running the vCPU but as all the other EMTs are gone
2634	* at this point, destroying the VM would hang.
2635	*
2636	* We seem to be at luck here though as destroying apparently works
2637	* from EMT(0) as well.
2638	*/
2639	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
2640	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
2641
2642	if (pVCpu->nem.s.pVmxStats)
2643	{
2644	RTMemFree(pVCpu->nem.s.pVmxStats);
2645	pVCpu->nem.s.pVmxStats = NULL;
2646	}
2647	}
2648
2649	pVM->nem.s.fCreatedEmts = false;
2650
2651	if (pVM->nem.s.fCreatedAsid)
2652	{
2653	hv_return_t hrc = hv_vm_space_destroy(pVM->nem.s.uVmAsid);
2654	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
2655	pVM->nem.s.fCreatedAsid = false;
2656	}
2657
2658	if (pVM->nem.s.fCreatedVm)
2659	{
2660	hv_return_t hrc = hv_vm_destroy();
2661	if (hrc != HV_SUCCESS)
2662	LogRel(("NEM: hv_vm_destroy() failed with %#x\n", hrc));
2663
2664	pVM->nem.s.fCreatedVm = false;
2665	}
2666	return VINF_SUCCESS;
2667	}
2668
2669
2670	/**
2671	* VM reset notification.
2672	*
2673	* @param pVM The cross context VM structure.
2674	*/
2675	void nemR3NativeReset(PVM pVM)
2676	{
2677	RT_NOREF(pVM);
2678	}
2679
2680
2681	/**
2682	* Reset CPU due to INIT IPI or hot (un)plugging.
2683	*
2684	* @param pVCpu The cross context virtual CPU structure of the CPU being
2685	* reset.
2686	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
2687	*/
2688	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
2689	{
2690	RT_NOREF(fInitIpi);
2691	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2692	}
2693
2694
2695	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2696	{
2697	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags));
2698	#ifdef LOG_ENABLED
2699	if (LogIs3Enabled())
2700	nemR3DarwinLogState(pVM, pVCpu);
2701	#endif
2702
2703	/*
2704	* Try switch to NEM runloop state.
2705	*/
2706	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2707	{ /* likely */ }
2708	else
2709	{
2710	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2711	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
2712	return VINF_SUCCESS;
2713	}
2714
2715	/*
2716	* The run loop.
2717	*
2718	* Current approach to state updating to use the sledgehammer and sync
2719	* everything every time. This will be optimized later.
2720	*/
2721
2722	VMXTRANSIENT VmxTransient;
2723	RT_ZERO(VmxTransient);
2724	VmxTransient.pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
2725
2726	const bool fSingleStepping = DBGFIsStepping(pVCpu);
2727	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2728	for (unsigned iLoop = 0;; iLoop++)
2729	{
2730	/*
2731	* Check and process force flag actions, some of which might require us to go back to ring-3.
2732	*/
2733	rcStrict = vmxHCCheckForceFlags(pVCpu, false /fIsNestedGuest/, fSingleStepping);
2734	if (rcStrict == VINF_SUCCESS)
2735	{ /likely / }
2736	else
2737	break;
2738
2739	/*
2740	* Evaluate events to be injected into the guest.
2741	*
2742	* Events in TRPM can be injected without inspecting the guest state.
2743	* If any new events (interrupts/NMI) are pending currently, we try to set up the
2744	* guest to cause a VM-exit the next time they are ready to receive the event.
2745	*/
2746	if (TRPMHasTrap(pVCpu))
2747	vmxHCTrpmTrapToPendingEvent(pVCpu);
2748
2749	uint32_t fIntrState;
2750	rcStrict = vmxHCEvaluatePendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, &fIntrState);
2751
2752	/*
2753	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
2754	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
2755	* also result in triple-faulting the VM.
2756	*
2757	* With nested-guests, the above does not apply since unrestricted guest execution is a
2758	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
2759	*/
2760	rcStrict = vmxHCInjectPendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, fIntrState, fSingleStepping);
2761	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2762	{ /* likely */ }
2763	else
2764	{
2765	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fSingleStepping),
2766	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
2767	break;
2768	}
2769
2770	int rc = nemR3DarwinExportGuestState(pVM, pVCpu, &VmxTransient);
2771	AssertRCReturn(rc, rc);
2772
2773	/*
2774	* Poll timers and run for a bit.
2775	*/
2776	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
2777	* the whole polling job when timers have changed... */
2778	uint64_t offDeltaIgnored;
2779	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
2780	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
2781	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2782	{
2783	LogFlowFunc(("Running vCPU\n"));
2784	pVCpu->nem.s.Event.fPending = false;
2785
2786	TMNotifyStartOfExecution(pVM, pVCpu);
2787
2788	Assert(!pVCpu->nem.s.fCtxChanged);
2789	hv_return_t hrc;
2790	if (hv_vcpu_run_until)
2791	hrc = hv_vcpu_run_until(pVCpu->nem.s.hVCpuId, HV_DEADLINE_FOREVER);
2792	else
2793	hrc = hv_vcpu_run(pVCpu->nem.s.hVCpuId);
2794
2795	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
2796
2797	/*
2798	* Sync the TPR shadow with our APIC state.
2799	*/
2800	if ( !VmxTransient.fIsNestedGuest
2801	&& (pVCpu->nem.s.VmcsInfo.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
2802	{
2803	uint64_t u64Tpr;
2804	hrc = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, HV_X86_TPR, &u64Tpr);
2805	Assert(hrc == HV_SUCCESS);
2806
2807	if (VmxTransient.u8GuestTpr != (uint8_t)u64Tpr)
2808	{
2809	rc = APICSetTpr(pVCpu, (uint8_t)u64Tpr);
2810	AssertRC(rc);
2811	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
2812	}
2813	}
2814
2815	if (hrc == HV_SUCCESS)
2816	{
2817	/*
2818	* Deal with the message.
2819	*/
2820	rcStrict = nemR3DarwinHandleExit(pVM, pVCpu, &VmxTransient);
2821	if (rcStrict == VINF_SUCCESS)
2822	{ /* hopefully likely */ }
2823	else
2824	{
2825	LogFlow(("NEM/%u: breaking: nemR3DarwinHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2826	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2827	break;
2828	}
2829	//Assert(!pVCpu->cpum.GstCtx.fExtrn);
2830	}
2831	else
2832	{
2833	AssertLogRelMsgFailedReturn(("hv_vcpu_run()) failed for CPU #%u: %#x %u\n",
2834	pVCpu->idCpu, hrc, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
2835	VERR_NEM_IPE_0);
2836	}
2837
2838	/*
2839	* If no relevant FFs are pending, loop.
2840	*/
2841	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2842	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2843	continue;
2844
2845	/** @todo Try handle pending flags, not just return to EM loops. Take care
2846	* not to set important RCs here unless we've handled a message. */
2847	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
2848	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
2849	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
2850	}
2851	else
2852	{
2853	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
2854	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
2855	}
2856	break;
2857	} /* the run loop */
2858
2859
2860	/*
2861	* Convert any pending HM events back to TRPM due to premature exits.
2862	*
2863	* This is because execution may continue from IEM and we would need to inject
2864	* the event from there (hence place it back in TRPM).
2865	*/
2866	if (pVCpu->nem.s.Event.fPending)
2867	{
2868	vmxHCPendingEventToTrpmTrap(pVCpu);
2869	Assert(!pVCpu->nem.s.Event.fPending);
2870
2871	/* Clear the events from the VMCS. */
2872	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
2873	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
2874	}
2875
2876
2877	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
2878	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2879
2880	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL))
2881	{
2882	/* Try anticipate what we might need. */
2883	uint64_t fImport = NEM_DARWIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
2884	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
2885	\|\| RT_FAILURE(rcStrict))
2886	fImport = CPUMCTX_EXTRN_ALL;
2887	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
2888	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2889	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
2890
2891	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
2892	{
2893	/* Only import what is external currently. */
2894	int rc2 = nemR3DarwinCopyStateFromHv(pVM, pVCpu, fImport);
2895	if (RT_SUCCESS(rc2))
2896	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
2897	else if (RT_SUCCESS(rcStrict))
2898	rcStrict = rc2;
2899	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
2900	{
2901	pVCpu->cpum.GstCtx.fExtrn = 0;
2902	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2903	}
2904	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
2905	}
2906	else
2907	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2908	}
2909	else
2910	{
2911	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2912	pVCpu->cpum.GstCtx.fExtrn = 0;
2913	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2914	}
2915
2916	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
2917	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
2918	return rcStrict;
2919	}
2920
2921
2922	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
2923	{
2924	NOREF(pVM);
2925	return PGMPhysIsA20Enabled(pVCpu);
2926	}
2927
2928
2929	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
2930	{
2931	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
2932	return false;
2933	}
2934
2935
2936	/**
2937	* Forced flag notification call from VMEmt.h.
2938	*
2939	* This is only called when pVCpu is in the VMCPUSTATE_STARTED_EXEC_NEM state.
2940	*
2941	* @param pVM The cross context VM structure.
2942	* @param pVCpu The cross context virtual CPU structure of the CPU
2943	* to be notified.
2944	* @param fFlags Notification flags, VMNOTIFYFF_FLAGS_XXX.
2945	*/
2946	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
2947	{
2948	LogFlowFunc(("pVM=%p pVCpu=%p fFlags=%#x\n", pVM, pVCpu, fFlags));
2949
2950	RT_NOREF(pVM, fFlags);
2951
2952	hv_return_t hrc = hv_vcpu_interrupt(&pVCpu->nem.s.hVCpuId, 1);
2953	if (hrc != HV_SUCCESS)
2954	LogRel(("NEM: hv_vcpu_interrupt(%u, 1) failed with %#x\n", pVCpu->nem.s.hVCpuId, hrc));
2955	}
2956
2957
2958	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
2959	uint8_t pu2State, uint32_t puNemRange)
2960	{
2961	RT_NOREF(pVM, puNemRange);
2962
2963	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p\n", GCPhys, cb, pvR3));
2964	#if defined(VBOX_WITH_PGM_NEM_MODE)
2965	if (pvR3)
2966	{
2967	int rc = nemR3DarwinMap(pVM, GCPhys, pvR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
2968	if (RT_SUCCESS(rc))
2969	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
2970	else
2971	{
2972	LogRel(("NEMR3NotifyPhysRamRegister: GCPhys=%RGp LB %RGp pvR3=%p rc=%Rrc\n", GCPhys, cb, pvR3, rc));
2973	return VERR_NEM_MAP_PAGES_FAILED;
2974	}
2975	}
2976	return VINF_SUCCESS;
2977	#else
2978	RT_NOREF(pVM, GCPhys, cb, pvR3);
2979	return VERR_NEM_MAP_PAGES_FAILED;
2980	#endif
2981	}
2982
2983
2984	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
2985	{
2986	RT_NOREF(pVM);
2987	return false;
2988	}
2989
2990
2991	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
2992	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
2993	{
2994	RT_NOREF(pVM, puNemRange, pvRam, fFlags);
2995
2996	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d)\n",
2997	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, *pu2State));
2998
2999	#if defined(VBOX_WITH_PGM_NEM_MODE)
3000	/*
3001	* Unmap the RAM we're replacing.
3002	*/
3003	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
3004	{
3005	int rc = nemR3DarwinUnmap(pVM, GCPhys, cb);
3006	if (RT_SUCCESS(rc))
3007	{ /* likely */ }
3008	else if (pvMmio2)
3009	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rc(ignored)\n",
3010	GCPhys, cb, fFlags, rc));
3011	else
3012	{
3013	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
3014	GCPhys, cb, fFlags, rc));
3015	return VERR_NEM_UNMAP_PAGES_FAILED;
3016	}
3017	}
3018
3019	/*
3020	* Map MMIO2 if any.
3021	*/
3022	if (pvMmio2)
3023	{
3024	Assert(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2);
3025	int rc = nemR3DarwinMap(pVM, GCPhys, pvMmio2, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
3026	if (RT_SUCCESS(rc))
3027	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
3028	else
3029	{
3030	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x pvMmio2=%p: Map -> rc=%Rrc\n",
3031	GCPhys, cb, fFlags, pvMmio2, rc));
3032	return VERR_NEM_MAP_PAGES_FAILED;
3033	}
3034	}
3035	else
3036	{
3037	Assert(!(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2));
3038	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3039	}
3040
3041	#else
3042	RT_NOREF(pVM, GCPhys, cb, pvRam, pvMmio2);
3043	*pu2State = (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE) ? UINT8_MAX : NEM_DARWIN_PAGE_STATE_UNMAPPED;
3044	#endif
3045	return VINF_SUCCESS;
3046	}
3047
3048
3049	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
3050	void pvRam, void pvMmio2, uint32_t *puNemRange)
3051	{
3052	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
3053	return VINF_SUCCESS;
3054	}
3055
3056
3057	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
3058	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
3059	{
3060	RT_NOREF(pVM, puNemRange);
3061
3062	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p uNemRange=%#x (%#x)\n",
3063	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
3064
3065	int rc = VINF_SUCCESS;
3066	#if defined(VBOX_WITH_PGM_NEM_MODE)
3067	/*
3068	* Unmap the MMIO2 pages.
3069	*/
3070	/** @todo If we implement aliasing (MMIO2 page aliased into MMIO range),
3071	* we may have more stuff to unmap even in case of pure MMIO... */
3072	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
3073	{
3074	rc = nemR3DarwinUnmap(pVM, GCPhys, cb);
3075	if (RT_FAILURE(rc))
3076	{
3077	LogRel2(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
3078	GCPhys, cb, fFlags, rc));
3079	rc = VERR_NEM_UNMAP_PAGES_FAILED;
3080	}
3081	}
3082
3083	/*
3084	* Restore the RAM we replaced.
3085	*/
3086	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
3087	{
3088	AssertPtr(pvRam);
3089	rc = nemR3DarwinMap(pVM, GCPhys, pvRam, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
3090	if (RT_SUCCESS(rc))
3091	{ /* likely */ }
3092	else
3093	{
3094	LogRel(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp pvMmio2=%p rc=%Rrc\n", GCPhys, cb, pvMmio2, rc));
3095	rc = VERR_NEM_MAP_PAGES_FAILED;
3096	}
3097	if (pu2State)
3098	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
3099	}
3100	/* Mark the pages as unmapped if relevant. */
3101	else if (pu2State)
3102	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3103
3104	RT_NOREF(pvMmio2);
3105	#else
3106	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
3107	if (pu2State)
3108	*pu2State = UINT8_MAX;
3109	rc = VERR_NEM_UNMAP_PAGES_FAILED;
3110	#endif
3111	return rc;
3112	}
3113
3114
3115	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
3116	void *pvBitmap, size_t cbBitmap)
3117	{
3118	RT_NOREF(pVM, GCPhys, cb, uNemRange, pvBitmap, cbBitmap);
3119	AssertFailed();
3120	return VERR_NOT_IMPLEMENTED;
3121	}
3122
3123
3124	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
3125	uint8_t pu2State, uint32_t puNemRange)
3126	{
3127	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
3128
3129	Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
3130	*pu2State = UINT8_MAX;
3131	*puNemRange = 0;
3132	return VINF_SUCCESS;
3133	}
3134
3135
3136	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
3137	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
3138	{
3139	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
3140	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
3141	*pu2State = UINT8_MAX;
3142
3143	#if defined(VBOX_WITH_PGM_NEM_MODE)
3144	/*
3145	* (Re-)map readonly.
3146	*/
3147	AssertPtrReturn(pvPages, VERR_INVALID_POINTER);
3148	int rc = nemR3DarwinMap(pVM, GCPhys, pvPages, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE);
3149	if (RT_SUCCESS(rc))
3150	*pu2State = NEM_DARWIN_PAGE_STATE_READABLE;
3151	else
3152	{
3153	LogRel(("nemR3NativeNotifyPhysRomRegisterLate: GCPhys=%RGp LB %RGp pvPages=%p fFlags=%#x rc=%Rrc\n",
3154	GCPhys, cb, pvPages, fFlags, rc));
3155	return VERR_NEM_MAP_PAGES_FAILED;
3156	}
3157	RT_NOREF(pVM, fFlags, puNemRange);
3158	return VINF_SUCCESS;
3159	#else
3160	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
3161	return VERR_NEM_MAP_PAGES_FAILED;
3162	#endif
3163	}
3164
3165
3166	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
3167	RTR3PTR pvMemR3, uint8_t *pu2State)
3168	{
3169	RT_NOREF(pVM);
3170
3171	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
3172	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
3173
3174	*pu2State = UINT8_MAX;
3175	#if defined(VBOX_WITH_PGM_NEM_MODE)
3176	if (pvMemR3)
3177	{
3178	int rc = nemR3DarwinMap(pVM, GCPhys, pvMemR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
3179	if (RT_SUCCESS(rc))
3180	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
3181	else
3182	AssertLogRelMsgFailed(("NEMHCNotifyHandlerPhysicalDeregister: nemR3DarwinMap(,%p,%RGp,%RGp,) -> %Rrc\n",
3183	pvMemR3, GCPhys, cb, rc));
3184	}
3185	RT_NOREF(enmKind);
3186	#else
3187	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
3188	AssertFailed();
3189	#endif
3190	}
3191
3192
3193	static int nemHCJustUnmapPage(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
3194	{
3195	if (*pu2State <= NEM_DARWIN_PAGE_STATE_UNMAPPED)
3196	{
3197	Log5(("nemHCJustUnmapPage: %RGp == unmapped\n", GCPhysDst));
3198	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3199	return VINF_SUCCESS;
3200	}
3201
3202	int rc = nemR3DarwinUnmap(pVM, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
3203	if (RT_SUCCESS(rc))
3204	{
3205	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
3206	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3207	Log5(("nemHCJustUnmapPage: %RGp => unmapped\n", GCPhysDst));
3208	return VINF_SUCCESS;
3209	}
3210	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
3211	LogRel(("nemHCJustUnmapPage(%RGp): failed! rc=%Rrc\n",
3212	GCPhysDst, rc));
3213	return VERR_NEM_IPE_6;
3214	}
3215
3216
3217	/**
3218	* Called when the A20 state changes.
3219	*
3220	* @param pVCpu The CPU the A20 state changed on.
3221	* @param fEnabled Whether it was enabled (true) or disabled.
3222	*/
3223	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
3224	{
3225	Log(("NEMR3NotifySetA20: fEnabled=%RTbool\n", fEnabled));
3226	RT_NOREF(pVCpu, fEnabled);
3227	}
3228
3229
3230	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
3231	{
3232	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
3233	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
3234	}
3235
3236
3237	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
3238	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
3239	{
3240	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
3241	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
3242	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
3243	}
3244
3245
3246	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
3247	PGMPAGETYPE enmType, uint8_t *pu2State)
3248	{
3249	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3250	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3251	RT_NOREF(HCPhys, fPageProt, enmType);
3252
3253	return nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3254	}
3255
3256
3257	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
3258	PGMPAGETYPE enmType, uint8_t *pu2State)
3259	{
3260	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3261	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3262	RT_NOREF(HCPhys, pvR3, fPageProt, enmType)
3263
3264	nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3265	}
3266
3267
3268	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
3269	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
3270	{
3271	Log5(("NEMHCNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3272	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
3273	RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType);
3274
3275	nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3276	}
3277
3278
3279	/**
3280	* Interface for importing state on demand (used by IEM).
3281	*
3282	* @returns VBox status code.
3283	* @param pVCpu The cross context CPU structure.
3284	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3285	*/
3286	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
3287	{
3288	LogFlowFunc(("pVCpu=%p fWhat=%RX64\n", pVCpu, fWhat));
3289	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
3290
3291	return nemR3DarwinCopyStateFromHv(pVCpu->pVMR3, pVCpu, fWhat);
3292	}
3293
3294
3295	/**
3296	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
3297	*
3298	* @returns VBox status code.
3299	* @param pVCpu The cross context CPU structure.
3300	* @param pcTicks Where to return the CPU tick count.
3301	* @param puAux Where to return the TSC_AUX register value.
3302	*/
3303	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
3304	{
3305	LogFlowFunc(("pVCpu=%p pcTicks=%RX64 puAux=%RX32\n", pVCpu, pcTicks, puAux));
3306	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
3307
3308	int rc = nemR3DarwinMsrRead(pVCpu, MSR_IA32_TSC, pcTicks);
3309	if ( RT_SUCCESS(rc)
3310	&& puAux)
3311	{
3312	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX)
3313	{
3314	/** @todo Why the heck is puAux a uint32_t?. */
3315	uint64_t u64Aux;
3316	rc = nemR3DarwinMsrRead(pVCpu, MSR_K8_TSC_AUX, &u64Aux);
3317	if (RT_SUCCESS(rc))
3318	*puAux = (uint32_t)u64Aux;
3319	}
3320	else
3321	*puAux = CPUMGetGuestTscAux(pVCpu);
3322	}
3323
3324	return rc;
3325	}
3326
3327
3328	/**
3329	* Resumes CPU clock (TSC) on all virtual CPUs.
3330	*
3331	* This is called by TM when the VM is started, restored, resumed or similar.
3332	*
3333	* @returns VBox status code.
3334	* @param pVM The cross context VM structure.
3335	* @param pVCpu The cross context CPU structure of the calling EMT.
3336	* @param uPausedTscValue The TSC value at the time of pausing.
3337	*/
3338	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
3339	{
3340	LogFlowFunc(("pVM=%p pVCpu=%p uPausedTscValue=%RX64\n", pVCpu, uPausedTscValue));
3341	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
3342	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
3343
3344	hv_return_t hrc = hv_vm_sync_tsc(uPausedTscValue);
3345	if (RT_LIKELY(hrc == HV_SUCCESS))
3346	{
3347	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_TSC_AUX);
3348	return VINF_SUCCESS;
3349	}
3350
3351	return nemR3DarwinHvSts2Rc(hrc);
3352	}
3353
3354
3355	/**
3356	* Returns features supported by the NEM backend.
3357	*
3358	* @returns Flags of features supported by the native NEM backend.
3359	* @param pVM The cross context VM structure.
3360	*/
3361	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
3362	{
3363	RT_NOREF(pVM);
3364	/*
3365	* Apple's Hypervisor.framework is not supported if the CPU doesn't support nested paging
3366	* and unrestricted guest execution support so we can safely return these flags here always.
3367	*/
3368	return NEM_FEAT_F_NESTED_PAGING \| NEM_FEAT_F_FULL_GST_EXEC \| NEM_FEAT_F_XSAVE_XRSTOR;
3369	}
3370
3371
3372	/** @page pg_nem_darwin NEM/darwin - Native Execution Manager, macOS.
3373	*
3374	* @todo Add notes as the implementation progresses...
3375	*/
3376

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-darwin.cpp@ 93115

Download in other formats: