HWACCMR0.cpp@ 36236

Last change on this file since 36236 was 36236, checked in by vboxsync, 14 years ago
HWACCMR0: CPU bounds check. CPUMR0: updated message.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 69.4 KB

Line
1	/* $Id: HWACCMR0.cpp 36236 2011-03-09 17:27:22Z vboxsync $ */
2	/** @file
3	* HWACCM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <VBox/vmm/hwaccm.h>
24	#include <VBox/vmm/pgm.h>
25	#include "HWACCMInternal.h"
26	#include <VBox/vmm/vm.h>
27	#include <VBox/x86.h>
28	#include <VBox/vmm/hwacc_vmx.h>
29	#include <VBox/vmm/hwacc_svm.h>
30	#include <VBox/err.h>
31	#include <VBox/log.h>
32	#include <iprt/assert.h>
33	#include <iprt/asm.h>
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/cpuset.h>
36	#include <iprt/memobj.h>
37	#include <iprt/param.h>
38	#include <iprt/power.h>
39	#include <iprt/string.h>
40	#include <iprt/thread.h>
41	#include "HWVMXR0.h"
42	#include "HWSVMR0.h"
43
44	/*******************************************************************************
45	* Internal Functions *
46	*******************************************************************************/
47	static DECLCALLBACK(void) hwaccmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
48	static DECLCALLBACK(void) hwaccmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
49	static DECLCALLBACK(void) HWACCMR0InitCPU(RTCPUID idCpu, void pvUser1, void pvUser2);
50	static int hwaccmR0CheckCpuRcArray(int paRc, unsigned cErrorCodes, RTCPUID pidCpu);
51	static bool hwaccmR0IsSubjectToVmxPreemptionTimerErratum(void);
52	static DECLCALLBACK(void) hwaccmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser);
53
54	/*******************************************************************************
55	* Global Variables *
56	*******************************************************************************/
57
58	static struct
59	{
60	HWACCM_CPUINFO aCpuInfo[RTCPUSET_MAX_CPUS];
61
62	/** Ring 0 handlers for VT-x and AMD-V. */
63	DECLR0CALLBACKMEMBER(int, pfnEnterSession,(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu));
64	DECLR0CALLBACKMEMBER(int, pfnLeaveSession,(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx));
65	DECLR0CALLBACKMEMBER(int, pfnSaveHostState,(PVM pVM, PVMCPU pVCpu));
66	DECLR0CALLBACKMEMBER(int, pfnLoadGuestState,(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx));
67	DECLR0CALLBACKMEMBER(int, pfnRunGuestCode,(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx));
68	DECLR0CALLBACKMEMBER(int, pfnEnableCpu, (PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys));
69	DECLR0CALLBACKMEMBER(int, pfnDisableCpu, (PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys));
70	DECLR0CALLBACKMEMBER(int, pfnInitVM, (PVM pVM));
71	DECLR0CALLBACKMEMBER(int, pfnTermVM, (PVM pVM));
72	DECLR0CALLBACKMEMBER(int, pfnSetupVM, (PVM pVM));
73
74	/** Maximum ASID allowed. */
75	uint32_t uMaxASID;
76
77	struct
78	{
79	/** Set to by us to indicate VMX is supported by the CPU. */
80	bool fSupported;
81	/** Whether we're using SUPR0EnableVTx or not. */
82	bool fUsingSUPR0EnableVTx;
83	/** Whether we're using the preemption timer or not. */
84	bool fUsePreemptTimer;
85	/** The shift mask employed by the VMX-Preemption timer. */
86	uint8_t cPreemptTimerShift;
87
88	/** Host CR4 value (set by ring-0 VMX init) */
89	uint64_t hostCR4;
90
91	/** Host EFER value (set by ring-0 VMX init) */
92	uint64_t hostEFER;
93
94	/** VMX MSR values */
95	struct
96	{
97	uint64_t feature_ctrl;
98	uint64_t vmx_basic_info;
99	VMX_CAPABILITY vmx_pin_ctls;
100	VMX_CAPABILITY vmx_proc_ctls;
101	VMX_CAPABILITY vmx_proc_ctls2;
102	VMX_CAPABILITY vmx_exit;
103	VMX_CAPABILITY vmx_entry;
104	uint64_t vmx_misc;
105	uint64_t vmx_cr0_fixed0;
106	uint64_t vmx_cr0_fixed1;
107	uint64_t vmx_cr4_fixed0;
108	uint64_t vmx_cr4_fixed1;
109	uint64_t vmx_vmcs_enum;
110	uint64_t vmx_eptcaps;
111	} msr;
112	/* Last instruction error */
113	uint32_t ulLastInstrError;
114	} vmx;
115	struct
116	{
117	/* HWCR msr (for diagnostics) */
118	uint64_t msrHWCR;
119
120	/** SVM revision. */
121	uint32_t u32Rev;
122
123	/** SVM feature bits from cpuid 0x8000000a */
124	uint32_t u32Features;
125
126	/** Set by us to indicate SVM is supported by the CPU. */
127	bool fSupported;
128	} svm;
129	/** Saved error from detection */
130	int32_t lLastError;
131
132	struct
133	{
134	uint32_t u32AMDFeatureECX;
135	uint32_t u32AMDFeatureEDX;
136	} cpuid;
137
138	HWACCMSTATE enmHwAccmState;
139
140	bool fGlobalInit;
141	volatile bool fSuspended;
142	} HWACCMR0Globals;
143
144
145
146	/**
147	* Does global Ring-0 HWACCM initialization.
148	*
149	* @returns VBox status code.
150	*/
151	VMMR0DECL(int) HWACCMR0Init(void)
152	{
153	int rc;
154	bool fAMDVPresent = false;
155
156	HWACCMR0Globals.enmHwAccmState = HWACCMSTATE_UNINITIALIZED;
157	for (unsigned i = 0; i < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo); i++)
158	HWACCMR0Globals.aCpuInfo[i].pMemObj = NIL_RTR0MEMOBJ;
159
160	/* Fill in all callbacks with placeholders. */
161	HWACCMR0Globals.pfnEnterSession = HWACCMR0DummyEnter;
162	HWACCMR0Globals.pfnLeaveSession = HWACCMR0DummyLeave;
163	HWACCMR0Globals.pfnSaveHostState = HWACCMR0DummySaveHostState;
164	HWACCMR0Globals.pfnLoadGuestState = HWACCMR0DummyLoadGuestState;
165	HWACCMR0Globals.pfnRunGuestCode = HWACCMR0DummyRunGuestCode;
166	HWACCMR0Globals.pfnEnableCpu = HWACCMR0DummyEnableCpu;
167	HWACCMR0Globals.pfnDisableCpu = HWACCMR0DummyDisableCpu;
168	HWACCMR0Globals.pfnInitVM = HWACCMR0DummyInitVM;
169	HWACCMR0Globals.pfnTermVM = HWACCMR0DummyTermVM;
170	HWACCMR0Globals.pfnSetupVM = HWACCMR0DummySetupVM;
171
172	/* Default is global VT-x/AMD-V init */
173	HWACCMR0Globals.fGlobalInit = true;
174
175	/*
176	* Make sure aCpuInfo is big enough for all the CPUs on this system.
177	*/
178	if (RTMpGetArraySize() > RT_ELEMENTS(HWACCMR0Globals.aCpuInfo))
179	{
180	LogRel(("HWACCM: Too many real CPUs/cores/threads - %u, max %u\n", RTMpGetArraySize(), RT_ELEMENTS(HWACCMR0Globals.aCpuInfo)));
181	return VERR_TOO_MANY_CPUS;
182	}
183
184	/*
185	* Check for VT-x and AMD-V capabilities
186	*/
187	if (ASMHasCpuId())
188	{
189	uint32_t u32FeaturesECX;
190	uint32_t u32Dummy;
191	uint32_t u32FeaturesEDX;
192	uint32_t u32VendorEBX, u32VendorECX, u32VendorEDX;
193
194	ASMCpuId(0, &u32Dummy, &u32VendorEBX, &u32VendorECX, &u32VendorEDX);
195	ASMCpuId(1, &u32Dummy, &u32Dummy, &u32FeaturesECX, &u32FeaturesEDX);
196	/* Query AMD features. */
197	ASMCpuId(0x80000001, &u32Dummy, &u32Dummy, &HWACCMR0Globals.cpuid.u32AMDFeatureECX, &HWACCMR0Globals.cpuid.u32AMDFeatureEDX);
198
199	if ( u32VendorEBX == X86_CPUID_VENDOR_INTEL_EBX
200	&& u32VendorECX == X86_CPUID_VENDOR_INTEL_ECX
201	&& u32VendorEDX == X86_CPUID_VENDOR_INTEL_EDX
202	)
203	{
204	/*
205	* Read all VMX MSRs if VMX is available. (same goes for RDMSR/WRMSR)
206	* We also assume all VMX-enabled CPUs support fxsave/fxrstor.
207	*/
208	if ( (u32FeaturesECX & X86_CPUID_FEATURE_ECX_VMX)
209	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_MSR)
210	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_FXSR)
211	)
212	{
213	int aRc[RTCPUSET_MAX_CPUS];
214	RTCPUID idCpu = 0;
215
216	HWACCMR0Globals.vmx.msr.feature_ctrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
217
218	/*
219	* First try use native kernel API for controlling VT-x.
220	* (This is only supported by some Mac OS X kernels atm.)
221	*/
222	HWACCMR0Globals.lLastError = rc = SUPR0EnableVTx(true /* fEnable */);
223	if (rc != VERR_NOT_SUPPORTED)
224	{
225	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_VMX_IN_VMX_ROOT_MODE \|\| rc == VERR_VMX_NO_VMX, ("%Rrc\n", rc));
226	HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx = true;
227	if (RT_SUCCESS(rc))
228	{
229	HWACCMR0Globals.vmx.fSupported = true;
230	rc = SUPR0EnableVTx(false /* fEnable */);
231	AssertRC(rc);
232	}
233	}
234	else
235	{
236	HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx = false;
237
238	/* We need to check if VT-x has been properly initialized on all CPUs. Some BIOSes do a lousy job. */
239	memset(aRc, 0, sizeof(aRc));
240	HWACCMR0Globals.lLastError = RTMpOnAll(HWACCMR0InitCPU, (void *)u32VendorEBX, aRc);
241
242	/* Check the return code of all invocations. */
243	if (RT_SUCCESS(HWACCMR0Globals.lLastError))
244	HWACCMR0Globals.lLastError = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
245	}
246	if (RT_SUCCESS(HWACCMR0Globals.lLastError))
247	{
248	/* Reread in case we've changed it. */
249	HWACCMR0Globals.vmx.msr.feature_ctrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
250
251	if ( (HWACCMR0Globals.vmx.msr.feature_ctrl & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
252	== (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
253	{
254	RTR0MEMOBJ pScatchMemObj;
255	void *pvScatchPage;
256	RTHCPHYS pScatchPagePhys;
257
258	HWACCMR0Globals.vmx.msr.vmx_basic_info = ASMRdMsr(MSR_IA32_VMX_BASIC_INFO);
259	HWACCMR0Globals.vmx.msr.vmx_pin_ctls.u = ASMRdMsr(MSR_IA32_VMX_PINBASED_CTLS);
260	HWACCMR0Globals.vmx.msr.vmx_proc_ctls.u = ASMRdMsr(MSR_IA32_VMX_PROCBASED_CTLS);
261	HWACCMR0Globals.vmx.msr.vmx_exit.u = ASMRdMsr(MSR_IA32_VMX_EXIT_CTLS);
262	HWACCMR0Globals.vmx.msr.vmx_entry.u = ASMRdMsr(MSR_IA32_VMX_ENTRY_CTLS);
263	HWACCMR0Globals.vmx.msr.vmx_misc = ASMRdMsr(MSR_IA32_VMX_MISC);
264	HWACCMR0Globals.vmx.msr.vmx_cr0_fixed0 = ASMRdMsr(MSR_IA32_VMX_CR0_FIXED0);
265	HWACCMR0Globals.vmx.msr.vmx_cr0_fixed1 = ASMRdMsr(MSR_IA32_VMX_CR0_FIXED1);
266	HWACCMR0Globals.vmx.msr.vmx_cr4_fixed0 = ASMRdMsr(MSR_IA32_VMX_CR4_FIXED0);
267	HWACCMR0Globals.vmx.msr.vmx_cr4_fixed1 = ASMRdMsr(MSR_IA32_VMX_CR4_FIXED1);
268	HWACCMR0Globals.vmx.msr.vmx_vmcs_enum = ASMRdMsr(MSR_IA32_VMX_VMCS_ENUM);
269	/* VPID 16 bits ASID. */
270	HWACCMR0Globals.uMaxASID = 0x10000; /* exclusive */
271
272	if (HWACCMR0Globals.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
273	{
274	HWACCMR0Globals.vmx.msr.vmx_proc_ctls2.u = ASMRdMsr(MSR_IA32_VMX_PROCBASED_CTLS2);
275	if (HWACCMR0Globals.vmx.msr.vmx_proc_ctls2.n.allowed1 & (VMX_VMCS_CTRL_PROC_EXEC2_EPT\|VMX_VMCS_CTRL_PROC_EXEC2_VPID))
276	HWACCMR0Globals.vmx.msr.vmx_eptcaps = ASMRdMsr(MSR_IA32_VMX_EPT_CAPS);
277	}
278
279	if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
280	{
281	HWACCMR0Globals.vmx.hostCR4 = ASMGetCR4();
282	HWACCMR0Globals.vmx.hostEFER = ASMRdMsr(MSR_K6_EFER);
283
284	rc = RTR0MemObjAllocCont(&pScatchMemObj, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
285	if (RT_FAILURE(rc))
286	return rc;
287
288	pvScatchPage = RTR0MemObjAddress(pScatchMemObj);
289	pScatchPagePhys = RTR0MemObjGetPagePhysAddr(pScatchMemObj, 0);
290	memset(pvScatchPage, 0, PAGE_SIZE);
291
292	/* Set revision dword at the beginning of the structure. */
293	(uint32_t )pvScatchPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(HWACCMR0Globals.vmx.msr.vmx_basic_info);
294
295	/* Make sure we don't get rescheduled to another cpu during this probe. */
296	RTCCUINTREG fFlags = ASMIntDisableFlags();
297
298	/*
299	* Check CR4.VMXE
300	*/
301	if (!(HWACCMR0Globals.vmx.hostCR4 & X86_CR4_VMXE))
302	{
303	/* In theory this bit could be cleared behind our back. Which would cause #UD faults when we
304	* try to execute the VMX instructions...
305	*/
306	ASMSetCR4(HWACCMR0Globals.vmx.hostCR4 \| X86_CR4_VMXE);
307	}
308
309	/* Enter VMX Root Mode */
310	rc = VMXEnable(pScatchPagePhys);
311	if (RT_FAILURE(rc))
312	{
313	/* KVM leaves the CPU in VMX root mode. Not only is this not allowed, it will crash the host when we enter raw mode, because
314	* (a) clearing X86_CR4_VMXE in CR4 causes a #GP (we no longer modify this bit)
315	* (b) turning off paging causes a #GP (unavoidable when switching from long to 32 bits mode or 32 bits to PAE)
316	*
317	* They should fix their code, but until they do we simply refuse to run.
318	*/
319	HWACCMR0Globals.lLastError = VERR_VMX_IN_VMX_ROOT_MODE;
320	}
321	else
322	{
323	HWACCMR0Globals.vmx.fSupported = true;
324	VMXDisable();
325
326	/*
327	* Check for the VMX-Preemption Timer and adjust for the
328	* "VMX-Preemption Timer Does Not Count Down at the Rate Specified" erratum.
329	*/
330	if ( HWACCMR0Globals.vmx.msr.vmx_pin_ctls.n.allowed1
331	& VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER)
332	{
333	HWACCMR0Globals.vmx.fUsePreemptTimer = true;
334	HWACCMR0Globals.vmx.cPreemptTimerShift = MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(HWACCMR0Globals.vmx.msr.vmx_misc);
335	if (hwaccmR0IsSubjectToVmxPreemptionTimerErratum())
336	HWACCMR0Globals.vmx.cPreemptTimerShift = 0; /* This is about right most of the time here. */
337	}
338	}
339
340	/* Restore CR4 again; don't leave the X86_CR4_VMXE flag set if it wasn't so before (some software could incorrectly think it's in VMX mode) */
341	ASMSetCR4(HWACCMR0Globals.vmx.hostCR4);
342	ASMSetFlags(fFlags);
343
344	RTR0MemObjFree(pScatchMemObj, false);
345	if (RT_FAILURE(HWACCMR0Globals.lLastError))
346	return HWACCMR0Globals.lLastError;
347	}
348	}
349	else
350	{
351	AssertFailed(); /* can't hit this case anymore */
352	HWACCMR0Globals.lLastError = VERR_VMX_ILLEGAL_FEATURE_CONTROL_MSR;
353	}
354	}
355	#ifdef LOG_ENABLED
356	else
357	SUPR0Printf("HWACCMR0InitCPU failed with rc=%d\n", HWACCMR0Globals.lLastError);
358	#endif
359	}
360	else
361	HWACCMR0Globals.lLastError = VERR_VMX_NO_VMX;
362	}
363	else
364	if ( u32VendorEBX == X86_CPUID_VENDOR_AMD_EBX
365	&& u32VendorECX == X86_CPUID_VENDOR_AMD_ECX
366	&& u32VendorEDX == X86_CPUID_VENDOR_AMD_EDX
367	)
368	{
369	/*
370	* Read all SVM MSRs if SVM is available. (same goes for RDMSR/WRMSR)
371	* We also assume all SVM-enabled CPUs support fxsave/fxrstor.
372	*/
373	if ( (HWACCMR0Globals.cpuid.u32AMDFeatureECX & X86_CPUID_AMD_FEATURE_ECX_SVM)
374	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_MSR)
375	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_FXSR)
376	)
377	{
378	int aRc[RTCPUSET_MAX_CPUS];
379	RTCPUID idCpu = 0;
380
381	fAMDVPresent = true;
382
383	/* Query AMD features. */
384	ASMCpuId(0x8000000A, &HWACCMR0Globals.svm.u32Rev, &HWACCMR0Globals.uMaxASID, &u32Dummy, &HWACCMR0Globals.svm.u32Features);
385
386	/* We need to check if AMD-V has been properly initialized on all CPUs. Some BIOSes might do a poor job. */
387	memset(aRc, 0, sizeof(aRc));
388	rc = RTMpOnAll(HWACCMR0InitCPU, (void *)u32VendorEBX, aRc);
389	AssertRC(rc);
390
391	/* Check the return code of all invocations. */
392	if (RT_SUCCESS(rc))
393	rc = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
394
395	#ifndef DEBUG_bird
396	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_SVM_IN_USE, ("HWACCMR0InitCPU failed for cpu %d with rc=%d\n", idCpu, rc));
397	#endif
398	if (RT_SUCCESS(rc))
399	{
400	/* Read the HWCR msr for diagnostics. */
401	HWACCMR0Globals.svm.msrHWCR = ASMRdMsr(MSR_K8_HWCR);
402	HWACCMR0Globals.svm.fSupported = true;
403	}
404	else
405	HWACCMR0Globals.lLastError = rc;
406	}
407	else
408	HWACCMR0Globals.lLastError = VERR_SVM_NO_SVM;
409	}
410	else
411	HWACCMR0Globals.lLastError = VERR_HWACCM_UNKNOWN_CPU;
412	}
413	else
414	HWACCMR0Globals.lLastError = VERR_HWACCM_NO_CPUID;
415
416	if (HWACCMR0Globals.vmx.fSupported)
417	{
418	HWACCMR0Globals.pfnEnterSession = VMXR0Enter;
419	HWACCMR0Globals.pfnLeaveSession = VMXR0Leave;
420	HWACCMR0Globals.pfnSaveHostState = VMXR0SaveHostState;
421	HWACCMR0Globals.pfnLoadGuestState = VMXR0LoadGuestState;
422	HWACCMR0Globals.pfnRunGuestCode = VMXR0RunGuestCode;
423	HWACCMR0Globals.pfnEnableCpu = VMXR0EnableCpu;
424	HWACCMR0Globals.pfnDisableCpu = VMXR0DisableCpu;
425	HWACCMR0Globals.pfnInitVM = VMXR0InitVM;
426	HWACCMR0Globals.pfnTermVM = VMXR0TermVM;
427	HWACCMR0Globals.pfnSetupVM = VMXR0SetupVM;
428	}
429	else
430	if (fAMDVPresent)
431	{
432	HWACCMR0Globals.pfnEnterSession = SVMR0Enter;
433	HWACCMR0Globals.pfnLeaveSession = SVMR0Leave;
434	HWACCMR0Globals.pfnSaveHostState = SVMR0SaveHostState;
435	HWACCMR0Globals.pfnLoadGuestState = SVMR0LoadGuestState;
436	HWACCMR0Globals.pfnRunGuestCode = SVMR0RunGuestCode;
437	HWACCMR0Globals.pfnEnableCpu = SVMR0EnableCpu;
438	HWACCMR0Globals.pfnDisableCpu = SVMR0DisableCpu;
439	HWACCMR0Globals.pfnInitVM = SVMR0InitVM;
440	HWACCMR0Globals.pfnTermVM = SVMR0TermVM;
441	HWACCMR0Globals.pfnSetupVM = SVMR0SetupVM;
442	}
443
444	if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
445	{
446	rc = RTPowerNotificationRegister(hwaccmR0PowerCallback, 0);
447	AssertRC(rc);
448	}
449
450	return VINF_SUCCESS;
451	}
452
453
454	/**
455	* Checks the error code array filled in for each cpu in the system.
456	*
457	* @returns VBox status code.
458	* @param paRc Error code array
459	* @param cErrorCodes Array size
460	* @param pidCpu Value of the first cpu that set an error (out)
461	*/
462	static int hwaccmR0CheckCpuRcArray(int paRc, unsigned cErrorCodes, RTCPUID pidCpu)
463	{
464	int rc = VINF_SUCCESS;
465
466	Assert(cErrorCodes == RTCPUSET_MAX_CPUS);
467
468	for (unsigned i=0;i<cErrorCodes;i++)
469	{
470	if (RTMpIsCpuOnline(i))
471	{
472	if (RT_FAILURE(paRc[i]))
473	{
474	rc = paRc[i];
475	*pidCpu = i;
476	break;
477	}
478	}
479	}
480	return rc;
481	}
482
483
484	/**
485	* Checks if the CPU is subject to the "VMX-Preemption Timer Does Not Count
486	* Down at the Rate Specified" erratum.
487	*
488	* Errata names and related steppings:
489	* - BA86 - D0.
490	* - AAX65 - C2.
491	* - AAU65 - C2, K0.
492	* - AAO95 - B1.
493	* - AAT59 - C2.
494	* - AAK139 - D0.
495	* - AAM126 - C0, C1, D0.
496	* - AAN92 - B1.
497	* - AAJ124 - C0, D0.
498	*
499	* - AAP86 - B1.
500	*
501	* Steppings: B1, C0, C1, C2, D0, K0.
502	*
503	* @returns true if subject to it, false if not.
504	*/
505	static bool hwaccmR0IsSubjectToVmxPreemptionTimerErratum(void)
506	{
507	uint32_t u = ASMCpuId_EAX(1);
508	u &= ~(RT_BIT_32(14) \| RT_BIT_32(15) \| RT_BIT_32(28) \| RT_BIT_32(29) \| RT_BIT_32(30) \| RT_BIT_32(31));
509	if ( u == UINT32_C(0x000206E6) /* 323344.pdf - BA86 - D0 - Intel Xeon Processor 7500 Series */
510	\|\| u == UINT32_C(0x00020652) /* 323056.pdf - AAX65 - C2 - Intel Xeon Processor L3406 */
511	\|\| u == UINT32_C(0x00020652) /* 322814.pdf - AAT59 - C2 - Intel CoreTM i7-600, i5-500, i5-400 and i3-300 Mobile Processor Series */
512	\|\| u == UINT32_C(0x00020652) /* 322911.pdf - AAU65 - C2 - Intel CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
513	\|\| u == UINT32_C(0x00020655) /* 322911.pdf - AAU65 - K0 - Intel CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
514	\|\| u == UINT32_C(0x000106E5) /* 322373.pdf - AAO95 - B1 - Intel Xeon Processor 3400 Series */
515	\|\| u == UINT32_C(0x000106E5) /* 322166.pdf - AAN92 - B1 - Intel CoreTM i7-800 and i5-700 Desktop Processor Series */
516	\|\| u == UINT32_C(0x000106E5) /* 320767.pdf - AAP86 - B1 - Intel Core i7-900 Mobile Processor Extreme Edition Series, Intel Core i7-800 and i7-700 Mobile Processor Series */
517	\|\| u == UINT32_C(0x000106A0) /?321333.pdf - AAM126 - C0 - Intel Xeon Processor 3500 Series Specification /
518	\|\| u == UINT32_C(0x000106A1) /?321333.pdf - AAM126 - C1 - Intel Xeon Processor 3500 Series Specification /
519	\|\| u == UINT32_C(0x000106A4) /* 320836.pdf - AAJ124 - C0 - Intel Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
520	\|\| u == UINT32_C(0x000106A5) /* 321333.pdf - AAM126 - D0 - Intel Xeon Processor 3500 Series Specification */
521	\|\| u == UINT32_C(0x000106A5) /* 321324.pdf - AAK139 - D0 - Intel Xeon Processor 5500 Series Specification */
522	\|\| u == UINT32_C(0x000106A5) /* 320836.pdf - AAJ124 - D0 - Intel Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
523	)
524	return true;
525	return false;
526	}
527
528
529	/**
530	* Does global Ring-0 HWACCM termination.
531	*
532	* @returns VBox status code.
533	*/
534	VMMR0DECL(int) HWACCMR0Term(void)
535	{
536	int rc;
537	if ( HWACCMR0Globals.vmx.fSupported
538	&& HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
539	{
540	Assert(HWACCMR0Globals.fGlobalInit);
541	rc = SUPR0EnableVTx(false /* fEnable */);
542	for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo); iCpu++)
543	{
544	HWACCMR0Globals.aCpuInfo[iCpu].fConfigured = false;
545	Assert(HWACCMR0Globals.aCpuInfo[iCpu].pMemObj == NIL_RTR0MEMOBJ);
546	}
547	}
548	else
549	{
550	Assert(!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
551	if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
552	{
553	rc = RTPowerNotificationDeregister(hwaccmR0PowerCallback, 0);
554	AssertRC(rc);
555	}
556	else
557	rc = VINF_SUCCESS;
558
559	/* Only disable VT-x/AMD-V on all CPUs if we enabled it before. */
560	if (HWACCMR0Globals.fGlobalInit)
561	{
562	int aRc[RTCPUSET_MAX_CPUS];
563
564	memset(aRc, 0, sizeof(aRc));
565	rc = RTMpOnAll(hwaccmR0DisableCpuCallback, aRc, NULL);
566	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
567	#ifdef VBOX_STRICT
568	for (unsigned i=0;i<RT_ELEMENTS(HWACCMR0Globals.aCpuInfo);i++)
569	AssertMsgRC(aRc[i], ("hwaccmR0DisableCpuCallback failed for cpu %d with rc=%d\n", i, aRc[i]));
570	#endif
571	}
572
573	/* Free the per-cpu pages used for VT-x and AMD-V */
574	for (unsigned i=0;i<RT_ELEMENTS(HWACCMR0Globals.aCpuInfo);i++)
575	{
576	if (HWACCMR0Globals.aCpuInfo[i].pMemObj != NIL_RTR0MEMOBJ)
577	{
578	RTR0MemObjFree(HWACCMR0Globals.aCpuInfo[i].pMemObj, false);
579	HWACCMR0Globals.aCpuInfo[i].pMemObj = NIL_RTR0MEMOBJ;
580	}
581	}
582	}
583	return rc;
584	}
585
586
587	/**
588	* Worker function passed to RTMpOnAll, RTMpOnOthers and RTMpOnSpecific that
589	* is to be called on the target cpus.
590	*
591	* @param idCpu The identifier for the CPU the function is called on.
592	* @param pvUser1 The 1st user argument.
593	* @param pvUser2 The 2nd user argument.
594	*/
595	static DECLCALLBACK(void) HWACCMR0InitCPU(RTCPUID idCpu, void pvUser1, void pvUser2)
596	{
597	unsigned u32VendorEBX = (uintptr_t)pvUser1;
598	int paRc = (int )pvUser2;
599	uint64_t val;
600
601	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
602	SUPR0Printf("HWACCMR0InitCPU cpu %d\n", idCpu);
603	#endif
604	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /// @todo fix idCpu == index assumption (rainy day)
605
606	if (u32VendorEBX == X86_CPUID_VENDOR_INTEL_EBX)
607	{
608	val = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
609
610	/*
611	* Both the LOCK and VMXON bit must be set; otherwise VMXON will generate a #GP.
612	* Once the lock bit is set, this MSR can no longer be modified.
613	*/
614	if ( !(val & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
615	\|\| ((val & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK)) == MSR_IA32_FEATURE_CONTROL_VMXON) /* Some BIOSes forget to set the locked bit. */
616	)
617	{
618	/* MSR is not yet locked; we can change it ourselves here */
619	ASMWrMsr(MSR_IA32_FEATURE_CONTROL, HWACCMR0Globals.vmx.msr.feature_ctrl \| MSR_IA32_FEATURE_CONTROL_VMXON \| MSR_IA32_FEATURE_CONTROL_LOCK);
620	val = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
621	}
622	if ( (val & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
623	== (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
624	paRc[idCpu] = VINF_SUCCESS;
625	else
626	paRc[idCpu] = VERR_VMX_MSR_LOCKED_OR_DISABLED;
627	}
628	else
629	if (u32VendorEBX == X86_CPUID_VENDOR_AMD_EBX)
630	{
631	/* Check if SVM is disabled */
632	val = ASMRdMsr(MSR_K8_VM_CR);
633	if (!(val & MSR_K8_VM_CR_SVM_DISABLE))
634	{
635	/* Turn on SVM in the EFER MSR. */
636	val = ASMRdMsr(MSR_K6_EFER);
637	if (val & MSR_K6_EFER_SVME)
638	{
639	paRc[idCpu] = VERR_SVM_IN_USE;
640	}
641	else
642	{
643	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
644
645	/* Paranoia. */
646	val = ASMRdMsr(MSR_K6_EFER);
647	if (val & MSR_K6_EFER_SVME)
648	{
649	/* Restore previous value. */
650	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
651	paRc[idCpu] = VINF_SUCCESS;
652	}
653	else
654	paRc[idCpu] = VERR_SVM_ILLEGAL_EFER_MSR;
655	}
656	}
657	else
658	paRc[idCpu] = VERR_SVM_DISABLED;
659	}
660	else
661	AssertFailed(); /* can't happen */
662	return;
663	}
664
665
666	/**
667	* Sets up HWACCM on all cpus.
668	*
669	* @returns VBox status code.
670	* @param pVM The VM to operate on.
671	*
672	*/
673	VMMR0DECL(int) HWACCMR0EnableAllCpus(PVM pVM)
674	{
675	AssertCompile(sizeof(HWACCMR0Globals.enmHwAccmState) == sizeof(uint32_t));
676
677	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
678	if (ASMAtomicReadBool(&HWACCMR0Globals.fSuspended))
679	return VERR_HWACCM_SUSPEND_PENDING;
680
681	if (ASMAtomicCmpXchgU32((volatile uint32_t *)&HWACCMR0Globals.enmHwAccmState, HWACCMSTATE_ENABLED, HWACCMSTATE_UNINITIALIZED))
682	{
683	int rc;
684
685	HWACCMR0Globals.fGlobalInit = pVM->hwaccm.s.fGlobalInit;
686
687	if ( HWACCMR0Globals.vmx.fSupported
688	&& HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
689	{
690	rc = SUPR0EnableVTx(true /* fEnable */);
691	if (RT_SUCCESS(rc))
692	{
693	for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo); iCpu++)
694	{
695	HWACCMR0Globals.aCpuInfo[iCpu].fConfigured = true;
696	Assert(HWACCMR0Globals.aCpuInfo[iCpu].pMemObj == NIL_RTR0MEMOBJ);
697	}
698	/* If the host provides a VT-x init API, then we'll rely on that for global init. */
699	HWACCMR0Globals.fGlobalInit = pVM->hwaccm.s.fGlobalInit = true;
700	}
701	else
702	AssertMsgFailed(("HWACCMR0EnableAllCpus/SUPR0EnableVTx: rc=%Rrc\n", rc));
703	}
704	else
705	{
706	int aRc[RTCPUSET_MAX_CPUS];
707	RTCPUID idCpu = 0;
708
709	memset(aRc, 0, sizeof(aRc));
710
711	/* Allocate one page per cpu for the global vt-x and amd-v pages */
712	for (unsigned i=0;i<RT_ELEMENTS(HWACCMR0Globals.aCpuInfo);i++)
713	{
714	Assert(!HWACCMR0Globals.aCpuInfo[i].pMemObj);
715
716	/** @todo this is rather dangerous if cpus can be taken offline; we don't care for now */
717	if (RTMpIsCpuOnline(i))
718	{
719	rc = RTR0MemObjAllocCont(&HWACCMR0Globals.aCpuInfo[i].pMemObj, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
720	AssertRC(rc);
721	if (RT_FAILURE(rc))
722	return rc;
723
724	void *pvR0 = RTR0MemObjAddress(HWACCMR0Globals.aCpuInfo[i].pMemObj);
725	Assert(pvR0);
726	ASMMemZeroPage(pvR0);
727
728	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
729	SUPR0Printf("address %x phys %x\n", pvR0, (uint32_t)RTR0MemObjGetPagePhysAddr(HWACCMR0Globals.aCpuInfo[i].pMemObj, 0));
730	#endif
731	}
732	}
733	if (HWACCMR0Globals.fGlobalInit)
734	{
735	/* First time, so initialize each cpu/core */
736	rc = RTMpOnAll(hwaccmR0EnableCpuCallback, (void *)pVM, aRc);
737
738	/* Check the return code of all invocations. */
739	if (RT_SUCCESS(rc))
740	rc = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
741	AssertMsgRC(rc, ("HWACCMR0EnableAllCpus failed for cpu %d with rc=%d\n", idCpu, rc));
742	}
743	else
744	rc = VINF_SUCCESS;
745	}
746
747	return rc;
748	}
749	return VINF_SUCCESS;
750	}
751
752	/**
753	* Disable VT-x or AMD-V on the current CPU
754	*
755	* @returns VBox status code.
756	* @param pVM VM handle (can be 0!)
757	* @param idCpu The identifier for the CPU the function is called on.
758	*/
759	static int hwaccmR0EnableCpu(PVM pVM, RTCPUID idCpu)
760	{
761	void *pvPageCpu;
762	RTHCPHYS pPageCpuPhys;
763	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
764
765	Assert(!HWACCMR0Globals.vmx.fSupported \|\| !HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
766	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /// @todo fix idCpu == index assumption (rainy day)
767	Assert(idCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo));
768	Assert(!pCpu->fConfigured);
769	Assert(!HWACCMR0Globals.fGlobalInit \|\| ASMAtomicReadBool(&pCpu->fInUse) == false);
770
771	pCpu->idCpu = idCpu;
772
773	/* Make sure we start with a clean TLB. */
774	pCpu->fFlushTLB = true;
775
776	pCpu->uCurrentASID = 0; /* we'll aways increment this the first time (host uses ASID 0) */
777	pCpu->cTLBFlushes = 0;
778
779	/* Should never happen */
780	if (!pCpu->pMemObj)
781	{
782	AssertFailed();
783	return VERR_INTERNAL_ERROR;
784	}
785
786	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
787	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
788
789	int rc = HWACCMR0Globals.pfnEnableCpu(pCpu, pVM, pvPageCpu, pPageCpuPhys);
790	AssertRC(rc);
791	if (RT_SUCCESS(rc))
792	pCpu->fConfigured = true;
793
794	return rc;
795	}
796
797
798	/**
799	* Worker function passed to RTMpOnAll, RTMpOnOthers and RTMpOnSpecific that
800	* is to be called on the target cpus.
801	*
802	* @param idCpu The identifier for the CPU the function is called on.
803	* @param pvUser1 The 1st user argument.
804	* @param pvUser2 The 2nd user argument.
805	*/
806	static DECLCALLBACK(void) hwaccmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
807	{
808	PVM pVM = (PVM)pvUser1; /* can be NULL! */
809	int paRc = (int )pvUser2;
810
811	if (!HWACCMR0Globals.fGlobalInit)
812	{
813	paRc[idCpu] = VINF_SUCCESS;
814	AssertFailed();
815	return;
816	}
817
818	paRc[idCpu] = hwaccmR0EnableCpu(pVM, idCpu);
819	}
820
821
822	/**
823	* Disable VT-x or AMD-V on the current CPU
824	*
825	* @returns VBox status code.
826	* @param idCpu The identifier for the CPU the function is called on.
827	*/
828	static int hwaccmR0DisableCpu(RTCPUID idCpu)
829	{
830	void *pvPageCpu;
831	RTHCPHYS pPageCpuPhys;
832	int rc;
833	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
834
835	Assert(!HWACCMR0Globals.vmx.fSupported \|\| !HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
836	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /// @todo fix idCpu == index assumption (rainy day)
837	Assert(idCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo));
838	Assert(!HWACCMR0Globals.fGlobalInit \|\| ASMAtomicReadBool(&pCpu->fInUse) == false);
839	Assert(!pCpu->fConfigured \|\| pCpu->pMemObj);
840
841	if (!pCpu->pMemObj)
842	return (pCpu->fConfigured) ? VERR_NO_MEMORY : VINF_SUCCESS /* not initialized. */;
843
844	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
845	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
846
847	if (pCpu->fConfigured)
848	{
849	rc = HWACCMR0Globals.pfnDisableCpu(pCpu, pvPageCpu, pPageCpuPhys);
850	AssertRC(rc);
851	pCpu->fConfigured = false;
852	}
853	else
854	rc = VINF_SUCCESS; /* nothing to do */
855
856	pCpu->uCurrentASID = 0;
857	return rc;
858	}
859
860	/**
861	* Worker function passed to RTMpOnAll, RTMpOnOthers and RTMpOnSpecific that
862	* is to be called on the target cpus.
863	*
864	* @param idCpu The identifier for the CPU the function is called on.
865	* @param pvUser1 The 1st user argument.
866	* @param pvUser2 The 2nd user argument.
867	*/
868	static DECLCALLBACK(void) hwaccmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
869	{
870	int paRc = (int )pvUser1;
871
872	if (!HWACCMR0Globals.fGlobalInit)
873	{
874	paRc[idCpu] = VINF_SUCCESS;
875	AssertFailed();
876	return;
877	}
878
879	paRc[idCpu] = hwaccmR0DisableCpu(idCpu);
880	}
881
882	/**
883	* Called whenever a system power state change occurs.
884	*
885	* @param enmEvent Power event
886	* @param pvUser User argument
887	*/
888	static DECLCALLBACK(void) hwaccmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser)
889	{
890	NOREF(pvUser);
891	Assert(!HWACCMR0Globals.vmx.fSupported \|\| !HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
892
893	#ifdef LOG_ENABLED
894	if (enmEvent == RTPOWEREVENT_SUSPEND)
895	SUPR0Printf("hwaccmR0PowerCallback RTPOWEREVENT_SUSPEND\n");
896	else
897	SUPR0Printf("hwaccmR0PowerCallback RTPOWEREVENT_RESUME\n");
898	#endif
899
900	if (enmEvent == RTPOWEREVENT_SUSPEND)
901	ASMAtomicWriteBool(&HWACCMR0Globals.fSuspended, true);
902
903	if (HWACCMR0Globals.enmHwAccmState == HWACCMSTATE_ENABLED)
904	{
905	int aRc[RTCPUSET_MAX_CPUS];
906	int rc;
907	RTCPUID idCpu;
908
909	memset(aRc, 0, sizeof(aRc));
910	if (enmEvent == RTPOWEREVENT_SUSPEND)
911	{
912	if (HWACCMR0Globals.fGlobalInit)
913	{
914	/* Turn off VT-x or AMD-V on all CPUs. */
915	rc = RTMpOnAll(hwaccmR0DisableCpuCallback, aRc, NULL);
916	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
917	}
918	/* else nothing to do here for the local init case */
919	}
920	else
921	{
922	/* Reinit the CPUs from scratch as the suspend state might have messed with the MSRs. (lousy BIOSes as usual) */
923	rc = RTMpOnAll(HWACCMR0InitCPU, (void *)((HWACCMR0Globals.vmx.fSupported) ? X86_CPUID_VENDOR_INTEL_EBX : X86_CPUID_VENDOR_AMD_EBX), aRc);
924	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
925
926	if (RT_SUCCESS(rc))
927	rc = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
928	#ifdef LOG_ENABLED
929	if (RT_FAILURE(rc))
930	SUPR0Printf("hwaccmR0PowerCallback HWACCMR0InitCPU failed with %d\n", rc);
931	#endif
932
933	if (HWACCMR0Globals.fGlobalInit)
934	{
935	/* Turn VT-x or AMD-V back on on all CPUs. */
936	rc = RTMpOnAll(hwaccmR0EnableCpuCallback, NULL, aRc);
937	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
938	}
939	/* else nothing to do here for the local init case */
940	}
941	}
942	if (enmEvent == RTPOWEREVENT_RESUME)
943	ASMAtomicWriteBool(&HWACCMR0Globals.fSuspended, false);
944	}
945
946
947	/**
948	* Does Ring-0 per VM HWACCM initialization.
949	*
950	* This is mainly to check that the Host CPU mode is compatible
951	* with VMX.
952	*
953	* @returns VBox status code.
954	* @param pVM The VM to operate on.
955	*/
956	VMMR0DECL(int) HWACCMR0InitVM(PVM pVM)
957	{
958	int rc;
959
960	AssertReturn(pVM, VERR_INVALID_PARAMETER);
961
962	#ifdef LOG_ENABLED
963	SUPR0Printf("HWACCMR0InitVM: %p\n", pVM);
964	#endif
965
966	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
967	if (ASMAtomicReadBool(&HWACCMR0Globals.fSuspended))
968	return VERR_HWACCM_SUSPEND_PENDING;
969
970	pVM->hwaccm.s.vmx.fSupported = HWACCMR0Globals.vmx.fSupported;
971	pVM->hwaccm.s.svm.fSupported = HWACCMR0Globals.svm.fSupported;
972
973	pVM->hwaccm.s.vmx.fUsePreemptTimer = HWACCMR0Globals.vmx.fUsePreemptTimer;
974	pVM->hwaccm.s.vmx.cPreemptTimerShift = HWACCMR0Globals.vmx.cPreemptTimerShift;
975	pVM->hwaccm.s.vmx.msr.feature_ctrl = HWACCMR0Globals.vmx.msr.feature_ctrl;
976	pVM->hwaccm.s.vmx.hostCR4 = HWACCMR0Globals.vmx.hostCR4;
977	pVM->hwaccm.s.vmx.hostEFER = HWACCMR0Globals.vmx.hostEFER;
978	pVM->hwaccm.s.vmx.msr.vmx_basic_info = HWACCMR0Globals.vmx.msr.vmx_basic_info;
979	pVM->hwaccm.s.vmx.msr.vmx_pin_ctls = HWACCMR0Globals.vmx.msr.vmx_pin_ctls;
980	pVM->hwaccm.s.vmx.msr.vmx_proc_ctls = HWACCMR0Globals.vmx.msr.vmx_proc_ctls;
981	pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2 = HWACCMR0Globals.vmx.msr.vmx_proc_ctls2;
982	pVM->hwaccm.s.vmx.msr.vmx_exit = HWACCMR0Globals.vmx.msr.vmx_exit;
983	pVM->hwaccm.s.vmx.msr.vmx_entry = HWACCMR0Globals.vmx.msr.vmx_entry;
984	pVM->hwaccm.s.vmx.msr.vmx_misc = HWACCMR0Globals.vmx.msr.vmx_misc;
985	pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0 = HWACCMR0Globals.vmx.msr.vmx_cr0_fixed0;
986	pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1 = HWACCMR0Globals.vmx.msr.vmx_cr0_fixed1;
987	pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0 = HWACCMR0Globals.vmx.msr.vmx_cr4_fixed0;
988	pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1 = HWACCMR0Globals.vmx.msr.vmx_cr4_fixed1;
989	pVM->hwaccm.s.vmx.msr.vmx_vmcs_enum = HWACCMR0Globals.vmx.msr.vmx_vmcs_enum;
990	pVM->hwaccm.s.vmx.msr.vmx_eptcaps = HWACCMR0Globals.vmx.msr.vmx_eptcaps;
991	pVM->hwaccm.s.svm.msrHWCR = HWACCMR0Globals.svm.msrHWCR;
992	pVM->hwaccm.s.svm.u32Rev = HWACCMR0Globals.svm.u32Rev;
993	pVM->hwaccm.s.svm.u32Features = HWACCMR0Globals.svm.u32Features;
994	pVM->hwaccm.s.cpuid.u32AMDFeatureECX = HWACCMR0Globals.cpuid.u32AMDFeatureECX;
995	pVM->hwaccm.s.cpuid.u32AMDFeatureEDX = HWACCMR0Globals.cpuid.u32AMDFeatureEDX;
996	pVM->hwaccm.s.lLastError = HWACCMR0Globals.lLastError;
997
998	pVM->hwaccm.s.uMaxASID = HWACCMR0Globals.uMaxASID;
999
1000
1001	if (!pVM->hwaccm.s.cMaxResumeLoops) /* allow ring-3 overrides */
1002	{
1003	pVM->hwaccm.s.cMaxResumeLoops = 1024;
1004	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1005	if (RTThreadPreemptIsPendingTrusty())
1006	pVM->hwaccm.s.cMaxResumeLoops = 8192;
1007	#endif
1008	}
1009
1010	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1011	{
1012	PVMCPU pVCpu = &pVM->aCpus[i];
1013
1014	pVCpu->hwaccm.s.idEnteredCpu = NIL_RTCPUID;
1015
1016	/* Invalidate the last cpu we were running on. */
1017	pVCpu->hwaccm.s.idLastCpu = NIL_RTCPUID;
1018
1019	/* we'll aways increment this the first time (host uses ASID 0) */
1020	pVCpu->hwaccm.s.uCurrentASID = 0;
1021	}
1022
1023	RTCCUINTREG fFlags = ASMIntDisableFlags();
1024	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1025
1026	/* Note: Not correct as we can be rescheduled to a different cpu, but the fInUse case is mostly for debugging. */
1027	ASMAtomicWriteBool(&pCpu->fInUse, true);
1028	ASMSetFlags(fFlags);
1029
1030	/* Init a VT-x or AMD-V VM. */
1031	rc = HWACCMR0Globals.pfnInitVM(pVM);
1032
1033	ASMAtomicWriteBool(&pCpu->fInUse, false);
1034	return rc;
1035	}
1036
1037
1038	/**
1039	* Does Ring-0 per VM HWACCM termination.
1040	*
1041	* @returns VBox status code.
1042	* @param pVM The VM to operate on.
1043	*/
1044	VMMR0DECL(int) HWACCMR0TermVM(PVM pVM)
1045	{
1046	int rc;
1047
1048	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1049
1050	#ifdef LOG_ENABLED
1051	SUPR0Printf("HWACCMR0TermVM: %p\n", pVM);
1052	#endif
1053
1054	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
1055	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1056
1057	/* @note Not correct as we can be rescheduled to a different cpu, but the fInUse case is mostly for debugging. */
1058	RTCCUINTREG fFlags = ASMIntDisableFlags();
1059	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1060
1061	ASMAtomicWriteBool(&pCpu->fInUse, true);
1062	ASMSetFlags(fFlags);
1063
1064	/* Terminate a VT-x or AMD-V VM. */
1065	rc = HWACCMR0Globals.pfnTermVM(pVM);
1066
1067	ASMAtomicWriteBool(&pCpu->fInUse, false);
1068	return rc;
1069	}
1070
1071
1072	/**
1073	* Sets up a VT-x or AMD-V session
1074	*
1075	* @returns VBox status code.
1076	* @param pVM The VM to operate on.
1077	*/
1078	VMMR0DECL(int) HWACCMR0SetupVM(PVM pVM)
1079	{
1080	int rc;
1081	RTCPUID idCpu = RTMpCpuId();
1082	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1083
1084	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1085
1086	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
1087	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1088
1089	#ifdef LOG_ENABLED
1090	SUPR0Printf("HWACCMR0SetupVM: %p\n", pVM);
1091	#endif
1092
1093	ASMAtomicWriteBool(&pCpu->fInUse, true);
1094
1095	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1096	{
1097	/* On first entry we'll sync everything. */
1098	pVM->aCpus[i].hwaccm.s.fContextUseFlags = HWACCM_CHANGED_ALL;
1099	}
1100
1101	/* Enable VT-x or AMD-V if local init is required. */
1102	if (!HWACCMR0Globals.fGlobalInit)
1103	{
1104	rc = hwaccmR0EnableCpu(pVM, idCpu);
1105	AssertRCReturn(rc, rc);
1106	}
1107
1108	/* Setup VT-x or AMD-V. */
1109	rc = HWACCMR0Globals.pfnSetupVM(pVM);
1110
1111	/* Disable VT-x or AMD-V if local init was done before. */
1112	if (!HWACCMR0Globals.fGlobalInit)
1113	{
1114	rc = hwaccmR0DisableCpu(idCpu);
1115	AssertRC(rc);
1116	}
1117
1118	ASMAtomicWriteBool(&pCpu->fInUse, false);
1119
1120	return rc;
1121	}
1122
1123
1124	/**
1125	* Enters the VT-x or AMD-V session
1126	*
1127	* @returns VBox status code.
1128	* @param pVM The VM to operate on.
1129	* @param pVCpu VMCPUD id.
1130	*/
1131	VMMR0DECL(int) HWACCMR0Enter(PVM pVM, PVMCPU pVCpu)
1132	{
1133	PCPUMCTX pCtx;
1134	int rc;
1135	RTCPUID idCpu = RTMpCpuId();
1136	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1137
1138	/* Make sure we can't enter a session after we've disabled hwaccm in preparation of a suspend. */
1139	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1140	ASMAtomicWriteBool(&pCpu->fInUse, true);
1141
1142	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == NIL_RTCPUID, ("%d", (int)pVCpu->hwaccm.s.idEnteredCpu));
1143	pVCpu->hwaccm.s.idEnteredCpu = idCpu;
1144
1145	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1146
1147	/* Always load the guest's FPU/XMM state on-demand. */
1148	CPUMDeactivateGuestFPUState(pVCpu);
1149
1150	/* Always load the guest's debug state on-demand. */
1151	CPUMDeactivateGuestDebugState(pVCpu);
1152
1153	/* Always reload the host context and the guest's CR0 register. (!!!!) */
1154	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_HOST_CONTEXT;
1155
1156	/* Setup the register and mask according to the current execution mode. */
1157	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1158	pVM->hwaccm.s.u64RegisterMask = UINT64_C(0xFFFFFFFFFFFFFFFF);
1159	else
1160	pVM->hwaccm.s.u64RegisterMask = UINT64_C(0xFFFFFFFF);
1161
1162	/* Enable VT-x or AMD-V if local init is required. */
1163	if (!HWACCMR0Globals.fGlobalInit)
1164	{
1165	rc = hwaccmR0EnableCpu(pVM, idCpu);
1166	AssertRCReturn(rc, rc);
1167	}
1168
1169	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1170	bool fStartedSet = PGMR0DynMapStartOrMigrateAutoSet(pVCpu);
1171	#endif
1172
1173	rc = HWACCMR0Globals.pfnEnterSession(pVM, pVCpu, pCpu);
1174	AssertRC(rc);
1175	/* We must save the host context here (VT-x) as we might be rescheduled on a different cpu after a long jump back to ring 3. */
1176	rc \|= HWACCMR0Globals.pfnSaveHostState(pVM, pVCpu);
1177	AssertRC(rc);
1178	rc \|= HWACCMR0Globals.pfnLoadGuestState(pVM, pVCpu, pCtx);
1179	AssertRC(rc);
1180
1181	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1182	if (fStartedSet)
1183	PGMRZDynMapReleaseAutoSet(pVCpu);
1184	#endif
1185
1186	/* keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
1187	if (RT_FAILURE(rc))
1188	pVCpu->hwaccm.s.idEnteredCpu = NIL_RTCPUID;
1189	return rc;
1190	}
1191
1192
1193	/**
1194	* Leaves the VT-x or AMD-V session
1195	*
1196	* @returns VBox status code.
1197	* @param pVM The VM to operate on.
1198	* @param pVCpu VMCPUD id.
1199	*/
1200	VMMR0DECL(int) HWACCMR0Leave(PVM pVM, PVMCPU pVCpu)
1201	{
1202	PCPUMCTX pCtx;
1203	int rc;
1204	RTCPUID idCpu = RTMpCpuId();
1205	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1206
1207	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1208
1209	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1210
1211	/* Note: It's rather tricky with longjmps done by e.g. Log statements or the page fault handler.
1212	* We must restore the host FPU here to make absolutely sure we don't leave the guest FPU state active
1213	* or trash somebody else's FPU state.
1214	*/
1215	/* Save the guest FPU and XMM state if necessary. */
1216	if (CPUMIsGuestFPUStateActive(pVCpu))
1217	{
1218	Log2(("CPUMR0SaveGuestFPU\n"));
1219	CPUMR0SaveGuestFPU(pVM, pVCpu, pCtx);
1220
1221	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1222	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
1223	}
1224
1225	rc = HWACCMR0Globals.pfnLeaveSession(pVM, pVCpu, pCtx);
1226
1227	/* We don't pass on invlpg information to the recompiler for nested paging guests, so we must make sure the recompiler flushes its TLB
1228	* the next time it executes code.
1229	*/
1230	if ( pVM->hwaccm.s.fNestedPaging
1231	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1232	{
1233	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1234	}
1235
1236	/* keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
1237	#ifdef RT_STRICT
1238	if (RT_UNLIKELY( pVCpu->hwaccm.s.idEnteredCpu != idCpu
1239	&& RT_FAILURE(rc)))
1240	{
1241	AssertMsgFailed(("Owner is %d, I'm %d", (int)pVCpu->hwaccm.s.idEnteredCpu, (int)idCpu));
1242	rc = VERR_INTERNAL_ERROR;
1243	}
1244	#endif
1245	pVCpu->hwaccm.s.idEnteredCpu = NIL_RTCPUID;
1246
1247	/* Disable VT-x or AMD-V if local init was done before. */
1248	if (!HWACCMR0Globals.fGlobalInit)
1249	{
1250	rc = hwaccmR0DisableCpu(idCpu);
1251	AssertRC(rc);
1252
1253	/* Reset these to force a TLB flush for the next entry. (-> EXPENSIVE) */
1254	pVCpu->hwaccm.s.idLastCpu = NIL_RTCPUID;
1255	pVCpu->hwaccm.s.uCurrentASID = 0;
1256	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1257	}
1258
1259	ASMAtomicWriteBool(&pCpu->fInUse, false);
1260	return rc;
1261	}
1262
1263	/**
1264	* Runs guest code in a hardware accelerated VM.
1265	*
1266	* @returns VBox status code.
1267	* @param pVM The VM to operate on.
1268	* @param pVCpu VMCPUD id.
1269	*/
1270	VMMR0DECL(int) HWACCMR0RunGuestCode(PVM pVM, PVMCPU pVCpu)
1271	{
1272	CPUMCTX *pCtx;
1273	int rc;
1274	#ifdef VBOX_STRICT
1275	RTCPUID idCpu = RTMpCpuId(); NOREF(idCpu);
1276	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1277	#endif
1278
1279	Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1280	Assert(HWACCMR0Globals.aCpuInfo[idCpu].fConfigured);
1281	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1282	Assert(ASMAtomicReadBool(&pCpu->fInUse) == true);
1283
1284	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1285	PGMRZDynMapStartAutoSet(pVCpu);
1286	#endif
1287
1288	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1289
1290	rc = HWACCMR0Globals.pfnRunGuestCode(pVM, pVCpu, pCtx);
1291
1292	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1293	PGMRZDynMapReleaseAutoSet(pVCpu);
1294	#endif
1295	return rc;
1296	}
1297
1298
1299	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1300	/**
1301	* Save guest FPU/XMM state (64 bits guest mode & 32 bits host only)
1302	*
1303	* @returns VBox status code.
1304	* @param pVM VM handle.
1305	* @param pVCpu VMCPU handle.
1306	* @param pCtx CPU context
1307	*/
1308	VMMR0DECL(int) HWACCMR0SaveFPUState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1309	{
1310	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFpu64SwitchBack);
1311	if (pVM->hwaccm.s.vmx.fSupported)
1312	return VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestFPU64, 0, NULL);
1313
1314	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestFPU64, 0, NULL);
1315	}
1316
1317	/**
1318	* Save guest debug state (64 bits guest mode & 32 bits host only)
1319	*
1320	* @returns VBox status code.
1321	* @param pVM VM handle.
1322	* @param pVCpu VMCPU handle.
1323	* @param pCtx CPU context
1324	*/
1325	VMMR0DECL(int) HWACCMR0SaveDebugState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1326	{
1327	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDebug64SwitchBack);
1328	if (pVM->hwaccm.s.vmx.fSupported)
1329	return VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestDebug64, 0, NULL);
1330
1331	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestDebug64, 0, NULL);
1332	}
1333
1334	/**
1335	* Test the 32->64 bits switcher
1336	*
1337	* @returns VBox status code.
1338	* @param pVM VM handle.
1339	*/
1340	VMMR0DECL(int) HWACCMR0TestSwitcher3264(PVM pVM)
1341	{
1342	PVMCPU pVCpu = &pVM->aCpus[0];
1343	CPUMCTX *pCtx;
1344	uint32_t aParam[5] = {0, 1, 2, 3, 4};
1345	int rc;
1346
1347	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1348
1349	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
1350	if (pVM->hwaccm.s.vmx.fSupported)
1351	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnTest64, 5, &aParam[0]);
1352	else
1353	rc = SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnTest64, 5, &aParam[0]);
1354	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
1355	return rc;
1356	}
1357
1358	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
1359
1360	/**
1361	* Returns suspend status of the host
1362	*
1363	* @returns Suspend pending or not
1364	*/
1365	VMMR0DECL(bool) HWACCMR0SuspendPending()
1366	{
1367	return ASMAtomicReadBool(&HWACCMR0Globals.fSuspended);
1368	}
1369
1370	/**
1371	* Returns the cpu structure for the current cpu.
1372	* Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
1373	*
1374	* @returns cpu structure pointer
1375	*/
1376	VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpu()
1377	{
1378	RTCPUID idCpu = RTMpCpuId();
1379
1380	return &HWACCMR0Globals.aCpuInfo[idCpu];
1381	}
1382
1383	/**
1384	* Returns the cpu structure for the current cpu.
1385	* Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
1386	*
1387	* @returns cpu structure pointer
1388	* @param idCpu id of the VCPU
1389	*/
1390	VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpuEx(RTCPUID idCpu)
1391	{
1392	return &HWACCMR0Globals.aCpuInfo[idCpu];
1393	}
1394
1395	/**
1396	* Save a pending IO read.
1397	*
1398	* @param pVCpu The VMCPU to operate on.
1399	* @param GCPtrRip Address of IO instruction
1400	* @param GCPtrRipNext Address of the next instruction
1401	* @param uPort Port address
1402	* @param uAndVal And mask for saving the result in eax
1403	* @param cbSize Read size
1404	*/
1405	VMMR0DECL(void) HWACCMR0SavePendingIOPortRead(PVMCPU pVCpu, RTGCPTR GCPtrRip, RTGCPTR GCPtrRipNext, unsigned uPort, unsigned uAndVal, unsigned cbSize)
1406	{
1407	pVCpu->hwaccm.s.PendingIO.enmType = HWACCMPENDINGIO_PORT_READ;
1408	pVCpu->hwaccm.s.PendingIO.GCPtrRip = GCPtrRip;
1409	pVCpu->hwaccm.s.PendingIO.GCPtrRipNext = GCPtrRipNext;
1410	pVCpu->hwaccm.s.PendingIO.s.Port.uPort = uPort;
1411	pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal = uAndVal;
1412	pVCpu->hwaccm.s.PendingIO.s.Port.cbSize = cbSize;
1413	return;
1414	}
1415
1416	/**
1417	* Save a pending IO write.
1418	*
1419	* @param pVCpu The VMCPU to operate on.
1420	* @param GCPtrRIP Address of IO instruction
1421	* @param uPort Port address
1422	* @param uAndVal And mask for fetching the result from eax
1423	* @param cbSize Read size
1424	*/
1425	VMMR0DECL(void) HWACCMR0SavePendingIOPortWrite(PVMCPU pVCpu, RTGCPTR GCPtrRip, RTGCPTR GCPtrRipNext, unsigned uPort, unsigned uAndVal, unsigned cbSize)
1426	{
1427	pVCpu->hwaccm.s.PendingIO.enmType = HWACCMPENDINGIO_PORT_WRITE;
1428	pVCpu->hwaccm.s.PendingIO.GCPtrRip = GCPtrRip;
1429	pVCpu->hwaccm.s.PendingIO.GCPtrRipNext = GCPtrRipNext;
1430	pVCpu->hwaccm.s.PendingIO.s.Port.uPort = uPort;
1431	pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal = uAndVal;
1432	pVCpu->hwaccm.s.PendingIO.s.Port.cbSize = cbSize;
1433	return;
1434	}
1435
1436	/**
1437	* Disable VT-x if it's active and the current switcher turns off paging
1438	*
1439	* @returns VBox status code.
1440	* @param pVM VM handle.
1441	* @param pfVTxDisabled VT-x was disabled or not (out)
1442	*/
1443	VMMR0DECL(int) HWACCMR0EnterSwitcher(PVM pVM, bool *pfVTxDisabled)
1444	{
1445	Assert(!(ASMGetFlags() & X86_EFL_IF) \|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1446
1447	*pfVTxDisabled = false;
1448
1449	if ( HWACCMR0Globals.enmHwAccmState != HWACCMSTATE_ENABLED
1450	\|\| !HWACCMR0Globals.vmx.fSupported /* no such issues with AMD-V */
1451	\|\| !HWACCMR0Globals.fGlobalInit /* Local init implies the CPU is currently not in VMX root mode. */)
1452	return VINF_SUCCESS; /* nothing to do */
1453
1454	switch(VMMGetSwitcher(pVM))
1455	{
1456	case VMMSWITCHER_32_TO_32:
1457	case VMMSWITCHER_PAE_TO_PAE:
1458	return VINF_SUCCESS; /* safe switchers as they don't turn off paging */
1459
1460	case VMMSWITCHER_32_TO_PAE:
1461	case VMMSWITCHER_PAE_TO_32: /* is this one actually used?? */
1462	case VMMSWITCHER_AMD64_TO_32:
1463	case VMMSWITCHER_AMD64_TO_PAE:
1464	break; /* unsafe switchers */
1465
1466	default:
1467	AssertFailed();
1468	return VERR_INTERNAL_ERROR;
1469	}
1470
1471	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1472	void *pvPageCpu;
1473	RTHCPHYS pPageCpuPhys;
1474
1475	AssertReturn(pCpu && pCpu->pMemObj, VERR_INTERNAL_ERROR);
1476	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
1477	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
1478
1479	*pfVTxDisabled = true;
1480	return VMXR0DisableCpu(pCpu, pvPageCpu, pPageCpuPhys);
1481	}
1482
1483	/**
1484	* Enable VT-x if was active and the current switcher turned off paging
1485	*
1486	* @returns VBox status code.
1487	* @param pVM VM handle.
1488	* @param fVTxDisabled VT-x was disabled or not
1489	*/
1490	VMMR0DECL(int) HWACCMR0LeaveSwitcher(PVM pVM, bool fVTxDisabled)
1491	{
1492	Assert(!(ASMGetFlags() & X86_EFL_IF));
1493
1494	if (!fVTxDisabled)
1495	return VINF_SUCCESS; /* nothing to do */
1496
1497	Assert( HWACCMR0Globals.enmHwAccmState == HWACCMSTATE_ENABLED
1498	&& HWACCMR0Globals.vmx.fSupported
1499	&& HWACCMR0Globals.fGlobalInit);
1500
1501	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1502	void *pvPageCpu;
1503	RTHCPHYS pPageCpuPhys;
1504
1505	AssertReturn(pCpu && pCpu->pMemObj, VERR_INTERNAL_ERROR);
1506	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
1507	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
1508
1509	return VMXR0EnableCpu(pCpu, pVM, pvPageCpu, pPageCpuPhys);
1510	}
1511
1512	#ifdef VBOX_STRICT
1513	/**
1514	* Dumps a descriptor.
1515	*
1516	* @param pDesc Descriptor to dump.
1517	* @param Sel Selector number.
1518	* @param pszMsg Message to prepend the log entry with.
1519	*/
1520	VMMR0DECL(void) HWACCMR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszMsg)
1521	{
1522	/*
1523	* Make variable description string.
1524	*/
1525	static struct
1526	{
1527	unsigned cch;
1528	const char *psz;
1529	} const s_aTypes[32] =
1530	{
1531	# define STRENTRY(str) { sizeof(str) - 1, str }
1532
1533	/* system */
1534	# if HC_ARCH_BITS == 64
1535	STRENTRY("Reserved0 "), /* 0x00 */
1536	STRENTRY("Reserved1 "), /* 0x01 */
1537	STRENTRY("LDT "), /* 0x02 */
1538	STRENTRY("Reserved3 "), /* 0x03 */
1539	STRENTRY("Reserved4 "), /* 0x04 */
1540	STRENTRY("Reserved5 "), /* 0x05 */
1541	STRENTRY("Reserved6 "), /* 0x06 */
1542	STRENTRY("Reserved7 "), /* 0x07 */
1543	STRENTRY("Reserved8 "), /* 0x08 */
1544	STRENTRY("TSS64Avail "), /* 0x09 */
1545	STRENTRY("ReservedA "), /* 0x0a */
1546	STRENTRY("TSS64Busy "), /* 0x0b */
1547	STRENTRY("Call64 "), /* 0x0c */
1548	STRENTRY("ReservedD "), /* 0x0d */
1549	STRENTRY("Int64 "), /* 0x0e */
1550	STRENTRY("Trap64 "), /* 0x0f */
1551	# else
1552	STRENTRY("Reserved0 "), /* 0x00 */
1553	STRENTRY("TSS16Avail "), /* 0x01 */
1554	STRENTRY("LDT "), /* 0x02 */
1555	STRENTRY("TSS16Busy "), /* 0x03 */
1556	STRENTRY("Call16 "), /* 0x04 */
1557	STRENTRY("Task "), /* 0x05 */
1558	STRENTRY("Int16 "), /* 0x06 */
1559	STRENTRY("Trap16 "), /* 0x07 */
1560	STRENTRY("Reserved8 "), /* 0x08 */
1561	STRENTRY("TSS32Avail "), /* 0x09 */
1562	STRENTRY("ReservedA "), /* 0x0a */
1563	STRENTRY("TSS32Busy "), /* 0x0b */
1564	STRENTRY("Call32 "), /* 0x0c */
1565	STRENTRY("ReservedD "), /* 0x0d */
1566	STRENTRY("Int32 "), /* 0x0e */
1567	STRENTRY("Trap32 "), /* 0x0f */
1568	# endif
1569	/* non system */
1570	STRENTRY("DataRO "), /* 0x10 */
1571	STRENTRY("DataRO Accessed "), /* 0x11 */
1572	STRENTRY("DataRW "), /* 0x12 */
1573	STRENTRY("DataRW Accessed "), /* 0x13 */
1574	STRENTRY("DataDownRO "), /* 0x14 */
1575	STRENTRY("DataDownRO Accessed "), /* 0x15 */
1576	STRENTRY("DataDownRW "), /* 0x16 */
1577	STRENTRY("DataDownRW Accessed "), /* 0x17 */
1578	STRENTRY("CodeEO "), /* 0x18 */
1579	STRENTRY("CodeEO Accessed "), /* 0x19 */
1580	STRENTRY("CodeER "), /* 0x1a */
1581	STRENTRY("CodeER Accessed "), /* 0x1b */
1582	STRENTRY("CodeConfEO "), /* 0x1c */
1583	STRENTRY("CodeConfEO Accessed "), /* 0x1d */
1584	STRENTRY("CodeConfER "), /* 0x1e */
1585	STRENTRY("CodeConfER Accessed ") /* 0x1f */
1586	# undef SYSENTRY
1587	};
1588	# define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
1589	char szMsg[128];
1590	char *psz = &szMsg[0];
1591	unsigned i = pDesc->Gen.u1DescType << 4 \| pDesc->Gen.u4Type;
1592	memcpy(psz, s_aTypes[i].psz, s_aTypes[i].cch);
1593	psz += s_aTypes[i].cch;
1594
1595	if (pDesc->Gen.u1Present)
1596	ADD_STR(psz, "Present ");
1597	else
1598	ADD_STR(psz, "Not-Present ");
1599	# if HC_ARCH_BITS == 64
1600	if (pDesc->Gen.u1Long)
1601	ADD_STR(psz, "64-bit ");
1602	else
1603	ADD_STR(psz, "Comp ");
1604	# else
1605	if (pDesc->Gen.u1Granularity)
1606	ADD_STR(psz, "Page ");
1607	if (pDesc->Gen.u1DefBig)
1608	ADD_STR(psz, "32-bit ");
1609	else
1610	ADD_STR(psz, "16-bit ");
1611	# endif
1612	# undef ADD_STR
1613	*psz = '\0';
1614
1615	/*
1616	* Limit and Base and format the output.
1617	*/
1618	uint32_t u32Limit = X86DESC_LIMIT(*pDesc);
1619	if (pDesc->Gen.u1Granularity)
1620	u32Limit = u32Limit << PAGE_SHIFT \| PAGE_OFFSET_MASK;
1621
1622	# if HC_ARCH_BITS == 64
1623	uint64_t u32Base = X86DESC64_BASE(*pDesc);
1624
1625	Log(("%s %04x - %RX64 %RX64 - base=%RX64 limit=%08x dpl=%d %s\n", pszMsg,
1626	Sel, pDesc->au64[0], pDesc->au64[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1627	# else
1628	uint32_t u32Base = X86DESC_BASE(*pDesc);
1629
1630	Log(("%s %04x - %08x %08x - base=%08x limit=%08x dpl=%d %s\n", pszMsg,
1631	Sel, pDesc->au32[0], pDesc->au32[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1632	# endif
1633	}
1634
1635	/**
1636	* Formats a full register dump.
1637	*
1638	* @param pVM The VM to operate on.
1639	* @param pVCpu The VMCPU to operate on.
1640	* @param pCtx The context to format.
1641	*/
1642	VMMR0DECL(void) HWACCMDumpRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1643	{
1644	/*
1645	* Format the flags.
1646	*/
1647	static struct
1648	{
1649	const char pszSet; const char pszClear; uint32_t fFlag;
1650	} aFlags[] =
1651	{
1652	{ "vip",NULL, X86_EFL_VIP },
1653	{ "vif",NULL, X86_EFL_VIF },
1654	{ "ac", NULL, X86_EFL_AC },
1655	{ "vm", NULL, X86_EFL_VM },
1656	{ "rf", NULL, X86_EFL_RF },
1657	{ "nt", NULL, X86_EFL_NT },
1658	{ "ov", "nv", X86_EFL_OF },
1659	{ "dn", "up", X86_EFL_DF },
1660	{ "ei", "di", X86_EFL_IF },
1661	{ "tf", NULL, X86_EFL_TF },
1662	{ "nt", "pl", X86_EFL_SF },
1663	{ "nz", "zr", X86_EFL_ZF },
1664	{ "ac", "na", X86_EFL_AF },
1665	{ "po", "pe", X86_EFL_PF },
1666	{ "cy", "nc", X86_EFL_CF },
1667	};
1668	char szEFlags[80];
1669	char *psz = szEFlags;
1670	uint32_t efl = pCtx->eflags.u32;
1671	for (unsigned i = 0; i < RT_ELEMENTS(aFlags); i++)
1672	{
1673	const char *pszAdd = aFlags[i].fFlag & efl ? aFlags[i].pszSet : aFlags[i].pszClear;
1674	if (pszAdd)
1675	{
1676	strcpy(psz, pszAdd);
1677	psz += strlen(pszAdd);
1678	*psz++ = ' ';
1679	}
1680	}
1681	psz[-1] = '\0';
1682
1683
1684	/*
1685	* Format the registers.
1686	*/
1687	if (CPUMIsGuestIn64BitCode(pVCpu, CPUMCTX2CORE(pCtx)))
1688	{
1689	Log(("rax=%016RX64 rbx=%016RX64 rcx=%016RX64 rdx=%016RX64\n"
1690	"rsi=%016RX64 rdi=%016RX64 r8 =%016RX64 r9 =%016RX64\n"
1691	"r10=%016RX64 r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
1692	"r14=%016RX64 r15=%016RX64\n"
1693	"rip=%016RX64 rsp=%016RX64 rbp=%016RX64 iopl=%d %*s\n"
1694	"cs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1695	"ds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1696	"es={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1697	"fs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1698	"gs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1699	"ss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1700	"cr0=%016RX64 cr2=%016RX64 cr3=%016RX64 cr4=%016RX64\n"
1701	"dr0=%016RX64 dr1=%016RX64 dr2=%016RX64 dr3=%016RX64\n"
1702	"dr4=%016RX64 dr5=%016RX64 dr6=%016RX64 dr7=%016RX64\n"
1703	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1704	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1705	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1706	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1707	,
1708	pCtx->rax, pCtx->rbx, pCtx->rcx, pCtx->rdx, pCtx->rsi, pCtx->rdi,
1709	pCtx->r8, pCtx->r9, pCtx->r10, pCtx->r11, pCtx->r12, pCtx->r13,
1710	pCtx->r14, pCtx->r15,
1711	pCtx->rip, pCtx->rsp, pCtx->rbp, X86_EFL_GET_IOPL(efl), 31, szEFlags,
1712	(RTSEL)pCtx->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u,
1713	(RTSEL)pCtx->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u,
1714	(RTSEL)pCtx->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u,
1715	(RTSEL)pCtx->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u,
1716	(RTSEL)pCtx->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u,
1717	(RTSEL)pCtx->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u,
1718	pCtx->cr0, pCtx->cr2, pCtx->cr3, pCtx->cr4,
1719	pCtx->dr[0], pCtx->dr[1], pCtx->dr[2], pCtx->dr[3],
1720	pCtx->dr[4], pCtx->dr[5], pCtx->dr[6], pCtx->dr[7],
1721	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, efl,
1722	(RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
1723	(RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
1724	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1725	}
1726	else
1727	Log(("eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
1728	"eip=%08x esp=%08x ebp=%08x iopl=%d %*s\n"
1729	"cs={%04x base=%016RX64 limit=%08x flags=%08x} dr0=%08RX64 dr1=%08RX64\n"
1730	"ds={%04x base=%016RX64 limit=%08x flags=%08x} dr2=%08RX64 dr3=%08RX64\n"
1731	"es={%04x base=%016RX64 limit=%08x flags=%08x} dr4=%08RX64 dr5=%08RX64\n"
1732	"fs={%04x base=%016RX64 limit=%08x flags=%08x} dr6=%08RX64 dr7=%08RX64\n"
1733	"gs={%04x base=%016RX64 limit=%08x flags=%08x} cr0=%08RX64 cr2=%08RX64\n"
1734	"ss={%04x base=%016RX64 limit=%08x flags=%08x} cr3=%08RX64 cr4=%08RX64\n"
1735	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1736	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1737	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1738	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1739	,
1740	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
1741	pCtx->eip, pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(efl), 31, szEFlags,
1742	(RTSEL)pCtx->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u, pCtx->dr[0], pCtx->dr[1],
1743	(RTSEL)pCtx->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u, pCtx->dr[2], pCtx->dr[3],
1744	(RTSEL)pCtx->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u, pCtx->dr[4], pCtx->dr[5],
1745	(RTSEL)pCtx->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u, pCtx->dr[6], pCtx->dr[7],
1746	(RTSEL)pCtx->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u, pCtx->cr0, pCtx->cr2,
1747	(RTSEL)pCtx->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u, pCtx->cr3, pCtx->cr4,
1748	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, efl,
1749	(RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
1750	(RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
1751	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1752
1753	Log(("FPU:\n"
1754	"FCW=%04x FSW=%04x FTW=%02x\n"
1755	"FOP=%04x FPUIP=%08x CS=%04x Rsvrd1=%04x\n"
1756	"FPUDP=%04x DS=%04x Rsvrd2=%04x MXCSR=%08x MXCSR_MASK=%08x\n"
1757	,
1758	pCtx->fpu.FCW, pCtx->fpu.FSW, pCtx->fpu.FTW,
1759	pCtx->fpu.FOP, pCtx->fpu.FPUIP, pCtx->fpu.CS, pCtx->fpu.Rsvrd1,
1760	pCtx->fpu.FPUDP, pCtx->fpu.DS, pCtx->fpu.Rsrvd2,
1761	pCtx->fpu.MXCSR, pCtx->fpu.MXCSR_MASK));
1762
1763
1764	Log(("MSR:\n"
1765	"EFER =%016RX64\n"
1766	"PAT =%016RX64\n"
1767	"STAR =%016RX64\n"
1768	"CSTAR =%016RX64\n"
1769	"LSTAR =%016RX64\n"
1770	"SFMASK =%016RX64\n"
1771	"KERNELGSBASE =%016RX64\n",
1772	pCtx->msrEFER,
1773	pCtx->msrPAT,
1774	pCtx->msrSTAR,
1775	pCtx->msrCSTAR,
1776	pCtx->msrLSTAR,
1777	pCtx->msrSFMASK,
1778	pCtx->msrKERNELGSBASE));
1779
1780	}
1781	#endif /* VBOX_STRICT */
1782
1783	/* Dummy callback handlers. */
1784	VMMR0DECL(int) HWACCMR0DummyEnter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
1785	{
1786	return VINF_SUCCESS;
1787	}
1788
1789	VMMR0DECL(int) HWACCMR0DummyLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1790	{
1791	return VINF_SUCCESS;
1792	}
1793
1794	VMMR0DECL(int) HWACCMR0DummyEnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
1795	{
1796	return VINF_SUCCESS;
1797	}
1798
1799	VMMR0DECL(int) HWACCMR0DummyDisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
1800	{
1801	return VINF_SUCCESS;
1802	}
1803
1804	VMMR0DECL(int) HWACCMR0DummyInitVM(PVM pVM)
1805	{
1806	return VINF_SUCCESS;
1807	}
1808
1809	VMMR0DECL(int) HWACCMR0DummyTermVM(PVM pVM)
1810	{
1811	return VINF_SUCCESS;
1812	}
1813
1814	VMMR0DECL(int) HWACCMR0DummySetupVM(PVM pVM)
1815	{
1816	return VINF_SUCCESS;
1817	}
1818
1819	VMMR0DECL(int) HWACCMR0DummyRunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1820	{
1821	return VINF_SUCCESS;
1822	}
1823
1824	VMMR0DECL(int) HWACCMR0DummySaveHostState(PVM pVM, PVMCPU pVCpu)
1825	{
1826	return VINF_SUCCESS;
1827	}
1828
1829	VMMR0DECL(int) HWACCMR0DummyLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1830	{
1831	return VINF_SUCCESS;
1832	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWACCMR0.cpp@ 36236

Download in other formats: