VirtualBox

source: vbox/trunk/src/VBox/VMM/HWACCM.cpp@ 7954

Last change on this file since 7954 was 7904, checked in by vboxsync, 17 years ago

Additional checks for pgmGstGetPaePDPtr return value

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 32.7 KB
Line 
1/* $Id: HWACCM.cpp 7904 2008-04-11 09:39:47Z vboxsync $ */
2/** @file
3 * HWACCM - Intel/AMD VM Hardware Support Manager
4 */
5
6/*
7 * Copyright (C) 2006-2007 innotek GmbH
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HWACCM
22#include <VBox/cpum.h>
23#include <VBox/stam.h>
24#include <VBox/mm.h>
25#include <VBox/pdm.h>
26#include <VBox/pgm.h>
27#include <VBox/trpm.h>
28#include <VBox/dbgf.h>
29#include <VBox/hwacc_vmx.h>
30#include <VBox/hwacc_svm.h>
31#include "HWACCMInternal.h"
32#include <VBox/vm.h>
33#include <VBox/err.h>
34#include <VBox/param.h>
35#include <VBox/patm.h>
36#include <VBox/csam.h>
37#include <VBox/selm.h>
38
39#include <iprt/assert.h>
40#include <VBox/log.h>
41#include <iprt/asm.h>
42#include <iprt/string.h>
43#include <iprt/thread.h>
44
45
46/*******************************************************************************
47* Internal Functions *
48*******************************************************************************/
49static DECLCALLBACK(int) hwaccmR3Save(PVM pVM, PSSMHANDLE pSSM);
50static DECLCALLBACK(int) hwaccmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version);
51
52
53/**
54 * Initializes the HWACCM.
55 *
56 * @returns VBox status code.
57 * @param pVM The VM to operate on.
58 */
59HWACCMR3DECL(int) HWACCMR3Init(PVM pVM)
60{
61 LogFlow(("HWACCMR3Init\n"));
62
63 /*
64 * Assert alignment and sizes.
65 */
66 AssertRelease(!(RT_OFFSETOF(VM, hwaccm.s) & 31));
67 AssertRelease(sizeof(pVM->hwaccm.s) <= sizeof(pVM->hwaccm.padding));
68
69 /* Some structure checks. */
70 AssertMsg(RT_OFFSETOF(SVM_VMCB, u8Reserved3) == 0xC0, ("u8Reserved3 offset = %x\n", RT_OFFSETOF(SVM_VMCB, u8Reserved3)));
71 AssertMsg(RT_OFFSETOF(SVM_VMCB, ctrl.EventInject) == 0xA8, ("ctrl.EventInject offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.EventInject)));
72 AssertMsg(RT_OFFSETOF(SVM_VMCB, ctrl.ExitIntInfo) == 0x88, ("ctrl.ExitIntInfo offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.ExitIntInfo)));
73 AssertMsg(RT_OFFSETOF(SVM_VMCB, ctrl.TLBCtrl) == 0x58, ("ctrl.TLBCtrl offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.TLBCtrl)));
74
75 AssertMsg(RT_OFFSETOF(SVM_VMCB, guest) == 0x400, ("guest offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest)));
76 AssertMsg(RT_OFFSETOF(SVM_VMCB, guest.u8Reserved4) == 0x4A0, ("guest.u8Reserved4 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u8Reserved4)));
77 AssertMsg(RT_OFFSETOF(SVM_VMCB, guest.u8Reserved6) == 0x4D8, ("guest.u8Reserved6 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u8Reserved6)));
78 AssertMsg(RT_OFFSETOF(SVM_VMCB, guest.u8Reserved7) == 0x580, ("guest.u8Reserved7 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u8Reserved7)));
79 AssertMsg(RT_OFFSETOF(SVM_VMCB, guest.u8Reserved9) == 0x648, ("guest.u8Reserved9 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u8Reserved9)));
80 AssertMsg(RT_OFFSETOF(SVM_VMCB, u8Reserved10) == 0x698, ("u8Reserved3 offset = %x\n", RT_OFFSETOF(SVM_VMCB, u8Reserved10)));
81 AssertMsg(sizeof(SVM_VMCB) == 0x1000, ("SVM_VMCB size = %x\n", sizeof(SVM_VMCB)));
82
83
84 /*
85 * Register the saved state data unit.
86 */
87 int rc = SSMR3RegisterInternal(pVM, "HWACCM", 0, HWACCM_SSM_VERSION, sizeof(HWACCM),
88 NULL, hwaccmR3Save, NULL,
89 NULL, hwaccmR3Load, NULL);
90 if (VBOX_FAILURE(rc))
91 return rc;
92
93 /* Misc initialisation. */
94 pVM->hwaccm.s.vmx.fSupported = false;
95 pVM->hwaccm.s.svm.fSupported = false;
96 pVM->hwaccm.s.vmx.fEnabled = false;
97 pVM->hwaccm.s.svm.fEnabled = false;
98
99 pVM->hwaccm.s.fActive = false;
100
101 /* On first entry we'll sync everything. */
102 pVM->hwaccm.s.fContextUseFlags = HWACCM_CHANGED_ALL;
103
104 pVM->hwaccm.s.vmx.cr0_mask = 0;
105 pVM->hwaccm.s.vmx.cr4_mask = 0;
106
107 /*
108 * Statistics.
109 */
110 STAM_REG(pVM, &pVM->hwaccm.s.StatEntry, STAMTYPE_PROFILE, "/PROF/HWACCM/SwitchToGC", STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode entry");
111 STAM_REG(pVM, &pVM->hwaccm.s.StatExit, STAMTYPE_PROFILE, "/PROF/HWACCM/SwitchFromGC", STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode exit");
112 STAM_REG(pVM, &pVM->hwaccm.s.StatInGC, STAMTYPE_PROFILE, "/PROF/HWACCM/InGC", STAMUNIT_TICKS_PER_CALL, "Profiling of vmlaunch");
113
114 STAM_REG(pVM, &pVM->hwaccm.s.StatExitShadowNM, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Shadow/#NM", STAMUNIT_OCCURENCES, "Nr of occurances");
115 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestNM, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#NM", STAMUNIT_OCCURENCES, "Nr of occurances");
116 STAM_REG(pVM, &pVM->hwaccm.s.StatExitShadowPF, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Shadow/#PF", STAMUNIT_OCCURENCES, "Nr of occurances");
117 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestPF, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#PF", STAMUNIT_OCCURENCES, "Nr of occurances");
118 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestUD, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#UD", STAMUNIT_OCCURENCES, "Nr of occurances");
119 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestSS, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#SS", STAMUNIT_OCCURENCES, "Nr of occurances");
120 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestNP, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#NP", STAMUNIT_OCCURENCES, "Nr of occurances");
121 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestGP, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#GP", STAMUNIT_OCCURENCES, "Nr of occurances");
122 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestMF, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#MF", STAMUNIT_OCCURENCES, "Nr of occurances");
123 STAM_REG(pVM, &pVM->hwaccm.s.StatExitGuestDE, STAMTYPE_COUNTER, "/HWACCM/Exit/Trap/Guest/#DE", STAMUNIT_OCCURENCES, "Nr of occurances");
124 STAM_REG(pVM, &pVM->hwaccm.s.StatExitInvpg, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/Invlpg", STAMUNIT_OCCURENCES, "Nr of occurances");
125 STAM_REG(pVM, &pVM->hwaccm.s.StatExitInvd, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/Invd", STAMUNIT_OCCURENCES, "Nr of occurances");
126 STAM_REG(pVM, &pVM->hwaccm.s.StatExitCpuid, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/Cpuid", STAMUNIT_OCCURENCES, "Nr of occurances");
127 STAM_REG(pVM, &pVM->hwaccm.s.StatExitRdtsc, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/Rdtsc", STAMUNIT_OCCURENCES, "Nr of occurances");
128 STAM_REG(pVM, &pVM->hwaccm.s.StatExitCRxWrite, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/CRx/Write", STAMUNIT_OCCURENCES, "Nr of occurances");
129 STAM_REG(pVM, &pVM->hwaccm.s.StatExitCRxRead, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/CRx/Read", STAMUNIT_OCCURENCES, "Nr of occurances");
130 STAM_REG(pVM, &pVM->hwaccm.s.StatExitDRxWrite, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/DRx/Write", STAMUNIT_OCCURENCES, "Nr of occurances");
131 STAM_REG(pVM, &pVM->hwaccm.s.StatExitDRxRead, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/DRx/Read", STAMUNIT_OCCURENCES, "Nr of occurances");
132 STAM_REG(pVM, &pVM->hwaccm.s.StatExitCLTS, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/CLTS", STAMUNIT_OCCURENCES, "Nr of occurances");
133 STAM_REG(pVM, &pVM->hwaccm.s.StatExitLMSW, STAMTYPE_COUNTER, "/HWACCM/Exit/Instr/LMSW", STAMUNIT_OCCURENCES, "Nr of occurances");
134 STAM_REG(pVM, &pVM->hwaccm.s.StatExitIOWrite, STAMTYPE_COUNTER, "/HWACCM/Exit/IO/Write", STAMUNIT_OCCURENCES, "Nr of occurances");
135 STAM_REG(pVM, &pVM->hwaccm.s.StatExitIORead, STAMTYPE_COUNTER, "/HWACCM/Exit/IO/Read", STAMUNIT_OCCURENCES, "Nr of occurances");
136 STAM_REG(pVM, &pVM->hwaccm.s.StatExitIOStringWrite, STAMTYPE_COUNTER, "/HWACCM/Exit/IO/WriteString", STAMUNIT_OCCURENCES, "Nr of occurances");
137 STAM_REG(pVM, &pVM->hwaccm.s.StatExitIOStringRead, STAMTYPE_COUNTER, "/HWACCM/Exit/IO/ReadString", STAMUNIT_OCCURENCES, "Nr of occurances");
138 STAM_REG(pVM, &pVM->hwaccm.s.StatExitIrqWindow, STAMTYPE_COUNTER, "/HWACCM/Exit/GuestIrq/Pending", STAMUNIT_OCCURENCES, "Nr of occurances");
139 STAM_REG(pVM, &pVM->hwaccm.s.StatExitMaxResume, STAMTYPE_COUNTER, "/HWACCM/Exit/Safety/MaxResume", STAMUNIT_OCCURENCES, "Nr of occurances");
140
141 STAM_REG(pVM, &pVM->hwaccm.s.StatSwitchGuestIrq,STAMTYPE_COUNTER, "/HWACCM/Switch/IrqPending", STAMUNIT_OCCURENCES, "Nr of occurances");
142 STAM_REG(pVM, &pVM->hwaccm.s.StatSwitchToR3, STAMTYPE_COUNTER, "/HWACCM/Switch/ToR3", STAMUNIT_OCCURENCES, "Nr of occurances");
143
144 STAM_REG(pVM, &pVM->hwaccm.s.StatIntInject, STAMTYPE_COUNTER, "/HWACCM/Irq/Inject", STAMUNIT_OCCURENCES, "Nr of occurances");
145 STAM_REG(pVM, &pVM->hwaccm.s.StatIntReinject, STAMTYPE_COUNTER, "/HWACCM/Irq/Reinject", STAMUNIT_OCCURENCES, "Nr of occurances");
146 STAM_REG(pVM, &pVM->hwaccm.s.StatPendingHostIrq,STAMTYPE_COUNTER, "/HWACCM/Irq/PendingOnHost", STAMUNIT_OCCURENCES, "Nr of occurances");
147
148 pVM->hwaccm.s.pStatExitReason = 0;
149
150#ifdef VBOX_WITH_STATISTICS
151 rc = MMHyperAlloc(pVM, MAX_EXITREASON_STAT*sizeof(*pVM->hwaccm.s.pStatExitReason), 0, MM_TAG_HWACCM, (void **)&pVM->hwaccm.s.pStatExitReason);
152 AssertRC(rc);
153 if (VBOX_SUCCESS(rc))
154 {
155 for (int i=0;i<MAX_EXITREASON_STAT;i++)
156 {
157 char szName[64];
158 RTStrPrintf(szName, sizeof(szName), "/HWACCM/Exit/Reason/%02x", i);
159 int rc = STAMR3Register(pVM, &pVM->hwaccm.s.pStatExitReason[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, "Exit reason");
160 AssertRC(rc);
161 }
162 }
163 pVM->hwaccm.s.pStatExitReasonR0 = MMHyperR3ToR0(pVM, pVM->hwaccm.s.pStatExitReason);
164 Assert(pVM->hwaccm.s.pStatExitReasonR0);
165#endif
166
167 /* Disabled by default. */
168 pVM->fHWACCMEnabled = false;
169
170 /* HWACCM support must be explicitely enabled in the configuration file. */
171 pVM->hwaccm.s.fAllowed = false;
172 CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "HWVirtExt/"), "Enabled", &pVM->hwaccm.s.fAllowed);
173
174 return VINF_SUCCESS;
175}
176
177
178/**
179 * Turns off normal raw mode features
180 *
181 * @param pVM The VM to operate on.
182 */
183static void hwaccmr3DisableRawMode(PVM pVM)
184{
185 /* Disable PATM & CSAM. */
186 PATMR3AllowPatching(pVM, false);
187 CSAMDisableScanning(pVM);
188
189 /* Turn off IDT/LDT/GDT and TSS monitoring and sycing. */
190 SELMR3DisableMonitoring(pVM);
191 TRPMR3DisableMonitoring(pVM);
192
193 /* The hidden selector registers are now valid. */
194 CPUMSetHiddenSelRegsValid(pVM, true);
195
196 /* Disable the switcher code (safety precaution). */
197 VMMR3DisableSwitcher(pVM);
198
199 /* Disable mapping of the hypervisor into the shadow page table. */
200 PGMR3ChangeShwPDMappings(pVM, false);
201
202 /* Disable the switcher */
203 VMMR3DisableSwitcher(pVM);
204}
205
206/**
207 * Initialize VT-x or AMD-V.
208 *
209 * @returns VBox status code.
210 * @param pVM The VM handle.
211 */
212HWACCMR3DECL(int) HWACCMR3InitFinalizeR0(PVM pVM)
213{
214 int rc;
215
216 if ( !pVM->hwaccm.s.vmx.fSupported
217 && !pVM->hwaccm.s.svm.fSupported)
218 {
219 LogRel(("HWACCM: No VMX or SVM CPU extension found. Reason %Vrc\n", pVM->hwaccm.s.lLastError));
220 LogRel(("HWACCM: VMX MSR_IA32_FEATURE_CONTROL=%VX64\n", pVM->hwaccm.s.vmx.msr.feature_ctrl));
221 return VINF_SUCCESS;
222 }
223
224 /*
225 * Note that we have a global setting for VT-x/AMD-V usage. VMX root mode changes the way the CPU operates. Our 64 bits switcher will trap
226 * because it turns off paging, which is not allowed in VMX root mode.
227 *
228 * To simplify matters we'll just force all running VMs to either use raw or hwaccm mode. No mixing allowed.
229 *
230 */
231 /* If we enabled or disabled hwaccm mode, then it can't be changed until all the VMs are shutdown. */
232 rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_HWACC_ENABLE, (pVM->hwaccm.s.fAllowed) ? HWACCMSTATE_ENABLED : HWACCMSTATE_DISABLED, NULL);
233 if (VBOX_FAILURE(rc))
234 {
235 LogRel(("HWACCMR3InitFinalize: SUPCallVMMR0Ex VMMR0_DO_HWACC_ENABLE failed with %Vrc\n", rc));
236 LogRel(("HWACCMR3InitFinalize: disallowed %s of HWACCM\n", pVM->hwaccm.s.fAllowed ? "enabling" : "disabling"));
237 /* Invert the selection */
238 pVM->hwaccm.s.fAllowed ^= 1;
239 LogRel(("HWACCMR3InitFinalize: new HWACCM status = %s\n", pVM->hwaccm.s.fAllowed ? "enabled" : "disabled"));
240
241 if (pVM->hwaccm.s.fAllowed)
242 {
243 if (pVM->hwaccm.s.vmx.fSupported)
244 VMSetRuntimeError(pVM, false, "HwAccmModeChangeDisallowed", "An active VM already uses Intel VT-x hardware acceleration. It is not allowed to simultaneously use software virtualization, therefore this VM will be run using VT-x as well.\n");
245 else
246 VMSetRuntimeError(pVM, false, "HwAccmModeChangeDisallowed", "An active VM already uses AMD-V hardware acceleration. It is not allowed to simultaneously use software virtualization, therefore this VM will be run using AMD-V as well.\n");
247 }
248 else
249 VMSetRuntimeError(pVM, false, "HwAccmModeChangeDisallowed", "An active VM already uses software virtualization. It is not allowed to simultaneously use VT-x or AMD-V, therefore this VM will be run using software virtualization as well.\n");
250 }
251
252 if (pVM->hwaccm.s.fAllowed == false)
253 return VINF_SUCCESS; /* disabled */
254
255 Assert(!pVM->fHWACCMEnabled);
256
257 if (pVM->hwaccm.s.vmx.fSupported)
258 {
259 Log(("pVM->hwaccm.s.vmx.fSupported = %d\n", pVM->hwaccm.s.vmx.fSupported));
260
261 if ( pVM->hwaccm.s.fInitialized == false
262 && pVM->hwaccm.s.vmx.msr.feature_ctrl != 0)
263 {
264 uint64_t val;
265
266 LogRel(("HWACCM: Host CR4=%08X\n", pVM->hwaccm.s.vmx.hostCR4));
267 LogRel(("HWACCM: MSR_IA32_FEATURE_CONTROL = %VX64\n", pVM->hwaccm.s.vmx.msr.feature_ctrl));
268 LogRel(("HWACCM: MSR_IA32_VMX_BASIC_INFO = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_basic_info));
269 LogRel(("HWACCM: VMCS id = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
270 LogRel(("HWACCM: VMCS size = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
271 LogRel(("HWACCM: VMCS physical address limit = %s\n", MSR_IA32_VMX_BASIC_INFO_VMCS_PHYS_WIDTH(pVM->hwaccm.s.vmx.msr.vmx_basic_info) ? "< 4 GB" : "None"));
272 LogRel(("HWACCM: VMCS memory type = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_MEM_TYPE(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
273 LogRel(("HWACCM: Dual monitor treatment = %d\n", MSR_IA32_VMX_BASIC_INFO_VMCS_DUAL_MON(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
274
275 LogRel(("HWACCM: MSR_IA32_VMX_PINBASED_CTLS = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_pin_ctls));
276 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls >> 32ULL;
277 if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT)
278 LogRel(("HWACCM: VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT\n"));
279 if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT)
280 LogRel(("HWACCM: VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT\n"));
281 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls;
282 if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT)
283 LogRel(("HWACCM: VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT *must* be set\n"));
284 if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT)
285 LogRel(("HWACCM: VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT *must* be set\n"));
286
287 LogRel(("HWACCM: MSR_IA32_VMX_PROCBASED_CTLS = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_proc_ctls));
288 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls >> 32ULL;
289 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT)
290 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT\n"));
291 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET)
292 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET\n"));
293 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT)
294 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT\n"));
295 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT)
296 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT\n"));
297 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT)
298 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT\n"));
299 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT)
300 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT\n"));
301 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT)
302 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT\n"));
303 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT)
304 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT\n"));
305 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT)
306 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT\n"));
307 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
308 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW\n"));
309 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
310 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT\n"));
311 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT)
312 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT\n"));
313 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS)
314 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS\n"));
315 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
316 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS\n"));
317 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT)
318 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT\n"));
319 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
320 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT\n"));
321 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls;
322 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT)
323 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT *must* be set\n"));
324 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET)
325 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET *must* be set\n"));
326 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT)
327 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT *must* be set\n"));
328 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT)
329 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT *must* be set\n"));
330 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT)
331 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT *must* be set\n"));
332 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT)
333 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT *must* be set\n"));
334 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT)
335 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT *must* be set\n"));
336 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT)
337 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT *must* be set\n"));
338 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT)
339 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT *must* be set\n"));
340 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
341 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW *must* be set\n"));
342 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
343 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT *must* be set\n"));
344 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT)
345 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT *must* be set\n"));
346 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS)
347 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS *must* be set\n"));
348 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
349 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS *must* be set\n"));
350 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT)
351 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT *must* be set\n"));
352 if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
353 LogRel(("HWACCM: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT *must* be set\n"));
354
355 LogRel(("HWACCM: MSR_IA32_VMX_ENTRY_CTLS = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_entry));
356 val = pVM->hwaccm.s.vmx.msr.vmx_entry >> 32ULL;
357 if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE)
358 LogRel(("HWACCM: VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE\n"));
359 if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM)
360 LogRel(("HWACCM: VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM\n"));
361 if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON)
362 LogRel(("HWACCM: VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON\n"));
363 val = pVM->hwaccm.s.vmx.msr.vmx_entry;
364 if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE)
365 LogRel(("HWACCM: VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE *must* be set\n"));
366 if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM)
367 LogRel(("HWACCM: VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM *must* be set\n"));
368 if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON)
369 LogRel(("HWACCM: VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON *must* be set\n"));
370
371 LogRel(("HWACCM: MSR_IA32_VMX_EXIT_CTLS = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_exit));
372 val = pVM->hwaccm.s.vmx.msr.vmx_exit >> 32ULL;
373 if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64)
374 LogRel(("HWACCM: VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64\n"));
375 if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ)
376 LogRel(("HWACCM: VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ\n"));
377 val = pVM->hwaccm.s.vmx.msr.vmx_exit;
378 if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64)
379 LogRel(("HWACCM: VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64 *must* be set\n"));
380 if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ)
381 LogRel(("HWACCM: VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ *must* be set\n"));
382
383 LogRel(("HWACCM: MSR_IA32_VMX_MISC = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_misc));
384 LogRel(("HWACCM: MSR_IA32_VMX_MISC_ACTIVITY_STATES %x\n", MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hwaccm.s.vmx.msr.vmx_misc)));
385 LogRel(("HWACCM: MSR_IA32_VMX_MISC_CR3_TARGET %x\n", MSR_IA32_VMX_MISC_CR3_TARGET(pVM->hwaccm.s.vmx.msr.vmx_misc)));
386 LogRel(("HWACCM: MSR_IA32_VMX_MISC_MAX_MSR %x\n", MSR_IA32_VMX_MISC_MAX_MSR(pVM->hwaccm.s.vmx.msr.vmx_misc)));
387 LogRel(("HWACCM: MSR_IA32_VMX_MISC_MSEG_ID %x\n", MSR_IA32_VMX_MISC_MSEG_ID(pVM->hwaccm.s.vmx.msr.vmx_misc)));
388
389 LogRel(("HWACCM: MSR_IA32_VMX_CR0_FIXED0 = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0));
390 LogRel(("HWACCM: MSR_IA32_VMX_CR0_FIXED1 = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1));
391 LogRel(("HWACCM: MSR_IA32_VMX_CR4_FIXED0 = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0));
392 LogRel(("HWACCM: MSR_IA32_VMX_CR4_FIXED1 = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1));
393 LogRel(("HWACCM: MSR_IA32_VMX_VMCS_ENUM = %VX64\n", pVM->hwaccm.s.vmx.msr.vmx_vmcs_enum));
394
395 /* Only try once. */
396 pVM->hwaccm.s.fInitialized = true;
397
398 rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_HWACC_SETUP_VM, 0, NULL);
399 AssertRC(rc);
400 if (rc == VINF_SUCCESS)
401 {
402 hwaccmr3DisableRawMode(pVM);
403
404 pVM->fHWACCMEnabled = true;
405 pVM->hwaccm.s.vmx.fEnabled = true;
406 CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
407#if 0 /* not yet */
408 CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
409#endif
410 LogRel(("HWACCM: VMX enabled!\n"));
411 }
412 else
413 {
414 LogRel(("HWACCM: VMX setup failed with rc=%Vrc!\n", rc));
415 LogRel(("HWACCM: Last instruction error %x\n", pVM->hwaccm.s.vmx.ulLastInstrError));
416 pVM->fHWACCMEnabled = false;
417 }
418 }
419 }
420 else
421 if (pVM->hwaccm.s.svm.fSupported)
422 {
423 Log(("pVM->hwaccm.s.svm.fSupported = %d\n", pVM->hwaccm.s.svm.fSupported));
424
425 if (pVM->hwaccm.s.fInitialized == false)
426 {
427 LogRel(("HWACMM: cpuid 0x80000001.u32AMDFeatureECX = %VX32\n", pVM->hwaccm.s.cpuid.u32AMDFeatureECX));
428 LogRel(("HWACMM: cpuid 0x80000001.u32AMDFeatureEDX = %VX32\n", pVM->hwaccm.s.cpuid.u32AMDFeatureEDX));
429 LogRel(("HWACCM: SVM revision = %X\n", pVM->hwaccm.s.svm.u32Rev));
430 LogRel(("HWACCM: SVM max ASID = %d\n", pVM->hwaccm.s.svm.u32MaxASID));
431
432 /* Only try once. */
433 pVM->hwaccm.s.fInitialized = true;
434
435 rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_HWACC_SETUP_VM, 0, NULL);
436 AssertRC(rc);
437 if (rc == VINF_SUCCESS)
438 {
439 hwaccmr3DisableRawMode(pVM);
440 CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
441#if 0 /* not yet */
442 CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
443#endif
444
445 pVM->fHWACCMEnabled = true;
446 pVM->hwaccm.s.svm.fEnabled = true;
447 }
448 else
449 {
450 pVM->fHWACCMEnabled = false;
451 }
452 }
453 }
454 return VINF_SUCCESS;
455}
456
457/**
458 * Applies relocations to data and code managed by this
459 * component. This function will be called at init and
460 * whenever the VMM need to relocate it self inside the GC.
461 *
462 * @param pVM The VM.
463 */
464HWACCMR3DECL(void) HWACCMR3Relocate(PVM pVM)
465{
466 Log(("HWACCMR3Relocate to %VGv\n", MMHyperGetArea(pVM, 0)));
467 return;
468}
469
470
471/**
472 * Checks hardware accelerated raw mode is allowed.
473 *
474 * @returns boolean
475 * @param pVM The VM to operate on.
476 */
477HWACCMR3DECL(bool) HWACCMR3IsAllowed(PVM pVM)
478{
479 return pVM->hwaccm.s.fAllowed;
480}
481
482
483/**
484 * Notification callback which is called whenever there is a chance that a CR3
485 * value might have changed.
486 * This is called by PGM.
487 *
488 * @param pVM The VM to operate on.
489 * @param enmShadowMode New paging mode.
490 */
491HWACCMR3DECL(void) HWACCMR3PagingModeChanged(PVM pVM, PGMMODE enmShadowMode)
492{
493 pVM->hwaccm.s.enmShadowMode = enmShadowMode;
494}
495
496/**
497 * Terminates the HWACCM.
498 *
499 * Termination means cleaning up and freeing all resources,
500 * the VM it self is at this point powered off or suspended.
501 *
502 * @returns VBox status code.
503 * @param pVM The VM to operate on.
504 */
505HWACCMR3DECL(int) HWACCMR3Term(PVM pVM)
506{
507 if (pVM->hwaccm.s.pStatExitReason)
508 {
509 MMHyperFree(pVM, pVM->hwaccm.s.pStatExitReason);
510 pVM->hwaccm.s.pStatExitReason = 0;
511 }
512 return 0;
513}
514
515
516/**
517 * The VM is being reset.
518 *
519 * For the HWACCM component this means that any GDT/LDT/TSS monitors
520 * needs to be removed.
521 *
522 * @param pVM VM handle.
523 */
524HWACCMR3DECL(void) HWACCMR3Reset(PVM pVM)
525{
526 LogFlow(("HWACCMR3Reset:\n"));
527
528 if (pVM->fHWACCMEnabled)
529 hwaccmr3DisableRawMode(pVM);
530
531 /* On first entry we'll sync everything. */
532 pVM->hwaccm.s.fContextUseFlags = HWACCM_CHANGED_ALL;
533
534 pVM->hwaccm.s.vmx.cr0_mask = 0;
535 pVM->hwaccm.s.vmx.cr4_mask = 0;
536
537 pVM->hwaccm.s.Event.fPending = false;
538}
539
540/**
541 * Checks if we can currently use hardware accelerated raw mode.
542 *
543 * @returns boolean
544 * @param pVM The VM to operate on.
545 * @param pCtx Partial VM execution context
546 */
547HWACCMR3DECL(bool) HWACCMR3CanExecuteGuest(PVM pVM, PCPUMCTX pCtx)
548{
549 Assert(pVM->fHWACCMEnabled);
550
551 /* AMD SVM supports real & protected mode with or without paging. */
552 if (pVM->hwaccm.s.svm.fEnabled)
553 {
554 pVM->hwaccm.s.fActive = true;
555 return true;
556 }
557
558 /* @todo we can support real-mode by using v86 and protected mode without paging with identity mapped pages.
559 * (but do we really care?)
560 */
561
562 pVM->hwaccm.s.fActive = false;
563
564 /** @note The context supplied by REM is partial. If we add more checks here, be sure to verify that REM provides this info! */
565
566#ifndef HWACCM_VMX_EMULATE_ALL
567 /* Too early for VMX. */
568 if (pCtx->idtr.pIdt == 0 || pCtx->idtr.cbIdt == 0 || pCtx->tr == 0)
569 return false;
570
571 /* The guest is about to complete the switch to protected mode. Wait a bit longer. */
572 if (pCtx->csHid.Attr.n.u1Present == 0)
573 return false;
574 if (pCtx->ssHid.Attr.n.u1Present == 0)
575 return false;
576#endif
577
578 if (pVM->hwaccm.s.vmx.fEnabled)
579 {
580 uint32_t mask;
581
582 /* if bit N is set in cr0_fixed0, then it must be set in the guest's cr0. */
583 mask = (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0;
584 /* Note: We ignore the NE bit here on purpose; see vmmr0\hwaccmr0.cpp for details. */
585 mask &= ~X86_CR0_NE;
586#ifdef HWACCM_VMX_EMULATE_ALL
587 /* Note: We ignore the PE & PG bits here on purpose; we emulate real and protected mode without paging. */
588 mask &= ~(X86_CR0_PG|X86_CR0_PE);
589#endif
590 if ((pCtx->cr0 & mask) != mask)
591 return false;
592
593 /* if bit N is cleared in cr0_fixed1, then it must be zero in the guest's cr0. */
594 mask = (uint32_t)~pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1;
595 if ((pCtx->cr0 & mask) != 0)
596 return false;
597
598 /* if bit N is set in cr4_fixed0, then it must be set in the guest's cr4. */
599 mask = (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
600 mask &= ~X86_CR4_VMXE;
601 if ((pCtx->cr4 & mask) != mask)
602 return false;
603
604 /* if bit N is cleared in cr4_fixed1, then it must be zero in the guest's cr4. */
605 mask = (uint32_t)~pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1;
606 if ((pCtx->cr4 & mask) != 0)
607 return false;
608
609 pVM->hwaccm.s.fActive = true;
610 return true;
611 }
612
613 return false;
614}
615
616/**
617 * Checks if we are currently using hardware accelerated raw mode.
618 *
619 * @returns boolean
620 * @param pVM The VM to operate on.
621 */
622HWACCMR3DECL(bool) HWACCMR3IsActive(PVM pVM)
623{
624 return pVM->hwaccm.s.fActive;
625}
626
627/**
628 * Checks if internal events are pending. In that case we are not allowed to dispatch interrupts.
629 *
630 * @returns boolean
631 * @param pVM The VM to operate on.
632 */
633HWACCMR3DECL(bool) HWACCMR3IsEventPending(PVM pVM)
634{
635 return HWACCMIsEnabled(pVM) && pVM->hwaccm.s.Event.fPending;
636}
637
638/**
639 * Execute state save operation.
640 *
641 * @returns VBox status code.
642 * @param pVM VM Handle.
643 * @param pSSM SSM operation handle.
644 */
645static DECLCALLBACK(int) hwaccmR3Save(PVM pVM, PSSMHANDLE pSSM)
646{
647 int rc;
648
649 Log(("hwaccmR3Save:\n"));
650
651 /*
652 * Save the basic bits - fortunately all the other things can be resynced on load.
653 */
654 rc = SSMR3PutU32(pSSM, pVM->hwaccm.s.Event.fPending);
655 AssertRCReturn(rc, rc);
656 rc = SSMR3PutU32(pSSM, pVM->hwaccm.s.Event.errCode);
657 AssertRCReturn(rc, rc);
658 rc = SSMR3PutU64(pSSM, pVM->hwaccm.s.Event.intInfo);
659 AssertRCReturn(rc, rc);
660
661 return VINF_SUCCESS;
662}
663
664
665/**
666 * Execute state load operation.
667 *
668 * @returns VBox status code.
669 * @param pVM VM Handle.
670 * @param pSSM SSM operation handle.
671 * @param u32Version Data layout version.
672 */
673static DECLCALLBACK(int) hwaccmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version)
674{
675 int rc;
676
677 Log(("hwaccmR3Load:\n"));
678
679 /*
680 * Validate version.
681 */
682 if (u32Version != HWACCM_SSM_VERSION)
683 {
684 Log(("hwaccmR3Load: Invalid version u32Version=%d!\n", u32Version));
685 return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
686 }
687 rc = SSMR3GetU32(pSSM, &pVM->hwaccm.s.Event.fPending);
688 AssertRCReturn(rc, rc);
689 rc = SSMR3GetU32(pSSM, &pVM->hwaccm.s.Event.errCode);
690 AssertRCReturn(rc, rc);
691 rc = SSMR3GetU64(pSSM, &pVM->hwaccm.s.Event.intInfo);
692 AssertRCReturn(rc, rc);
693
694 return VINF_SUCCESS;
695}
696
697
698
699
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette