testboxscript_real.py@ 97271

Last change on this file since 97271 was 96407, checked in by vboxsync, 2 years ago
scm copyright and license note update
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 47.7 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: testboxscript_real.py 96407 2022-08-22 17:43:14Z vboxsync $
4
5	"""
6	TestBox Script - main().
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2022 Oracle and/or its affiliates.
12
13	This file is part of VirtualBox base platform packages, as
14	available from https://www.virtualbox.org.
15
16	This program is free software; you can redistribute it and/or
17	modify it under the terms of the GNU General Public License
18	as published by the Free Software Foundation, in version 3 of the
19	License.
20
21	This program is distributed in the hope that it will be useful, but
22	WITHOUT ANY WARRANTY; without even the implied warranty of
23	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	General Public License for more details.
25
26	You should have received a copy of the GNU General Public License
27	along with this program; if not, see <https://www.gnu.org/licenses>.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32	in the VirtualBox distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37
38	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39	"""
40	__version__ = "$Revision: 96407 $"
41
42
43	# Standard python imports.
44	import math
45	import os
46	from optparse import OptionParser # pylint: disable=deprecated-module
47	import platform
48	import random
49	import shutil
50	import sys
51	import tempfile
52	import time
53	import uuid
54
55	# Only the main script needs to modify the path.
56	try: __file__
57	except: __file__ = sys.argv[0];
58	g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
59	g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
60	sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
61
62	# Validation Kit imports.
63	from common import constants;
64	from common import utils;
65	import testboxcommons;
66	from testboxcommons import TestBoxException;
67	from testboxcommand import TestBoxCommand;
68	from testboxconnection import TestBoxConnection;
69	from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
70
71	# Python 3 hacks:
72	if sys.version_info[0] >= 3:
73	long = int; # pylint: disable=redefined-builtin,invalid-name
74
75
76	class TestBoxScriptException(Exception):
77	""" For raising exceptions during TestBoxScript.__init__. """
78	pass; # pylint: disable=unnecessary-pass
79
80
81	class TestBoxScript(object):
82	"""
83	Implementation of the test box script.
84	Communicate with test manager and perform offered actions.
85	"""
86
87	## @name Class Constants.
88	# @{
89
90	# Scratch space round value (MB).
91	kcMbScratchSpaceRounding = 64
92	# Memory size round value (MB).
93	kcMbMemoryRounding = 4
94	# A NULL UUID in string form.
95	ksNullUuid = '00000000-0000-0000-0000-000000000000';
96	# The minimum dispatch loop delay.
97	kcSecMinDelay = 12;
98	# The maximum dispatch loop delay (inclusive).
99	kcSecMaxDelay = 24;
100	# The minimum sign-on delay.
101	kcSecMinSignOnDelay = 30;
102	# The maximum sign-on delay (inclusive).
103	kcSecMaxSignOnDelay = 60;
104
105	# Keys for config params
106	VALUE = 'value'
107	FN = 'fn' # pylint: disable=invalid-name
108
109	## @}
110
111
112	def __init__(self, oOptions):
113	"""
114	Initialize internals
115	"""
116	self._oOptions = oOptions;
117	self._sTestBoxHelper = None;
118
119	# Signed-on state
120	self._cSignOnAttempts = 0;
121	self._fSignedOn = False;
122	self._fNeedReSignOn = False;
123	self._fFirstSignOn = True;
124	self._idTestBox = None;
125	self._sTestBoxName = '';
126	self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
127
128	# Command processor.
129	self._oCommand = TestBoxCommand(self);
130
131	#
132	# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
133	# many many GBs for some test scenarios and /tmp can be backed by swap
134	# or be a fast+small disk of some kind, while /var/tmp is normally
135	# larger, if slower. /var/tmp is generally not cleaned up on reboot,
136	# /tmp often is, this would break host panic / triple-fault detection.
137	#
138	if self._oOptions.sScratchRoot is None:
139	if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
140	# We need lots of space, so avoid /tmp as it may be a memory
141	# file system backed by the swap file, or worse.
142	self._oOptions.sScratchRoot = tempfile.gettempdir();
143	else:
144	self._oOptions.sScratchRoot = '/var/tmp';
145	sSubDir = 'testbox';
146	try:
147	sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=no-member
148	except:
149	pass;
150	self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
151
152	self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
153	self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
154	self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
155
156	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
157	if not os.path.isdir(sDir):
158	os.makedirs(sDir, 0o700);
159
160	# We count consecutive reinitScratch failures and will reboot the
161	# testbox after a while in the hope that it will correct the issue.
162	self._cReinitScratchErrors = 0;
163
164	#
165	# Mount builds and test resources if requested.
166	#
167	self.mountShares();
168
169	#
170	# Sign-on parameters: Packed into list of records of format:
171	# { <Parameter ID>: { <Current value>, <Check function> } }
172	#
173	self._ddSignOnParams = \
174	{
175	constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
176	constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
177	constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
178	constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
179	constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
180	constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
181	constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
182	constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
183	constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
184	constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
185	constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
186	#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
187	constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
188	constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
189	constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
190	constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: utils.getPresentCpuCount },
191	constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
192	constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
193	}
194	for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
195	if self._ddSignOnParams[sItem][self.FN] is not None:
196	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
197
198	testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
199	testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
200	testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
201	for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
202	testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
203
204	#
205	# The System UUID is the primary identification of the machine, so
206	# refuse to cooperate if it's NULL.
207	#
208	self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
209	if self._sTestBoxUuid == self.ksNullUuid:
210	raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
211
212	#
213	# Export environment variables, clearing any we don't know yet.
214	#
215	for sEnvVar in self._oOptions.asEnvVars:
216	iEqual = sEnvVar.find('=');
217	if iEqual == -1: # No '=', remove it.
218	if sEnvVar in os.environ:
219	del os.environ[sEnvVar];
220	elif iEqual > 0: # Set it.
221	os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
222	else: # Starts with '=', bad user.
223	raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
224
225	os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
226	os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
227	os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
228	os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
229	os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
230	os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
231	os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
232	os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
233	os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
234	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
235	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
236	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
237	#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
238	os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
239	os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
240	os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
241	os.environ['TESTBOX_REPORTER'] = 'remote';
242	os.environ['TESTBOX_NAME'] = '';
243	os.environ['TESTBOX_ID'] = '';
244	os.environ['TESTBOX_TEST_SET_ID'] = '';
245	os.environ['TESTBOX_TIMEOUT'] = '0';
246	os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
247
248	if utils.getHostOs() == 'win':
249	os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
250	# Currently omitting any kBuild tools.
251
252	def mountShares(self):
253	"""
254	Mounts the shares.
255	Raises exception on failure.
256	"""
257	self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
258	self._oOptions.sBuildsServerShare,
259	self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd,
260	self._oOptions.sBuildsServerMountOpt, 'builds');
261	self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
262	self._oOptions.sTestRsrcServerShare,
263	self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd,
264	self._oOptions.sTestRsrcServerMountOpt, 'testrsrc');
265	return True;
266
267	def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sMountOpt, sWhat):
268	"""
269	Mounts the specified share if needed.
270	Raises exception on failure.
271	"""
272	# Only mount if the type is specified.
273	if sType is None:
274	return True;
275
276	# Test if already mounted.
277	sTestFile = os.path.join(sMountPoint + os.path.sep, os.path.basename(sShare) + '-new.txt');
278	if os.path.isfile(sTestFile):
279	return True;
280
281	#
282	# Platform specific mount code.
283	#
284	sHostOs = utils.getHostOs()
285	if sHostOs in ('darwin', 'freebsd'):
286	if sMountOpt != '':
287	sMountOpt = ',' + sMountOpt
288	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
289	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
290	utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=no-member
291	if sType == 'cifs':
292	# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
293	utils.processOutputChecked(['/sbin/mount_smbfs',
294	'-o',
295	'automounted,nostreams,soft,noowners,noatime,rdonly' + sMountOpt,
296	'-f', '0555', '-d', '0555',
297	'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
298	sMountPoint]);
299	else:
300	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
301
302	elif sHostOs == 'linux':
303	if sMountOpt != '':
304	sMountOpt = ',' + sMountOpt
305	utils.sudoProcessCall(['/bin/umount', sMountPoint]);
306	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
307	if sType == 'cifs':
308	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
309	'-o',
310	'user=' + sUser
311	+ ',password=' + sPassword
312	+ ',sec=ntlmv2'
313	+ ',uid=' + str(os.getuid()) # pylint: disable=no-member
314	+ ',gid=' + str(os.getgid()) # pylint: disable=no-member
315	+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro'
316	+ sMountOpt,
317	'//%s/%s' % (sServer, sShare),
318	sMountPoint]);
319	elif sType == 'nfs':
320	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
321	'-o', 'soft,ro' + sMountOpt,
322	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
323	sMountPoint]);
324
325	else:
326	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
327
328	elif sHostOs == 'solaris':
329	if sMountOpt != '':
330	sMountOpt = ',' + sMountOpt
331	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
332	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
333	if sType == 'cifs':
334	## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
335	oPasswdFile = tempfile.TemporaryFile(); # pylint: disable=consider-using-with
336	oPasswdFile.write(sPassword + '\n');
337	oPasswdFile.flush();
338	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
339	'-o',
340	'user=' + sUser
341	+ ',uid=' + str(os.getuid()) # pylint: disable=no-member
342	+ ',gid=' + str(os.getgid()) # pylint: disable=no-member
343	+ ',fileperms=0555,dirperms=0555,noxattr,ro'
344	+ sMountOpt,
345	'//%s/%s' % (sServer, sShare),
346	sMountPoint],
347	stdin = oPasswdFile);
348	oPasswdFile.close();
349	elif sType == 'nfs':
350	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
351	'-o', 'noxattr,ro' + sMountOpt,
352	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
353	sMountPoint]);
354
355	else:
356	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
357
358
359	elif sHostOs == 'win':
360	if sType != 'cifs':
361	raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
362	utils.processCall(['net', 'use', sMountPoint, '/d']);
363	utils.processOutputChecked(['net', 'use', sMountPoint,
364	'\\\\' + sServer + '\\' + sShare,
365	sPassword,
366	'/USER:' + sUser,]);
367	else:
368	raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
369
370	#
371	# Re-test.
372	#
373	if not os.path.isfile(sTestFile):
374	raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
375	% (sWhat, sServer, sShare, sMountPoint, sTestFile));
376
377	return True;
378
379	## @name Signon property releated methods.
380	# @{
381
382	def _getHelperOutput(self, sCmd):
383	"""
384	Invokes TestBoxHelper to obtain information hard to access from python.
385	"""
386	if self._sTestBoxHelper is None:
387	if not utils.isRunningFromCheckout():
388	# See VBoxTestBoxScript.zip for layout.
389	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
390	'TestBoxHelper');
391	else: # Only for in-tree testing, so don't bother be too accurate right now.
392	sType = os.environ.get('KBUILD_TYPE', 'debug');
393	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
394	utils.getHostOsDotArch(), sType, 'testboxscript', \
395	utils.getHostOs(), utils.getHostArch(), \
396	'TestBoxHelper');
397	if utils.getHostOs() in ['win', 'os2']:
398	self._sTestBoxHelper += '.exe';
399
400	return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
401
402	def _getHelperOutputTristate(self, sCmd, fDunnoValue):
403	"""
404	Invokes TestBoxHelper to obtain information hard to access from python.
405	"""
406	sValue = self._getHelperOutput(sCmd);
407	sValue = sValue.lower();
408	if sValue == 'true':
409	return True;
410	if sValue == 'false':
411	return False;
412	if sValue not in ('dunno', 'none',):
413	raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
414	return fDunnoValue;
415
416
417	@staticmethod
418	def _isUuidGood(sUuid):
419	"""
420	Checks if the UUID looks good.
421
422	There are systems with really bad UUIDs, for instance
423	"03000200-0400-0500-0006-000700080009".
424	"""
425	if sUuid == TestBoxScript.ksNullUuid:
426	return False;
427	sUuid = sUuid.lower();
428	for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
429	if sUuid.count(sDigit) > 16:
430	return False;
431	return True;
432
433	def _getHostSystemUuid(self):
434	"""
435	Get the system UUID string from the System, return null-uuid if
436	unable to get retrieve it.
437	"""
438	if self._oOptions.sSystemUuid is not None:
439	return self._oOptions.sSystemUuid;
440
441	sUuid = self.ksNullUuid;
442
443	#
444	# Try get at the firmware UUID.
445	#
446	if utils.getHostOs() == 'linux':
447	# NOTE: This requires to have kernel option enabled:
448	# Firmware Drivers -> Export DMI identification via sysfs to userspace
449	if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
450	try:
451	sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
452	sUuid = str(uuid.UUID(sVar.strip()));
453	except:
454	pass;
455	## @todo consider dmidecoder? What about EFI systems?
456
457	elif utils.getHostOs() == 'win':
458	# Windows: WMI
459	try:
460	import win32com.client; # pylint: disable=import-error
461	oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
462	oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
463	for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
464	if oItem.UUID is not None:
465	sUuid = str(uuid.UUID(oItem.UUID));
466	except:
467	pass;
468
469	elif utils.getHostOs() == 'darwin':
470	try:
471	sVar = utils.processOutputChecked(['/bin/sh', '-c',
472	'/usr/sbin/ioreg -k IOPlatformUUID' \
473	+ '\| /usr/bin/grep IOPlatformUUID' \
474	+ '\| /usr/bin/head -1']);
475	sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
476	sUuid = str(uuid.UUID(sVar));
477	except:
478	pass;
479
480	elif utils.getHostOs() == 'solaris':
481	# Solaris: The smbios util.
482	try:
483	sVar = utils.processOutputChecked(['/bin/sh', '-c',
484	'/usr/sbin/smbios ' \
485	+ '\| /usr/xpg4/bin/sed -ne \'s/^.UUID: //p\'' \
486	+ '\| /usr/bin/head -1']);
487	sUuid = str(uuid.UUID(sVar.strip()));
488	except:
489	pass;
490
491	if self._isUuidGood(sUuid):
492	return sUuid;
493
494	#
495	# Try add the MAC address.
496	# uuid.getnode may provide it, or it may return a random number...
497	#
498	lMacAddr = uuid.getnode();
499	sNode = '%012x' % (lMacAddr,)
500	if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
501	return sUuid[:-12] + sNode;
502
503	return sUuid;
504
505	def _getHostCpuVendor(self):
506	"""
507	Get the CPUID vendor string on intel HW.
508	"""
509	return self._getHelperOutput('cpuvendor');
510
511	def _getHostCpuName(self):
512	"""
513	Get the CPU name/description string.
514	"""
515	return self._getHelperOutput('cpuname');
516
517	def _getHostCpuRevision(self):
518	"""
519	Get the CPU revision (family/model/stepping) value.
520	"""
521	return self._getHelperOutput('cpurevision');
522
523	def _hasHostHwVirt(self):
524	"""
525	Check if the host supports AMD-V or VT-x
526	"""
527	if self._oOptions.fHasHwVirt is None:
528	self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
529	return self._oOptions.fHasHwVirt;
530
531	def _hasHostNestedPaging(self):
532	"""
533	Check if the host supports nested paging.
534	"""
535	if not self._hasHostHwVirt():
536	return False;
537	if self._oOptions.fHasNestedPaging is None:
538	self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
539	return self._oOptions.fHasNestedPaging;
540
541	def _can64BitGuest(self):
542	"""
543	Check if the we (VBox) can run 64-bit guests.
544	"""
545	if not self._hasHostHwVirt():
546	return False;
547	if self._oOptions.fCan64BitGuest is None:
548	self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
549	return self._oOptions.fCan64BitGuest;
550
551	def _hasHostIoMmu(self):
552	"""
553	Check if the host has an I/O MMU of the VT-d kind.
554	"""
555	if not self._hasHostHwVirt():
556	return False;
557	if self._oOptions.fHasIoMmu is None:
558	## @todo Any way to figure this one out on any host OS?
559	self._oOptions.fHasIoMmu = False;
560	return self._oOptions.fHasIoMmu;
561
562	def _withRawModeSupport(self):
563	"""
564	Check if the testbox is configured with raw-mode support or not.
565	"""
566	if self._oOptions.fWithRawMode is None:
567	self._oOptions.fWithRawMode = True;
568	return self._oOptions.fWithRawMode;
569
570	def _getHostReport(self):
571	"""
572	Generate a report about the host hardware and software.
573	"""
574	return self._getHelperOutput('report');
575
576
577	def _getHostMemSize(self):
578	"""
579	Gets the amount of physical memory on the host (and accessible to the
580	OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
581	Unit: MiB.
582	"""
583	cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
584
585	# Round it.
586	cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
587	return cMbMemory;
588
589	def _getFreeScratchSpace(self):
590	"""
591	Get free space on the volume where scratch directory is located and
592	return it in bytes rounded down to nearest 64MB
593	(currently works on Linux only)
594	Unit: MiB.
595	"""
596	if platform.system() == 'Windows':
597	import ctypes
598	cTypeMbFreeSpace = ctypes.c_ulonglong(0)
599	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
600	ctypes.pointer(cTypeMbFreeSpace))
601	cMbFreeSpace = cTypeMbFreeSpace.value
602	else:
603	stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=no-member
604	cMbFreeSpace = stats.f_frsize * stats.f_bfree
605
606	# Convert to MB
607	cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
608
609	# Round free space size
610	cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
611	return cMbFreeSpace;
612
613	def _getScriptRev(self):
614	"""
615	The script (subversion) revision number.
616	"""
617	sRev = '@VBOX_SVN_REV@';
618	sRev = sRev.strip(); # just in case...
619	try:
620	_ = int(sRev);
621	except:
622	return __version__[11:-1].strip();
623	return sRev;
624
625	def _getPythonHexVersion(self):
626	"""
627	The python hex version number.
628	"""
629	uHexVersion = getattr(sys, 'hexversion', None);
630	if uHexVersion is None:
631	uHexVersion = (sys.version_info[0] << 24) \| (sys.version_info[1] << 16) \| (sys.version_info[2] << 8);
632	if sys.version_info[3] == 'final':
633	uHexVersion \|= 0xf0;
634	return uHexVersion;
635
636	# @}
637
638	def openTestManagerConnection(self):
639	"""
640	Opens up a connection to the test manager.
641
642	Raises exception on failure.
643	"""
644	return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
645
646	def getSignOnParam(self, sName):
647	"""
648	Returns a sign-on parameter value as string.
649	Raises exception if the name is incorrect.
650	"""
651	return str(self._ddSignOnParams[sName][self.VALUE]);
652
653	def getPathState(self):
654	"""
655	Get the path to the state dir in the scratch area.
656	"""
657	return self._sScratchState;
658
659	def getPathScripts(self):
660	"""
661	Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
662	"""
663	return self._sScratchScripts;
664
665	def getPathSpill(self):
666	"""
667	Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
668	"""
669	return self._sScratchSpill;
670
671	def getPathBuilds(self):
672	"""
673	Get the path to the builds.
674	"""
675	return self._oOptions.sBuildsPath;
676
677	def getTestBoxId(self):
678	"""
679	Get the TestBox ID for state saving purposes.
680	"""
681	return self._idTestBox;
682
683	def getTestBoxName(self):
684	"""
685	Get the TestBox name for state saving purposes.
686	"""
687	return self._sTestBoxName;
688
689	def _reinitScratch(self, fnLog, fUseTheForce):
690	"""
691	Wipes the scratch directories and re-initializes them.
692
693	No exceptions raise, returns success indicator instead.
694	"""
695	if fUseTheForce is None:
696	fUseTheForce = self._fFirstSignOn;
697
698	class ErrorCallback(object): # pylint: disable=too-few-public-methods
699	"""
700	Callbacks + state for the cleanup.
701	"""
702	def __init__(self):
703	self.fRc = True;
704	def onErrorCallback(self, sFnName, sPath, aXcptInfo):
705	""" Logs error during shutil.rmtree operation. """
706	fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
707	self.fRc = False;
708	oRc = ErrorCallback();
709
710	#
711	# Cleanup.
712	#
713	for sName in os.listdir(self._oOptions.sScratchRoot):
714	sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
715	try:
716	if os.path.isdir(sFullName):
717	shutil.rmtree(sFullName, False, oRc.onErrorCallback);
718	else:
719	os.remove(sFullName);
720	if os.path.exists(sFullName):
721	raise Exception('Still exists after deletion, weird.');
722	except Exception as oXcpt:
723	if fUseTheForce is True \
724	and utils.getHostOs() not in ['win', 'os2'] \
725	and len(sFullName) >= 8 \
726	and sFullName[0] == '/' \
727	and sFullName[1] != '/' \
728	and sFullName.find('/../') < 0:
729	fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
730	try:
731	if os.path.isdir(sFullName):
732	iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
733	else:
734	iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
735	if iRc != 0:
736	raise Exception('exit code %s' % iRc);
737	if os.path.exists(sFullName):
738	raise Exception('Still exists after forced deletion, weird^2.');
739	except:
740	fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
741	oRc.fRc = False;
742	else:
743	fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
744	oRc.fRc = False;
745
746	# Display files left behind.
747	def dirEnumCallback(sName, oStat):
748	""" callback for dirEnumerateTree """
749	fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName));
750	utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback);
751
752	#
753	# Re-create the directories.
754	#
755	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
756	if not os.path.isdir(sDir):
757	try:
758	os.makedirs(sDir, 0o700);
759	except Exception as oXcpt:
760	fnLog('Error creating "%s": %s' % (sDir, oXcpt));
761	oRc.fRc = False;
762
763	if oRc.fRc is True:
764	self._cReinitScratchErrors = 0;
765	else:
766	self._cReinitScratchErrors += 1;
767	return oRc.fRc;
768
769	def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000):
770	"""
771	Wipes the scratch directories and re-initializes them.
772
773	Will retry according to the cRetries and cMsDelay parameters. Windows
774	forces us to apply this hack as it ships with services asynchronously
775	scanning files after they execute, thus racing us cleaning up after a
776	test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping
777	vts_rm.exe kind of open, somehow preventing us from removing the
778	directory containing it, despite not issuing any errors deleting the
779	file itself. The service is called "Application Experience", which
780	feels like a weird joke here.
781
782	No exceptions raise, returns success indicator instead.
783	"""
784	fRc = self._reinitScratch(fnLog, fUseTheForce)
785	while fRc is False and cRetries > 0:
786	time.sleep(cMsDelay / 1000.0);
787	fnLog('reinitScratch: Retrying...');
788	fRc = self._reinitScratch(fnLog, fUseTheForce)
789	cRetries -= 1;
790	return fRc;
791
792
793	def _doSignOn(self):
794	"""
795	Worker for _maybeSignOn that does the actual signing on.
796	"""
797	assert not self._oCommand.isRunning();
798
799	# Reset the siged-on state.
800	testboxcommons.log('Signing-on...')
801	self._fSignedOn = False
802	self._idTestBox = None
803	self._cSignOnAttempts += 1;
804
805	# Assemble SIGN-ON request parameters and send the request.
806	dParams = {};
807	for sParam in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
808	dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
809	oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
810
811	# Check response.
812	try:
813	sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
814	if sResult != constants.tbresp.STATUS_ACK:
815	raise TestBoxException('Result is %s' % (sResult,));
816	oResponse.checkParameterCount(3);
817	idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
818	sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
819	except TestBoxException as err:
820	testboxcommons.log('Failed to sign-on: %s' % (str(err),))
821	testboxcommons.log('Server response: %s' % (oResponse.toString(),));
822	return False;
823
824	# Successfully signed on, update the state.
825	self._fSignedOn = True;
826	self._fNeedReSignOn = False;
827	self._cSignOnAttempts = 0;
828	self._idTestBox = idTestBox;
829	self._sTestBoxName = sTestBoxName;
830
831	# Update the environment.
832	os.environ['TESTBOX_ID'] = str(self._idTestBox);
833	os.environ['TESTBOX_NAME'] = sTestBoxName;
834	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
835	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
836	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
837
838	testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
839	% (self._idTestBox, self._sTestBoxName));
840
841	# Set up the scratch area.
842	self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2);
843
844	self._fFirstSignOn = False;
845	return True;
846
847	def _maybeSignOn(self):
848	"""
849	Check if Test Box parameters were changed
850	and do sign-in in case of positive result
851	"""
852
853	# Skip sign-on check if background command is currently in
854	# running state (avoid infinite signing on).
855	if self._oCommand.isRunning():
856	return None;
857
858	# Refresh sign-on parameters, changes triggers sign-on.
859	fNeedSignOn = not self._fSignedOn or self._fNeedReSignOn;
860	for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
861	if self._ddSignOnParams[sItem][self.FN] is None:
862	continue
863
864	sOldValue = self._ddSignOnParams[sItem][self.VALUE]
865	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
866	if sOldValue != self._ddSignOnParams[sItem][self.VALUE]:
867	fNeedSignOn = True
868	testboxcommons.log('Detected %s parameter change: %s -> %s'
869	% (sItem, sOldValue, self._ddSignOnParams[sItem][self.VALUE],))
870
871	if fNeedSignOn:
872	self._doSignOn();
873	return None;
874
875	def dispatch(self):
876	"""
877	Receive orders from Test Manager and execute them
878	"""
879
880	(self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
881	self._fNeedReSignOn = self._fSignedOn;
882	if self._fSignedOn:
883	os.environ['TESTBOX_ID'] = str(self._idTestBox);
884	os.environ['TESTBOX_NAME'] = self._sTestBoxName;
885
886	while True:
887	# Make sure we're signed on before trying to do anything.
888	self._maybeSignOn();
889	while not self._fSignedOn:
890	iFactor = 1 if self._cSignOnAttempts < 100 else 4;
891	time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
892	self._maybeSignOn();
893
894	# Retrieve and handle command from the TM.
895	(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
896	self._idTestBox,
897	self._sTestBoxUuid,
898	self._oCommand.isRunning());
899	if oResponse is not None:
900	self._oCommand.handleCommand(oResponse, oConnection);
901	if oConnection is not None:
902	if oConnection.isConnected():
903	self._oCommand.flushLogOnConnection(oConnection);
904	oConnection.close();
905
906	# Automatically reboot if scratch init fails.
907	#if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False:
908	# testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
909	# % ( self._cReinitScratchErrors, ));
910	# self._oCommand.doReboot();
911
912	# delay a wee bit before looping.
913	## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
914	# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
915	iFactor = 1;
916	if self._cReinitScratchErrors > 0:
917	iFactor = 4;
918	time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
919
920	# Not reached.
921
922
923	@staticmethod
924	def main():
925	"""
926	Main function a la C/C++. Returns exit code.
927	"""
928
929	#
930	# Parse arguments.
931	#
932	sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
933	if utils.getHostOs() in ('win', 'os2'):
934	sDefTestRsrc = 'T:';
935	sDefBuilds = 'U:';
936	elif utils.getHostOs() == 'darwin':
937	sDefTestRsrc = '/Volumes/testrsrc';
938	sDefBuilds = '/Volumes/builds';
939	else:
940	sDefTestRsrc = '/mnt/testrsrc';
941	sDefBuilds = '/mnt/builds';
942
943	class MyOptionParser(OptionParser):
944	""" We need to override the exit code on --help, error and so on. """
945	def __init__(self, args, *kwargs):
946	OptionParser.__init__(self, args, *kwargs);
947	def exit(self, status = 0, msg = None):
948	OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
949
950	parser = MyOptionParser(version=__version__[11:-1].strip());
951	for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
952	sLower = sMixed.lower();
953	sPrefix = 's' + sMixed;
954	parser.add_option('--' + sLower + '-path',
955	dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
956	help='Where ' + sDesc + ' can be found');
957	parser.add_option('--' + sLower + '-server-type',
958	dest=sPrefix + 'ServerType', metavar='<nfs\|cifs>', default=sDefShareType,
959	help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
960	parser.add_option('--' + sLower + '-server-name',
961	dest=sPrefix + 'ServerName', metavar='<server>',
962	default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
963	help='The name of the server with the builds.');
964	parser.add_option('--' + sLower + '-server-share',
965	dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
966	help='The name of the builds share.');
967	parser.add_option('--' + sLower + '-server-user',
968	dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
969	help='The user name to use when accessing the ' + sDesc + ' share.');
970	parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
971	dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
972	help='The password to use when accessing the ' + sDesc + ' share.');
973	parser.add_option('--' + sLower + '-server-mountopt',
974	dest=sPrefix + 'ServerMountOpt', metavar='<mountopt>', default='',
975	help='The mount options to use when accessing the ' + sDesc + ' share.');
976
977	parser.add_option("--test-manager", metavar="<url>",
978	dest="sTestManagerUrl",
979	help="Test Manager URL",
980	default="http://tindertux.de.oracle.com/testmanager")
981	parser.add_option("--scratch-root", metavar="<abs-path>",
982	dest="sScratchRoot",
983	help="Path to the scratch directory",
984	default=None)
985	parser.add_option("--system-uuid", metavar="<uuid>",
986	dest="sSystemUuid",
987	help="The system UUID of the testbox, used for uniquely identifiying the machine",
988	default=None)
989	parser.add_option("--hwvirt",
990	dest="fHasHwVirt", action="store_true", default=None,
991	help="Hardware virtualization available in the CPU");
992	parser.add_option("--no-hwvirt",
993	dest="fHasHwVirt", action="store_false", default=None,
994	help="Hardware virtualization not available in the CPU");
995	parser.add_option("--nested-paging",
996	dest="fHasNestedPaging", action="store_true", default=None,
997	help="Nested paging is available");
998	parser.add_option("--no-nested-paging",
999	dest="fHasNestedPaging", action="store_false", default=None,
1000	help="Nested paging is not available");
1001	parser.add_option("--64-bit-guest",
1002	dest="fCan64BitGuest", action="store_true", default=None,
1003	help="Host can execute 64-bit guests");
1004	parser.add_option("--no-64-bit-guest",
1005	dest="fCan64BitGuest", action="store_false", default=None,
1006	help="Host cannot execute 64-bit guests");
1007	parser.add_option("--io-mmu",
1008	dest="fHasIoMmu", action="store_true", default=None,
1009	help="I/O MMU available");
1010	parser.add_option("--no-io-mmu",
1011	dest="fHasIoMmu", action="store_false", default=None,
1012	help="No I/O MMU available");
1013	parser.add_option("--raw-mode",
1014	dest="fWithRawMode", action="store_true", default=None,
1015	help="Use raw-mode on this host.");
1016	parser.add_option("--no-raw-mode",
1017	dest="fWithRawMode", action="store_false", default=None,
1018	help="Disables raw-mode tests on this host.");
1019	parser.add_option("--pidfile",
1020	dest="sPidFile", default=None,
1021	help="For the parent script, ignored.");
1022	parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
1023	dest = "asEnvVars", default = [],
1024	help = "Sets an environment variable. Can be repeated.");
1025	def sbp_callback(option, opt_str, value, parser):
1026	_, _, _ = opt_str, value, option
1027	parser.values.sTestManagerUrl = 'http://10.162.100.8/testmanager/'
1028	parser.values.sBuildsServerName = 'vbox-st02.ru.oracle.com'
1029	parser.values.sTestRsrcServerName = 'vbox-st02.ru.oracle.com'
1030	parser.values.sTestRsrcServerShare = 'scratch/data/testrsrc'
1031	parser.add_option("--spb", "--load-sbp-defaults", action="callback", callback=sbp_callback,
1032	help="Load defaults for the sbp setup.")
1033
1034	(oOptions, args) = parser.parse_args()
1035	# Check command line
1036	if args != []:
1037	parser.print_help();
1038	return TBS_EXITCODE_SYNTAX;
1039
1040	if oOptions.sSystemUuid is not None:
1041	uuid.UUID(oOptions.sSystemUuid);
1042	if not oOptions.sTestManagerUrl.startswith('http://') \
1043	and not oOptions.sTestManagerUrl.startswith('https://'):
1044	print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
1045	return TBS_EXITCODE_SYNTAX;
1046
1047	for sPrefix in ['sBuilds', 'sTestRsrc']:
1048	sType = getattr(oOptions, sPrefix + 'ServerType');
1049	if sType is None or not sType.strip():
1050	setattr(oOptions, sPrefix + 'ServerType', None);
1051	elif sType not in ['cifs', 'nfs']:
1052	print('Syntax error: Invalid server type "%s"' % (sType,));
1053	return TBS_EXITCODE_SYNTAX;
1054
1055
1056	#
1057	# Instantiate the testbox script and start dispatching work.
1058	#
1059	try:
1060	oTestBoxScript = TestBoxScript(oOptions);
1061	except TestBoxScriptException as oXcpt:
1062	print('Error: %s' % (oXcpt,));
1063	return TBS_EXITCODE_SYNTAX;
1064	oTestBoxScript.dispatch();
1065
1066	# Not supposed to get here...
1067	return TBS_EXITCODE_FAILURE;
1068
1069
1070
1071	if __name__ == '__main__':
1072	sys.exit(TestBoxScript.main());
1073

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 97271

Download in other formats: