testboxscript_real.py@ 93115

Last change on this file since 93115 was 93115, checked in by vboxsync, 3 years ago
scm --update-copyright-year
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 47.1 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: testboxscript_real.py 93115 2022-01-01 11:31:46Z vboxsync $
4
5	"""
6	TestBox Script - main().
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2022 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 93115 $"
31
32
33	# Standard python imports.
34	import math
35	import os
36	from optparse import OptionParser # pylint: disable=deprecated-module
37	import platform
38	import random
39	import shutil
40	import sys
41	import tempfile
42	import time
43	import uuid
44
45	# Only the main script needs to modify the path.
46	try: __file__
47	except: __file__ = sys.argv[0];
48	g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
49	g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
50	sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
51
52	# Validation Kit imports.
53	from common import constants;
54	from common import utils;
55	import testboxcommons;
56	from testboxcommons import TestBoxException;
57	from testboxcommand import TestBoxCommand;
58	from testboxconnection import TestBoxConnection;
59	from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
60
61	# Python 3 hacks:
62	if sys.version_info[0] >= 3:
63	long = int; # pylint: disable=redefined-builtin,invalid-name
64
65
66	class TestBoxScriptException(Exception):
67	""" For raising exceptions during TestBoxScript.__init__. """
68	pass; # pylint: disable=unnecessary-pass
69
70
71	class TestBoxScript(object):
72	"""
73	Implementation of the test box script.
74	Communicate with test manager and perform offered actions.
75	"""
76
77	## @name Class Constants.
78	# @{
79
80	# Scratch space round value (MB).
81	kcMbScratchSpaceRounding = 64
82	# Memory size round value (MB).
83	kcMbMemoryRounding = 4
84	# A NULL UUID in string form.
85	ksNullUuid = '00000000-0000-0000-0000-000000000000';
86	# The minimum dispatch loop delay.
87	kcSecMinDelay = 12;
88	# The maximum dispatch loop delay (inclusive).
89	kcSecMaxDelay = 24;
90	# The minimum sign-on delay.
91	kcSecMinSignOnDelay = 30;
92	# The maximum sign-on delay (inclusive).
93	kcSecMaxSignOnDelay = 60;
94
95	# Keys for config params
96	VALUE = 'value'
97	FN = 'fn' # pylint: disable=invalid-name
98
99	## @}
100
101
102	def __init__(self, oOptions):
103	"""
104	Initialize internals
105	"""
106	self._oOptions = oOptions;
107	self._sTestBoxHelper = None;
108
109	# Signed-on state
110	self._cSignOnAttempts = 0;
111	self._fSignedOn = False;
112	self._fNeedReSignOn = False;
113	self._fFirstSignOn = True;
114	self._idTestBox = None;
115	self._sTestBoxName = '';
116	self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
117
118	# Command processor.
119	self._oCommand = TestBoxCommand(self);
120
121	#
122	# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
123	# many many GBs for some test scenarios and /tmp can be backed by swap
124	# or be a fast+small disk of some kind, while /var/tmp is normally
125	# larger, if slower. /var/tmp is generally not cleaned up on reboot,
126	# /tmp often is, this would break host panic / triple-fault detection.
127	#
128	if self._oOptions.sScratchRoot is None:
129	if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
130	# We need lots of space, so avoid /tmp as it may be a memory
131	# file system backed by the swap file, or worse.
132	self._oOptions.sScratchRoot = tempfile.gettempdir();
133	else:
134	self._oOptions.sScratchRoot = '/var/tmp';
135	sSubDir = 'testbox';
136	try:
137	sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=no-member
138	except:
139	pass;
140	self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
141
142	self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
143	self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
144	self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
145
146	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
147	if not os.path.isdir(sDir):
148	os.makedirs(sDir, 0o700);
149
150	# We count consecutive reinitScratch failures and will reboot the
151	# testbox after a while in the hope that it will correct the issue.
152	self._cReinitScratchErrors = 0;
153
154	#
155	# Mount builds and test resources if requested.
156	#
157	self.mountShares();
158
159	#
160	# Sign-on parameters: Packed into list of records of format:
161	# { <Parameter ID>: { <Current value>, <Check function> } }
162	#
163	self._ddSignOnParams = \
164	{
165	constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
166	constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
167	constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
168	constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
169	constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
170	constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
171	constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
172	constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
173	constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
174	constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
175	constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
176	#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
177	constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
178	constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
179	constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
180	constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: utils.getPresentCpuCount },
181	constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
182	constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
183	}
184	for sItem in self._ddSignOnParams:
185	if self._ddSignOnParams[sItem][self.FN] is not None:
186	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
187
188	testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
189	testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
190	testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
191	for sItem in self._ddSignOnParams:
192	testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
193
194	#
195	# The System UUID is the primary identification of the machine, so
196	# refuse to cooperate if it's NULL.
197	#
198	self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
199	if self._sTestBoxUuid == self.ksNullUuid:
200	raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
201
202	#
203	# Export environment variables, clearing any we don't know yet.
204	#
205	for sEnvVar in self._oOptions.asEnvVars:
206	iEqual = sEnvVar.find('=');
207	if iEqual == -1: # No '=', remove it.
208	if sEnvVar in os.environ:
209	del os.environ[sEnvVar];
210	elif iEqual > 0: # Set it.
211	os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
212	else: # Starts with '=', bad user.
213	raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
214
215	os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
216	os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
217	os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
218	os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
219	os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
220	os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
221	os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
222	os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
223	os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
224	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
225	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
226	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
227	#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
228	os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
229	os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
230	os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
231	os.environ['TESTBOX_REPORTER'] = 'remote';
232	os.environ['TESTBOX_NAME'] = '';
233	os.environ['TESTBOX_ID'] = '';
234	os.environ['TESTBOX_TEST_SET_ID'] = '';
235	os.environ['TESTBOX_TIMEOUT'] = '0';
236	os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
237
238	if utils.getHostOs() == 'win':
239	os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
240	# Currently omitting any kBuild tools.
241
242	def mountShares(self):
243	"""
244	Mounts the shares.
245	Raises exception on failure.
246	"""
247	self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
248	self._oOptions.sBuildsServerShare,
249	self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd,
250	self._oOptions.sBuildsServerMountOpt, 'builds');
251	self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
252	self._oOptions.sTestRsrcServerShare,
253	self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd,
254	self._oOptions.sTestRsrcServerMountOpt, 'testrsrc');
255	return True;
256
257	def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sMountOpt, sWhat):
258	"""
259	Mounts the specified share if needed.
260	Raises exception on failure.
261	"""
262	# Only mount if the type is specified.
263	if sType is None:
264	return True;
265
266	# Test if already mounted.
267	sTestFile = os.path.join(sMountPoint + os.path.sep, os.path.basename(sShare) + '-new.txt');
268	if os.path.isfile(sTestFile):
269	return True;
270
271	#
272	# Platform specific mount code.
273	#
274	sHostOs = utils.getHostOs()
275	if sHostOs in ('darwin', 'freebsd'):
276	if sMountOpt != '':
277	sMountOpt = ',' + sMountOpt
278	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
279	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
280	utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=no-member
281	if sType == 'cifs':
282	# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
283	utils.processOutputChecked(['/sbin/mount_smbfs',
284	'-o',
285	'automounted,nostreams,soft,noowners,noatime,rdonly' + sMountOpt,
286	'-f', '0555', '-d', '0555',
287	'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
288	sMountPoint]);
289	else:
290	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
291
292	elif sHostOs == 'linux':
293	if sMountOpt != '':
294	sMountOpt = ',' + sMountOpt
295	utils.sudoProcessCall(['/bin/umount', sMountPoint]);
296	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
297	if sType == 'cifs':
298	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
299	'-o',
300	'user=' + sUser
301	+ ',password=' + sPassword
302	+ ',sec=ntlmv2'
303	+ ',uid=' + str(os.getuid()) # pylint: disable=no-member
304	+ ',gid=' + str(os.getgid()) # pylint: disable=no-member
305	+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro'
306	+ sMountOpt,
307	'//%s/%s' % (sServer, sShare),
308	sMountPoint]);
309	elif sType == 'nfs':
310	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
311	'-o', 'soft,ro' + sMountOpt,
312	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
313	sMountPoint]);
314
315	else:
316	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
317
318	elif sHostOs == 'solaris':
319	if sMountOpt != '':
320	sMountOpt = ',' + sMountOpt
321	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
322	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
323	if sType == 'cifs':
324	## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
325	oPasswdFile = tempfile.TemporaryFile();
326	oPasswdFile.write(sPassword + '\n');
327	oPasswdFile.flush();
328	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
329	'-o',
330	'user=' + sUser
331	+ ',uid=' + str(os.getuid()) # pylint: disable=no-member
332	+ ',gid=' + str(os.getgid()) # pylint: disable=no-member
333	+ ',fileperms=0555,dirperms=0555,noxattr,ro'
334	+ sMountOpt,
335	'//%s/%s' % (sServer, sShare),
336	sMountPoint],
337	stdin = oPasswdFile);
338	oPasswdFile.close();
339	elif sType == 'nfs':
340	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
341	'-o', 'noxattr,ro' + sMountOpt,
342	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
343	sMountPoint]);
344
345	else:
346	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
347
348
349	elif sHostOs == 'win':
350	if sType != 'cifs':
351	raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
352	utils.processCall(['net', 'use', sMountPoint, '/d']);
353	utils.processOutputChecked(['net', 'use', sMountPoint,
354	'\\\\' + sServer + '\\' + sShare,
355	sPassword,
356	'/USER:' + sUser,]);
357	else:
358	raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
359
360	#
361	# Re-test.
362	#
363	if not os.path.isfile(sTestFile):
364	raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
365	% (sWhat, sServer, sShare, sMountPoint, sTestFile));
366
367	return True;
368
369	## @name Signon property releated methods.
370	# @{
371
372	def _getHelperOutput(self, sCmd):
373	"""
374	Invokes TestBoxHelper to obtain information hard to access from python.
375	"""
376	if self._sTestBoxHelper is None:
377	if not utils.isRunningFromCheckout():
378	# See VBoxTestBoxScript.zip for layout.
379	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
380	'TestBoxHelper');
381	else: # Only for in-tree testing, so don't bother be too accurate right now.
382	sType = os.environ.get('KBUILD_TYPE', 'debug');
383	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
384	utils.getHostOsDotArch(), sType, 'testboxscript', \
385	utils.getHostOs(), utils.getHostArch(), \
386	'TestBoxHelper');
387	if utils.getHostOs() in ['win', 'os2']:
388	self._sTestBoxHelper += '.exe';
389
390	return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
391
392	def _getHelperOutputTristate(self, sCmd, fDunnoValue):
393	"""
394	Invokes TestBoxHelper to obtain information hard to access from python.
395	"""
396	sValue = self._getHelperOutput(sCmd);
397	sValue = sValue.lower();
398	if sValue == 'true':
399	return True;
400	if sValue == 'false':
401	return False;
402	if sValue not in ('dunno', 'none',):
403	raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
404	return fDunnoValue;
405
406
407	@staticmethod
408	def _isUuidGood(sUuid):
409	"""
410	Checks if the UUID looks good.
411
412	There are systems with really bad UUIDs, for instance
413	"03000200-0400-0500-0006-000700080009".
414	"""
415	if sUuid == TestBoxScript.ksNullUuid:
416	return False;
417	sUuid = sUuid.lower();
418	for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
419	if sUuid.count(sDigit) > 16:
420	return False;
421	return True;
422
423	def _getHostSystemUuid(self):
424	"""
425	Get the system UUID string from the System, return null-uuid if
426	unable to get retrieve it.
427	"""
428	if self._oOptions.sSystemUuid is not None:
429	return self._oOptions.sSystemUuid;
430
431	sUuid = self.ksNullUuid;
432
433	#
434	# Try get at the firmware UUID.
435	#
436	if utils.getHostOs() == 'linux':
437	# NOTE: This requires to have kernel option enabled:
438	# Firmware Drivers -> Export DMI identification via sysfs to userspace
439	if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
440	try:
441	sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
442	sUuid = str(uuid.UUID(sVar.strip()));
443	except:
444	pass;
445	## @todo consider dmidecoder? What about EFI systems?
446
447	elif utils.getHostOs() == 'win':
448	# Windows: WMI
449	try:
450	import win32com.client; # pylint: disable=import-error
451	oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
452	oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
453	for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
454	if oItem.UUID is not None:
455	sUuid = str(uuid.UUID(oItem.UUID));
456	except:
457	pass;
458
459	elif utils.getHostOs() == 'darwin':
460	try:
461	sVar = utils.processOutputChecked(['/bin/sh', '-c',
462	'/usr/sbin/ioreg -k IOPlatformUUID' \
463	+ '\| /usr/bin/grep IOPlatformUUID' \
464	+ '\| /usr/bin/head -1']);
465	sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
466	sUuid = str(uuid.UUID(sVar));
467	except:
468	pass;
469
470	elif utils.getHostOs() == 'solaris':
471	# Solaris: The smbios util.
472	try:
473	sVar = utils.processOutputChecked(['/bin/sh', '-c',
474	'/usr/sbin/smbios ' \
475	+ '\| /usr/xpg4/bin/sed -ne \'s/^.UUID: //p\'' \
476	+ '\| /usr/bin/head -1']);
477	sUuid = str(uuid.UUID(sVar.strip()));
478	except:
479	pass;
480
481	if self._isUuidGood(sUuid):
482	return sUuid;
483
484	#
485	# Try add the MAC address.
486	# uuid.getnode may provide it, or it may return a random number...
487	#
488	lMacAddr = uuid.getnode();
489	sNode = '%012x' % (lMacAddr,)
490	if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
491	return sUuid[:-12] + sNode;
492
493	return sUuid;
494
495	def _getHostCpuVendor(self):
496	"""
497	Get the CPUID vendor string on intel HW.
498	"""
499	return self._getHelperOutput('cpuvendor');
500
501	def _getHostCpuName(self):
502	"""
503	Get the CPU name/description string.
504	"""
505	return self._getHelperOutput('cpuname');
506
507	def _getHostCpuRevision(self):
508	"""
509	Get the CPU revision (family/model/stepping) value.
510	"""
511	return self._getHelperOutput('cpurevision');
512
513	def _hasHostHwVirt(self):
514	"""
515	Check if the host supports AMD-V or VT-x
516	"""
517	if self._oOptions.fHasHwVirt is None:
518	self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
519	return self._oOptions.fHasHwVirt;
520
521	def _hasHostNestedPaging(self):
522	"""
523	Check if the host supports nested paging.
524	"""
525	if not self._hasHostHwVirt():
526	return False;
527	if self._oOptions.fHasNestedPaging is None:
528	self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
529	return self._oOptions.fHasNestedPaging;
530
531	def _can64BitGuest(self):
532	"""
533	Check if the we (VBox) can run 64-bit guests.
534	"""
535	if not self._hasHostHwVirt():
536	return False;
537	if self._oOptions.fCan64BitGuest is None:
538	self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
539	return self._oOptions.fCan64BitGuest;
540
541	def _hasHostIoMmu(self):
542	"""
543	Check if the host has an I/O MMU of the VT-d kind.
544	"""
545	if not self._hasHostHwVirt():
546	return False;
547	if self._oOptions.fHasIoMmu is None:
548	## @todo Any way to figure this one out on any host OS?
549	self._oOptions.fHasIoMmu = False;
550	return self._oOptions.fHasIoMmu;
551
552	def _withRawModeSupport(self):
553	"""
554	Check if the testbox is configured with raw-mode support or not.
555	"""
556	if self._oOptions.fWithRawMode is None:
557	self._oOptions.fWithRawMode = True;
558	return self._oOptions.fWithRawMode;
559
560	def _getHostReport(self):
561	"""
562	Generate a report about the host hardware and software.
563	"""
564	return self._getHelperOutput('report');
565
566
567	def _getHostMemSize(self):
568	"""
569	Gets the amount of physical memory on the host (and accessible to the
570	OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
571	Unit: MiB.
572	"""
573	cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
574
575	# Round it.
576	cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
577	return cMbMemory;
578
579	def _getFreeScratchSpace(self):
580	"""
581	Get free space on the volume where scratch directory is located and
582	return it in bytes rounded down to nearest 64MB
583	(currently works on Linux only)
584	Unit: MiB.
585	"""
586	if platform.system() == 'Windows':
587	import ctypes
588	cTypeMbFreeSpace = ctypes.c_ulonglong(0)
589	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
590	ctypes.pointer(cTypeMbFreeSpace))
591	cMbFreeSpace = cTypeMbFreeSpace.value
592	else:
593	stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=no-member
594	cMbFreeSpace = stats.f_frsize * stats.f_bfree
595
596	# Convert to MB
597	cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
598
599	# Round free space size
600	cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
601	return cMbFreeSpace;
602
603	def _getScriptRev(self):
604	"""
605	The script (subversion) revision number.
606	"""
607	sRev = '@VBOX_SVN_REV@';
608	sRev = sRev.strip(); # just in case...
609	try:
610	_ = int(sRev);
611	except:
612	return __version__[11:-1].strip();
613	return sRev;
614
615	def _getPythonHexVersion(self):
616	"""
617	The python hex version number.
618	"""
619	uHexVersion = getattr(sys, 'hexversion', None);
620	if uHexVersion is None:
621	uHexVersion = (sys.version_info[0] << 24) \| (sys.version_info[1] << 16) \| (sys.version_info[2] << 8);
622	if sys.version_info[3] == 'final':
623	uHexVersion \|= 0xf0;
624	return uHexVersion;
625
626	# @}
627
628	def openTestManagerConnection(self):
629	"""
630	Opens up a connection to the test manager.
631
632	Raises exception on failure.
633	"""
634	return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
635
636	def getSignOnParam(self, sName):
637	"""
638	Returns a sign-on parameter value as string.
639	Raises exception if the name is incorrect.
640	"""
641	return str(self._ddSignOnParams[sName][self.VALUE]);
642
643	def getPathState(self):
644	"""
645	Get the path to the state dir in the scratch area.
646	"""
647	return self._sScratchState;
648
649	def getPathScripts(self):
650	"""
651	Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
652	"""
653	return self._sScratchScripts;
654
655	def getPathSpill(self):
656	"""
657	Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
658	"""
659	return self._sScratchSpill;
660
661	def getPathBuilds(self):
662	"""
663	Get the path to the builds.
664	"""
665	return self._oOptions.sBuildsPath;
666
667	def getTestBoxId(self):
668	"""
669	Get the TestBox ID for state saving purposes.
670	"""
671	return self._idTestBox;
672
673	def getTestBoxName(self):
674	"""
675	Get the TestBox name for state saving purposes.
676	"""
677	return self._sTestBoxName;
678
679	def _reinitScratch(self, fnLog, fUseTheForce):
680	"""
681	Wipes the scratch directories and re-initializes them.
682
683	No exceptions raise, returns success indicator instead.
684	"""
685	if fUseTheForce is None:
686	fUseTheForce = self._fFirstSignOn;
687
688	class ErrorCallback(object): # pylint: disable=too-few-public-methods
689	"""
690	Callbacks + state for the cleanup.
691	"""
692	def __init__(self):
693	self.fRc = True;
694	def onErrorCallback(self, sFnName, sPath, aXcptInfo):
695	""" Logs error during shutil.rmtree operation. """
696	fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
697	self.fRc = False;
698	oRc = ErrorCallback();
699
700	#
701	# Cleanup.
702	#
703	for sName in os.listdir(self._oOptions.sScratchRoot):
704	sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
705	try:
706	if os.path.isdir(sFullName):
707	shutil.rmtree(sFullName, False, oRc.onErrorCallback);
708	else:
709	os.remove(sFullName);
710	if os.path.exists(sFullName):
711	raise Exception('Still exists after deletion, weird.');
712	except Exception as oXcpt:
713	if fUseTheForce is True \
714	and utils.getHostOs() not in ['win', 'os2'] \
715	and len(sFullName) >= 8 \
716	and sFullName[0] == '/' \
717	and sFullName[1] != '/' \
718	and sFullName.find('/../') < 0:
719	fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
720	try:
721	if os.path.isdir(sFullName):
722	iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
723	else:
724	iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
725	if iRc != 0:
726	raise Exception('exit code %s' % iRc);
727	if os.path.exists(sFullName):
728	raise Exception('Still exists after forced deletion, weird^2.');
729	except:
730	fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
731	oRc.fRc = False;
732	else:
733	fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
734	oRc.fRc = False;
735
736	# Display files left behind.
737	def dirEnumCallback(sName, oStat):
738	""" callback for dirEnumerateTree """
739	fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName));
740	utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback);
741
742	#
743	# Re-create the directories.
744	#
745	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
746	if not os.path.isdir(sDir):
747	try:
748	os.makedirs(sDir, 0o700);
749	except Exception as oXcpt:
750	fnLog('Error creating "%s": %s' % (sDir, oXcpt));
751	oRc.fRc = False;
752
753	if oRc.fRc is True:
754	self._cReinitScratchErrors = 0;
755	else:
756	self._cReinitScratchErrors += 1;
757	return oRc.fRc;
758
759	def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000):
760	"""
761	Wipes the scratch directories and re-initializes them.
762
763	Will retry according to the cRetries and cMsDelay parameters. Windows
764	forces us to apply this hack as it ships with services asynchronously
765	scanning files after they execute, thus racing us cleaning up after a
766	test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping
767	vts_rm.exe kind of open, somehow preventing us from removing the
768	directory containing it, despite not issuing any errors deleting the
769	file itself. The service is called "Application Experience", which
770	feels like a weird joke here.
771
772	No exceptions raise, returns success indicator instead.
773	"""
774	fRc = self._reinitScratch(fnLog, fUseTheForce)
775	while fRc is False and cRetries > 0:
776	time.sleep(cMsDelay / 1000.0);
777	fnLog('reinitScratch: Retrying...');
778	fRc = self._reinitScratch(fnLog, fUseTheForce)
779	cRetries -= 1;
780	return fRc;
781
782
783	def _doSignOn(self):
784	"""
785	Worker for _maybeSignOn that does the actual signing on.
786	"""
787	assert not self._oCommand.isRunning();
788
789	# Reset the siged-on state.
790	testboxcommons.log('Signing-on...')
791	self._fSignedOn = False
792	self._idTestBox = None
793	self._cSignOnAttempts += 1;
794
795	# Assemble SIGN-ON request parameters and send the request.
796	dParams = {};
797	for sParam in self._ddSignOnParams:
798	dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
799	oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
800
801	# Check response.
802	try:
803	sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
804	if sResult != constants.tbresp.STATUS_ACK:
805	raise TestBoxException('Result is %s' % (sResult,));
806	oResponse.checkParameterCount(3);
807	idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
808	sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
809	except TestBoxException as err:
810	testboxcommons.log('Failed to sign-on: %s' % (str(err),))
811	testboxcommons.log('Server response: %s' % (oResponse.toString(),));
812	return False;
813
814	# Successfully signed on, update the state.
815	self._fSignedOn = True;
816	self._fNeedReSignOn = False;
817	self._cSignOnAttempts = 0;
818	self._idTestBox = idTestBox;
819	self._sTestBoxName = sTestBoxName;
820
821	# Update the environment.
822	os.environ['TESTBOX_ID'] = str(self._idTestBox);
823	os.environ['TESTBOX_NAME'] = sTestBoxName;
824	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
825	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
826	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
827
828	testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
829	% (self._idTestBox, self._sTestBoxName));
830
831	# Set up the scratch area.
832	self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2);
833
834	self._fFirstSignOn = False;
835	return True;
836
837	def _maybeSignOn(self):
838	"""
839	Check if Test Box parameters were changed
840	and do sign-in in case of positive result
841	"""
842
843	# Skip sign-on check if background command is currently in
844	# running state (avoid infinite signing on).
845	if self._oCommand.isRunning():
846	return None;
847
848	# Refresh sign-on parameters, changes triggers sign-on.
849	fNeedSignOn = not self._fSignedOn or self._fNeedReSignOn;
850	for item in self._ddSignOnParams:
851	if self._ddSignOnParams[item][self.FN] is None:
852	continue
853
854	sOldValue = self._ddSignOnParams[item][self.VALUE]
855	self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
856	if sOldValue != self._ddSignOnParams[item][self.VALUE]:
857	fNeedSignOn = True
858	testboxcommons.log('Detected %s parameter change: %s -> %s' %
859	(item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
860
861	if fNeedSignOn:
862	self._doSignOn();
863	return None;
864
865	def dispatch(self):
866	"""
867	Receive orders from Test Manager and execute them
868	"""
869
870	(self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
871	self._fNeedReSignOn = self._fSignedOn;
872	if self._fSignedOn:
873	os.environ['TESTBOX_ID'] = str(self._idTestBox);
874	os.environ['TESTBOX_NAME'] = self._sTestBoxName;
875
876	while True:
877	# Make sure we're signed on before trying to do anything.
878	self._maybeSignOn();
879	while not self._fSignedOn:
880	iFactor = 1 if self._cSignOnAttempts < 100 else 4;
881	time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
882	self._maybeSignOn();
883
884	# Retrieve and handle command from the TM.
885	(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
886	self._idTestBox,
887	self._sTestBoxUuid,
888	self._oCommand.isRunning());
889	if oResponse is not None:
890	self._oCommand.handleCommand(oResponse, oConnection);
891	if oConnection is not None:
892	if oConnection.isConnected():
893	self._oCommand.flushLogOnConnection(oConnection);
894	oConnection.close();
895
896	# Automatically reboot if scratch init fails.
897	#if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False:
898	# testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
899	# % ( self._cReinitScratchErrors, ));
900	# self._oCommand.doReboot();
901
902	# delay a wee bit before looping.
903	## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
904	# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
905	iFactor = 1;
906	if self._cReinitScratchErrors > 0:
907	iFactor = 4;
908	time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
909
910	# Not reached.
911
912
913	@staticmethod
914	def main():
915	"""
916	Main function a la C/C++. Returns exit code.
917	"""
918
919	#
920	# Parse arguments.
921	#
922	sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
923	if utils.getHostOs() in ('win', 'os2'):
924	sDefTestRsrc = 'T:';
925	sDefBuilds = 'U:';
926	elif utils.getHostOs() == 'darwin':
927	sDefTestRsrc = '/Volumes/testrsrc';
928	sDefBuilds = '/Volumes/builds';
929	else:
930	sDefTestRsrc = '/mnt/testrsrc';
931	sDefBuilds = '/mnt/builds';
932
933	class MyOptionParser(OptionParser):
934	""" We need to override the exit code on --help, error and so on. """
935	def __init__(self, args, *kwargs):
936	OptionParser.__init__(self, args, *kwargs);
937	def exit(self, status = 0, msg = None):
938	OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
939
940	parser = MyOptionParser(version=__version__[11:-1].strip());
941	for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
942	sLower = sMixed.lower();
943	sPrefix = 's' + sMixed;
944	parser.add_option('--' + sLower + '-path',
945	dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
946	help='Where ' + sDesc + ' can be found');
947	parser.add_option('--' + sLower + '-server-type',
948	dest=sPrefix + 'ServerType', metavar='<nfs\|cifs>', default=sDefShareType,
949	help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
950	parser.add_option('--' + sLower + '-server-name',
951	dest=sPrefix + 'ServerName', metavar='<server>',
952	default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
953	help='The name of the server with the builds.');
954	parser.add_option('--' + sLower + '-server-share',
955	dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
956	help='The name of the builds share.');
957	parser.add_option('--' + sLower + '-server-user',
958	dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
959	help='The user name to use when accessing the ' + sDesc + ' share.');
960	parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
961	dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
962	help='The password to use when accessing the ' + sDesc + ' share.');
963	parser.add_option('--' + sLower + '-server-mountopt',
964	dest=sPrefix + 'ServerMountOpt', metavar='<mountopt>', default='',
965	help='The mount options to use when accessing the ' + sDesc + ' share.');
966
967	parser.add_option("--test-manager", metavar="<url>",
968	dest="sTestManagerUrl",
969	help="Test Manager URL",
970	default="http://tindertux.de.oracle.com/testmanager")
971	parser.add_option("--scratch-root", metavar="<abs-path>",
972	dest="sScratchRoot",
973	help="Path to the scratch directory",
974	default=None)
975	parser.add_option("--system-uuid", metavar="<uuid>",
976	dest="sSystemUuid",
977	help="The system UUID of the testbox, used for uniquely identifiying the machine",
978	default=None)
979	parser.add_option("--hwvirt",
980	dest="fHasHwVirt", action="store_true", default=None,
981	help="Hardware virtualization available in the CPU");
982	parser.add_option("--no-hwvirt",
983	dest="fHasHwVirt", action="store_false", default=None,
984	help="Hardware virtualization not available in the CPU");
985	parser.add_option("--nested-paging",
986	dest="fHasNestedPaging", action="store_true", default=None,
987	help="Nested paging is available");
988	parser.add_option("--no-nested-paging",
989	dest="fHasNestedPaging", action="store_false", default=None,
990	help="Nested paging is not available");
991	parser.add_option("--64-bit-guest",
992	dest="fCan64BitGuest", action="store_true", default=None,
993	help="Host can execute 64-bit guests");
994	parser.add_option("--no-64-bit-guest",
995	dest="fCan64BitGuest", action="store_false", default=None,
996	help="Host cannot execute 64-bit guests");
997	parser.add_option("--io-mmu",
998	dest="fHasIoMmu", action="store_true", default=None,
999	help="I/O MMU available");
1000	parser.add_option("--no-io-mmu",
1001	dest="fHasIoMmu", action="store_false", default=None,
1002	help="No I/O MMU available");
1003	parser.add_option("--raw-mode",
1004	dest="fWithRawMode", action="store_true", default=None,
1005	help="Use raw-mode on this host.");
1006	parser.add_option("--no-raw-mode",
1007	dest="fWithRawMode", action="store_false", default=None,
1008	help="Disables raw-mode tests on this host.");
1009	parser.add_option("--pidfile",
1010	dest="sPidFile", default=None,
1011	help="For the parent script, ignored.");
1012	parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
1013	dest = "asEnvVars", default = [],
1014	help = "Sets an environment variable. Can be repeated.");
1015	def sbp_callback(option, opt_str, value, parser):
1016	_, _, _ = opt_str, value, option
1017	parser.values.sTestManagerUrl = 'http://10.162.100.8/testmanager/'
1018	parser.values.sBuildsServerName = 'vbox-st02.ru.oracle.com'
1019	parser.values.sTestRsrcServerName = 'vbox-st02.ru.oracle.com'
1020	parser.values.sTestRsrcServerShare = 'scratch/data/testrsrc'
1021	parser.add_option("--spb", "--load-sbp-defaults", action="callback", callback=sbp_callback,
1022	help="Load defaults for the sbp setup.")
1023
1024	(oOptions, args) = parser.parse_args()
1025	# Check command line
1026	if args != []:
1027	parser.print_help();
1028	return TBS_EXITCODE_SYNTAX;
1029
1030	if oOptions.sSystemUuid is not None:
1031	uuid.UUID(oOptions.sSystemUuid);
1032	if not oOptions.sTestManagerUrl.startswith('http://') \
1033	and not oOptions.sTestManagerUrl.startswith('https://'):
1034	print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
1035	return TBS_EXITCODE_SYNTAX;
1036
1037	for sPrefix in ['sBuilds', 'sTestRsrc']:
1038	sType = getattr(oOptions, sPrefix + 'ServerType');
1039	if sType is None or not sType.strip():
1040	setattr(oOptions, sPrefix + 'ServerType', None);
1041	elif sType not in ['cifs', 'nfs']:
1042	print('Syntax error: Invalid server type "%s"' % (sType,));
1043	return TBS_EXITCODE_SYNTAX;
1044
1045
1046	#
1047	# Instantiate the testbox script and start dispatching work.
1048	#
1049	try:
1050	oTestBoxScript = TestBoxScript(oOptions);
1051	except TestBoxScriptException as oXcpt:
1052	print('Error: %s' % (oXcpt,));
1053	return TBS_EXITCODE_SYNTAX;
1054	oTestBoxScript.dispatch();
1055
1056	# Not supposed to get here...
1057	return TBS_EXITCODE_FAILURE;
1058
1059
1060
1061	if __name__ == '__main__':
1062	sys.exit(TestBoxScript.main());
1063

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 93115

Download in other formats: