testboxscript_real.py@ 56292

Last change on this file since 56292 was 52776, checked in by vboxsync, 10 years ago
fix OSE
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 42.8 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: testboxscript_real.py 52776 2014-09-17 14:51:43Z vboxsync $
4
5	"""
6	TestBox Script - main().
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2014 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 52776 $"
31
32
33	# Standard python imports.
34	import math
35	import multiprocessing
36	import os
37	from optparse import OptionParser
38	import platform
39	import random
40	import shutil
41	import sys
42	import tempfile
43	import time
44	import uuid
45
46	# Only the main script needs to modify the path.
47	try: __file__
48	except: __file__ = sys.argv[0];
49	g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
50	g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
51	sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
52
53	# Validation Kit imports.
54	from common import constants;
55	from common import utils;
56	import testboxcommons;
57	from testboxcommons import TestBoxException;
58	from testboxcommand import TestBoxCommand;
59	from testboxconnection import TestBoxConnection;
60	from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
61
62
63	class TestBoxScriptException(Exception):
64	""" For raising exceptions during TestBoxScript.__init__. """
65	pass;
66
67
68	class TestBoxScript(object):
69	"""
70	Implementation of the test box script.
71	Communicate with test manager and perform offered actions.
72	"""
73
74	## @name Class Constants.
75	# @{
76
77	# Scratch space round value (MB).
78	kcMbScratchSpaceRounding = 64
79	# Memory size round value (MB).
80	kcMbMemoryRounding = 4
81	# A NULL UUID in string form.
82	ksNullUuid = '00000000-0000-0000-0000-000000000000';
83	# The minimum dispatch loop delay.
84	kcSecMinDelay = 12;
85	# The maximum dispatch loop delay (inclusive).
86	kcSecMaxDelay = 24;
87	# The minimum sign-on delay.
88	kcSecMinSignOnDelay = 30;
89	# The maximum sign-on delay (inclusive).
90	kcSecMaxSignOnDelay = 60;
91
92	# Keys for config params
93	VALUE = 'value'
94	FN = 'fn' # pylint: disable=C0103
95
96	## @}
97
98
99	def __init__(self, oOptions):
100	"""
101	Initialize internals
102	"""
103	self._oOptions = oOptions;
104	self._sTestBoxHelper = None;
105
106	# Signed-on state
107	self._cSignOnAttempts = 0;
108	self._fSignedOn = False;
109	self._fNeedReSignOn = False;
110	self._fFirstSignOn = True;
111	self._idTestBox = None;
112	self._sTestBoxName = '';
113	self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
114
115	# Command processor.
116	self._oCommand = TestBoxCommand(self);
117
118	#
119	# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
120	# many many GBs for some test scenarios and /tmp can be backed by swap
121	# or be a fast+small disk of some kind, while /var/tmp is normally
122	# larger, if slower. /var/tmp is generally not cleaned up on reboot,
123	# /tmp often is, this would break host panic / triple-fault detection.
124	#
125	if self._oOptions.sScratchRoot is None:
126	if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
127	# We need lots of space, so avoid /tmp as it may be a memory
128	# file system backed by the swap file, or worse.
129	self._oOptions.sScratchRoot = tempfile.gettempdir();
130	else:
131	self._oOptions.sScratchRoot = '/var/tmp';
132	sSubDir = 'testbox';
133	try:
134	sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
135	except:
136	pass;
137	self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
138
139	self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
140	self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
141	self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
142
143	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
144	if not os.path.isdir(sDir):
145	os.makedirs(sDir, 0700);
146
147	# We count consecutive reinitScratch failures and will reboot the
148	# testbox after a while in the hope that it will correct the issue.
149	self._cReinitScratchErrors = 0;
150
151	#
152	# Mount builds and test resources if requested.
153	#
154	self.mountShares();
155
156	#
157	# Sign-on parameters: Packed into list of records of format:
158	# { <Parameter ID>: { <Current value>, <Check function> } }
159	#
160	self._ddSignOnParams = \
161	{
162	constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
163	constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
164	constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
165	constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
166	constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
167	constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
168	constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
169	constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
170	constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
171	constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
172	constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
173	constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
174	constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
175	constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
176	constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
177	constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
178	constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
179	}
180	for sItem in self._ddSignOnParams:
181	if self._ddSignOnParams[sItem][self.FN] is not None:
182	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
183
184	testboxcommons.log('Starting Test Box script (%s)' % __version__)
185	testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
186	testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
187	for sItem in self._ddSignOnParams:
188	testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
189
190	#
191	# The System UUID is the primary identification of the machine, so
192	# refuse to cooperate if it's NULL.
193	#
194	self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
195	if self._sTestBoxUuid == self.ksNullUuid:
196	raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
197
198	#
199	# Export environment variables, clearing any we don't know yet.
200	#
201	for sEnvVar in self._oOptions.asEnvVars:
202	iEqual = sEnvVar.find('=');
203	if iEqual == -1: # No '=', remove it.
204	if sEnvVar in os.environ:
205	del os.environ[sEnvVar];
206	elif iEqual > 0: # Set it.
207	os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
208	else: # Starts with '=', bad user.
209	raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
210
211	os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
212	os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
213	os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
214	os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
215	os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
216	os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
217	os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
218	os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
219	os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
220	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
221	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
222	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
223	os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
224	os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
225	os.environ['TESTBOX_REPORTER'] = 'remote';
226	os.environ['TESTBOX_NAME'] = '';
227	os.environ['TESTBOX_ID'] = '';
228	os.environ['TESTBOX_TEST_SET_ID'] = '';
229	os.environ['TESTBOX_TIMEOUT'] = '0';
230	os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
231
232	if utils.getHostOs() is 'win':
233	os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
234	# Currently omitting any kBuild tools.
235
236	def mountShares(self):
237	"""
238	Mounts the shares.
239	Raises exception on failure.
240	"""
241	self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
242	self._oOptions.sBuildsServerShare,
243	self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd, 'builds');
244	self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
245	self._oOptions.sTestRsrcServerShare,
246	self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd, 'testrsrc');
247	return True;
248
249	def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sWhat):
250	"""
251	Mounts the specified share if needed.
252	Raises exception on failure.
253	"""
254	# Only mount if the type is specified.
255	if sType is None:
256	return True;
257
258	# Test if already mounted.
259	sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '.txt');
260	if os.path.isfile(sTestFile):
261	return True;
262
263	#
264	# Platform specific mount code.
265	#
266	sHostOs = utils.getHostOs()
267	if sHostOs in ('darwin', 'freebsd'):
268	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
269	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
270	utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
271	if sType == 'cifs':
272	# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
273	utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
274	'-f', '0555', '-d', '0555',
275	'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
276	sMountPoint]);
277	else:
278	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
279
280	elif sHostOs == 'linux':
281	utils.sudoProcessCall(['/bin/umount', sMountPoint]);
282	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
283	if sType == 'cifs':
284	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
285	'-o',
286	'user=' + sUser
287	+ ',password=' + sPassword
288	+ ',sec=ntlmv2'
289	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
290	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
291	+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
292	'//%s/%s' % (sServer, sShare),
293	sMountPoint]);
294	elif sType == 'nfs':
295	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
296	'-o', 'soft,ro',
297	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
298	sMountPoint]);
299
300	else:
301	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
302
303	elif sHostOs == 'solaris':
304	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
305	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
306	if sType == 'cifs':
307	## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
308	oPasswdFile = tempfile.TemporaryFile();
309	oPasswdFile.write(sPassword + '\n');
310	oPasswdFile.flush();
311	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
312	'-o',
313	'user=' + sUser
314	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
315	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
316	+ ',fileperms=0555,dirperms=0555,noxattr,ro',
317	'//%s/%s' % (sServer, sShare),
318	sMountPoint],
319	stdin = oPasswdFile);
320	oPasswdFile.close();
321	elif sType == 'nfs':
322	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
323	'-o', 'noxattr,ro',
324	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
325	sMountPoint]);
326
327	else:
328	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
329
330
331	elif sHostOs == 'win':
332	if sType != 'cifs':
333	raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
334	utils.processCall(['net', 'use', sMountPoint, '/d']);
335	utils.processOutputChecked(['net', 'use', sMountPoint,
336	'\\\\' + sServer + '\\' + sShare,
337	sPassword,
338	'/USER:' + sUser,]);
339	else:
340	raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
341
342	#
343	# Re-test.
344	#
345	if not os.path.isfile(sTestFile):
346	raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
347	% (sWhat, sServer, sShare, sMountPoint, sTestFile));
348
349	return True;
350
351	## @name Signon property releated methods.
352	# @{
353
354	def _getHelperOutput(self, sCmd):
355	"""
356	Invokes TestBoxHelper to obtain information hard to access from python.
357	"""
358	if self._sTestBoxHelper is None:
359	if not utils.isRunningFromCheckout():
360	# See VBoxTestBoxScript.zip for layout.
361	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
362	'TestBoxHelper');
363	else: # Only for in-tree testing, so don't bother be too accurate right now.
364	sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
365	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
366	utils.getHostOsDotArch(), sType, 'testboxscript', \
367	utils.getHostOs(), utils.getHostArch(), \
368	'TestBoxHelper');
369	if utils.getHostOs() in ['win', 'os2']:
370	self._sTestBoxHelper += '.exe';
371
372	return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
373
374	def _getHelperOutputTristate(self, sCmd, fDunnoValue):
375	"""
376	Invokes TestBoxHelper to obtain information hard to access from python.
377	"""
378	sValue = self._getHelperOutput(sCmd);
379	sValue = sValue.lower();
380	if sValue == 'true':
381	return True;
382	if sValue == 'false':
383	return False;
384	if sValue != 'dunno' and sValue != 'none':
385	raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
386	return fDunnoValue;
387
388
389	@staticmethod
390	def _isUuidGood(sUuid):
391	"""
392	Checks if the UUID looks good.
393
394	There are systems with really bad UUIDs, for instance
395	"03000200-0400-0500-0006-000700080009".
396	"""
397	if sUuid == TestBoxScript.ksNullUuid:
398	return False;
399	sUuid = sUuid.lower();
400	for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
401	if sUuid.count(sDigit) > 16:
402	return False;
403	return True;
404
405	def _getHostSystemUuid(self):
406	"""
407	Get the system UUID string from the System, return null-uuid if
408	unable to get retrieve it.
409	"""
410	if self._oOptions.sSystemUuid is not None:
411	return self._oOptions.sSystemUuid;
412
413	sUuid = self.ksNullUuid;
414
415	#
416	# Try get at the firmware UUID.
417	#
418	if utils.getHostOs() == 'linux':
419	# NOTE: This requires to have kernel option enabled:
420	# Firmware Drivers -> Export DMI identification via sysfs to userspace
421	if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
422	try:
423	sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
424	sUuid = str(uuid.UUID(sVar.strip()));
425	except:
426	pass;
427	## @todo consider dmidecoder? What about EFI systems?
428
429	elif utils.getHostOs() == 'win':
430	# Windows: WMI
431	try:
432	import win32com.client; # pylint: disable=F0401
433	oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
434	oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
435	for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
436	if oItem.UUID != None:
437	sUuid = str(uuid.UUID(oItem.UUID));
438	except:
439	pass;
440
441	elif utils.getHostOs() == 'darwin':
442	try:
443	sVar = utils.processOutputChecked(['/bin/sh', '-c',
444	'/usr/sbin/ioreg -k IOPlatformUUID' \
445	+ '\| /usr/bin/grep IOPlatformUUID' \
446	+ '\| /usr/bin/head -1']);
447	sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
448	sUuid = str(uuid.UUID(sVar));
449	except:
450	pass;
451
452	elif utils.getHostOs() == 'solaris':
453	# Solaris: The smbios util.
454	try:
455	sVar = utils.processOutputChecked(['/bin/sh', '-c',
456	'/usr/sbin/smbios ' \
457	+ '\| /usr/xpg4/bin/sed -ne \'s/^.UUID: //p\'' \
458	+ '\| /usr/bin/head -1']);
459	sUuid = str(uuid.UUID(sVar.strip()));
460	except:
461	pass;
462
463	if self._isUuidGood(sUuid):
464	return sUuid;
465
466	#
467	# Try add the MAC address.
468	# uuid.getnode may provide it, or it may return a random number...
469	#
470	lMacAddr = uuid.getnode();
471	sNode = '%012x' % (lMacAddr,)
472	if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
473	return sUuid[:-12] + sNode;
474
475	return sUuid;
476
477	def _getHostCpuVendor(self):
478	"""
479	Get the CPUID vendor string on intel HW.
480	"""
481	return self._getHelperOutput('cpuvendor');
482
483	def _getHostCpuName(self):
484	"""
485	Get the CPU name/description string.
486	"""
487	return self._getHelperOutput('cpuname');
488
489	def _getHostCpuRevision(self):
490	"""
491	Get the CPU revision (family/model/stepping) value.
492	"""
493	return self._getHelperOutput('cpurevision');
494
495	def _hasHostHwVirt(self):
496	"""
497	Check if the host supports AMD-V or VT-x
498	"""
499	if self._oOptions.fHasHwVirt is None:
500	self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
501	return self._oOptions.fHasHwVirt;
502
503	def _hasHostNestedPaging(self):
504	"""
505	Check if the host supports nested paging.
506	"""
507	if not self._hasHostHwVirt():
508	return False;
509	if self._oOptions.fHasNestedPaging is None:
510	self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
511	return self._oOptions.fHasNestedPaging;
512
513	def _can64BitGuest(self):
514	"""
515	Check if the we (VBox) can run 64-bit guests.
516	"""
517	if not self._hasHostHwVirt():
518	return False;
519	if self._oOptions.fCan64BitGuest is None:
520	self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
521	return self._oOptions.fCan64BitGuest;
522
523	def _hasHostIoMmu(self):
524	"""
525	Check if the host has an I/O MMU of the VT-d kind.
526	"""
527	if not self._hasHostHwVirt():
528	return False;
529	if self._oOptions.fHasIoMmu is None:
530	## @todo Any way to figure this one out on any host OS?
531	self._oOptions.fHasIoMmu = False;
532	return self._oOptions.fHasIoMmu;
533
534	def _getHostReport(self):
535	"""
536	Generate a report about the host hardware and software.
537	"""
538	return self._getHelperOutput('report');
539
540
541	def _getHostMemSize(self):
542	"""
543	Gets the amount of physical memory on the host (and accessible to the
544	OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
545	Unit: MiB.
546	"""
547	cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
548
549	# Round it.
550	cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
551	return cMbMemory;
552
553	def _getFreeScratchSpace(self):
554	"""
555	Get free space on the volume where scratch directory is located and
556	return it in bytes rounded down to nearest 64MB
557	(currently works on Linux only)
558	Unit: MiB.
559	"""
560	if platform.system() == 'Windows':
561	import ctypes
562	cTypeMbFreeSpace = ctypes.c_ulonglong(0)
563	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
564	ctypes.pointer(cTypeMbFreeSpace))
565	cMbFreeSpace = cTypeMbFreeSpace.value
566	else:
567	stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=E1101
568	cMbFreeSpace = stats.f_frsize * stats.f_bfree
569
570	# Convert to MB
571	cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
572
573	# Round free space size
574	cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
575	return cMbFreeSpace;
576
577	def _getScriptRev(self):
578	"""
579	The script (subversion) revision number.
580	"""
581	return __version__[11:-1].strip();
582
583	def _getPythonHexVersion(self):
584	"""
585	The python hex version number.
586	"""
587	uHexVersion = getattr(sys, 'hexversion', None);
588	if uHexVersion is None:
589	uHexVersion = (sys.version_info[0] << 24) \| (sys.version_info[1] << 16) \| (sys.version_info[2] << 8);
590	if sys.version_info[3] == 'final':
591	uHexVersion \|= 0xf0;
592	return uHexVersion;
593
594	# @}
595
596	def openTestManagerConnection(self):
597	"""
598	Opens up a connection to the test manager.
599
600	Raises exception on failure.
601	"""
602	return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
603
604	def getSignOnParam(self, sName):
605	"""
606	Returns a sign-on parameter value as string.
607	Raises exception if the name is incorrect.
608	"""
609	return str(self._ddSignOnParams[sName][self.VALUE]);
610
611	def getPathState(self):
612	"""
613	Get the path to the state dir in the scratch area.
614	"""
615	return self._sScratchState;
616
617	def getPathScripts(self):
618	"""
619	Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
620	"""
621	return self._sScratchScripts;
622
623	def getPathSpill(self):
624	"""
625	Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
626	"""
627	return self._sScratchSpill;
628
629	def getPathBuilds(self):
630	"""
631	Get the path to the builds.
632	"""
633	return self._oOptions.sBuildsPath;
634
635	def getTestBoxId(self):
636	"""
637	Get the TestBox ID for state saving purposes.
638	"""
639	return self._idTestBox;
640
641	def getTestBoxName(self):
642	"""
643	Get the TestBox name for state saving purposes.
644	"""
645	return self._sTestBoxName;
646
647	def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None):
648	"""
649	Wipes the scratch directories and re-initializes them.
650
651	No exceptions raise, returns success indicator instead.
652	"""
653	if fUseTheForce is None:
654	fUseTheForce = self._fFirstSignOn;
655
656	class ErrorCallback(object): # pylint: disable=R0903
657	"""
658	Callbacks + state for the cleanup.
659	"""
660	def __init__(self):
661	self.fRc = True;
662	def onErrorCallback(self, sFnName, sPath, aXcptInfo):
663	""" Logs error during shutil.rmtree operation. """
664	fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
665	self.fRc = False;
666	oRc = ErrorCallback();
667
668	#
669	# Cleanup.
670	#
671	for sName in os.listdir(self._oOptions.sScratchRoot):
672	sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
673	try:
674	if os.path.isdir(sFullName):
675	shutil.rmtree(sFullName, False, oRc.onErrorCallback);
676	else:
677	os.remove(sFullName);
678	if os.path.exists(sFullName):
679	raise Exception('Still exists after deletion, weird.');
680	except Exception, oXcpt:
681	if fUseTheForce is True \
682	and utils.getHostOs() not in ['win', 'os2'] \
683	and len(sFullName) >= 8 \
684	and sFullName[0] == '/' \
685	and sFullName[1] != '/' \
686	and sFullName.find('/../') < 0:
687	fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
688	try:
689	if os.path.isdir(sFullName):
690	iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
691	else:
692	iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
693	if iRc != 0:
694	raise Exception('exit code %s' % iRc);
695	if os.path.exists(sFullName):
696	raise Exception('Still exists after forced deletion, weird^2.');
697	except:
698	fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
699	oRc.fRc = False;
700	else:
701	fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
702	oRc.fRc = False;
703
704	#
705	# Re-create the directories.
706	#
707	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
708	if not os.path.isdir(sDir):
709	try:
710	os.makedirs(sDir, 0700);
711	except Exception, oXcpt:
712	fnLog('Error creating "%s": %s' % (sDir, oXcpt));
713	oRc.fRc = False;
714
715	if oRc.fRc is True:
716	self._cReinitScratchErrors = 0;
717	else:
718	self._cReinitScratchErrors += 1;
719	return oRc.fRc;
720
721	def _doSignOn(self):
722	"""
723	Worker for _maybeSignOn that does the actual signing on.
724	"""
725	assert not self._oCommand.isRunning();
726
727	# Reset the siged-on state.
728	testboxcommons.log('Signing-on...')
729	self._fSignedOn = False
730	self._idTestBox = None
731	self._cSignOnAttempts += 1;
732
733	# Assemble SIGN-ON request parameters and send the request.
734	dParams = {};
735	for sParam in self._ddSignOnParams:
736	dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
737	oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
738
739	# Check response.
740	try:
741	sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
742	if sResult != constants.tbresp.STATUS_ACK:
743	raise TestBoxException('Result is %s' % (sResult,));
744	oResponse.checkParameterCount(3);
745	idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
746	sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
747	except TestBoxException, err:
748	testboxcommons.log('Failed to sign-on: %s' % (str(err),))
749	testboxcommons.log('Server response: %s' % (oResponse.toString(),));
750	return False;
751
752	# Successfully signed on, update the state.
753	self._fSignedOn = True;
754	self._fNeedReSignOn = False;
755	self._cSignOnAttempts = 0;
756	self._idTestBox = idTestBox;
757	self._sTestBoxName = sTestBoxName;
758
759	# Update the environment.
760	os.environ['TESTBOX_ID'] = str(self._idTestBox);
761	os.environ['TESTBOX_NAME'] = sTestBoxName;
762	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
763	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
764	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
765
766	testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
767	% (self._idTestBox, self._sTestBoxName));
768
769	# Set up the scratch area.
770	self.reinitScratch(fUseTheForce = self._fFirstSignOn);
771
772	self._fFirstSignOn = False;
773	return True;
774
775	def _maybeSignOn(self):
776	"""
777	Check if Test Box parameters were changed
778	and do sign-in in case of positive result
779	"""
780
781	# Skip sign-on check if background command is currently in
782	# running state (avoid infinite signing on).
783	if self._oCommand.isRunning():
784	return None;
785
786	# Refresh sign-on parameters, changes triggers sign-on.
787	fNeedSignOn = (True if not self._fSignedOn or self._fNeedReSignOn else False)
788	for item in self._ddSignOnParams:
789	if self._ddSignOnParams[item][self.FN] is None:
790	continue
791
792	sOldValue = self._ddSignOnParams[item][self.VALUE]
793	self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
794	if sOldValue != self._ddSignOnParams[item][self.VALUE]:
795	fNeedSignOn = True
796	testboxcommons.log('Detected %s parameter change: %s -> %s' %
797	(item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
798
799	if fNeedSignOn:
800	self._doSignOn();
801	return None;
802
803	def dispatch(self):
804	"""
805	Receive orders from Test Manager and execute them
806	"""
807
808	(self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
809	self._fNeedReSignOn = self._fSignedOn;
810	if self._fSignedOn:
811	os.environ['TESTBOX_ID'] = str(self._idTestBox);
812	os.environ['TESTBOX_NAME'] = self._sTestBoxName;
813
814	while True:
815	# Make sure we're signed on before trying to do anything.
816	self._maybeSignOn();
817	while not self._fSignedOn:
818	iFactor = 1 if self._cSignOnAttempts < 100 else 4;
819	time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
820	self._maybeSignOn();
821
822	# Retrieve and handle command from the TM.
823	(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
824	self._idTestBox,
825	self._sTestBoxUuid,
826	self._oCommand.isRunning());
827	if oResponse is not None:
828	self._oCommand.handleCommand(oResponse, oConnection);
829	if oConnection is not None:
830	if oConnection.isConnected():
831	self._oCommand.flushLogOnConnection(oConnection);
832	oConnection.close();
833
834	# Automatically reboot if scratch init fails.
835	if self._cReinitScratchErrors > 8 and self.reinitScratch() is False:
836	testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
837	% ( self._cReinitScratchErrors, ));
838	self._oCommand.doReboot();
839
840	# delay a wee bit before looping.
841	## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
842	# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
843	iFactor = 1;
844	if self._cReinitScratchErrors > 0:
845	iFactor = 4;
846	time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
847
848	# Not reached.
849
850
851	@staticmethod
852	def main():
853	"""
854	Main function a la C/C++. Returns exit code.
855	"""
856
857	#
858	# Parse arguments.
859	#
860	if utils.getHostOs() in ('win', 'os2'):
861	sDefTestRsrc = 'T:';
862	sDefBuilds = 'U:';
863	elif utils.getHostOs() == 'darwin':
864	sDefTestRsrc = '/Volumes/testrsrc';
865	sDefBuilds = '/Volumes/builds';
866	else:
867	sDefTestRsrc = '/mnt/testrsrc';
868	sDefBuilds = '/mnt/builds';
869
870	class MyOptionParser(OptionParser):
871	""" We need to override the exit code on --help, error and so on. """
872	def __init__(self, args, *kwargs):
873	OptionParser.__init__(self, args, *kwargs);
874	def exit(self, status = 0, msg = None):
875	OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
876
877	parser = MyOptionParser(version=__version__[11:-1].strip());
878	for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
879	sLower = sMixed.lower();
880	sPrefix = 's' + sMixed;
881	parser.add_option('--' + sLower + '-path',
882	dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
883	help='Where ' + sDesc + ' can be found');
884	parser.add_option('--' + sLower + '-server-type',
885	dest=sPrefix + 'ServerType', metavar='<nfs\|cifs>', default=None,
886	help='The type of server, cifs or nfs. If empty (default), we won\'t try mount anything.');
887	parser.add_option('--' + sLower + '-server-name',
888	dest=sPrefix + 'ServerName', metavar='<server>', default='solserv.de.oracle.com',
889	help='The name of the server with the builds.');
890	parser.add_option('--' + sLower + '-server-share',
891	dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
892	help='The name of the builds share.');
893	parser.add_option('--' + sLower + '-server-user',
894	dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
895	help='The user name to use when accessing the ' + sDesc + ' share.');
896	parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
897	dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
898	help='The password to use when accessing the ' + sDesc + ' share.');
899
900	parser.add_option("--test-manager", metavar="<url>",
901	dest="sTestManagerUrl",
902	help="Test Manager URL",
903	default="http://tindertux.de.oracle.com/testmanager")
904	parser.add_option("--scratch-root", metavar="<abs-path>",
905	dest="sScratchRoot",
906	help="Path to the scratch directory",
907	default=None)
908	parser.add_option("--system-uuid", metavar="<uuid>",
909	dest="sSystemUuid",
910	help="The system UUID of the testbox, used for uniquely identifiying the machine",
911	default=None)
912	parser.add_option("--hwvirt",
913	dest="fHasHwVirt", action="store_true", default=None,
914	help="Hardware virtualization available in the CPU");
915	parser.add_option("--no-hwvirt",
916	dest="fHasHwVirt", action="store_false", default=None,
917	help="Hardware virtualization not available in the CPU");
918	parser.add_option("--nested-paging",
919	dest="fHasNestedPaging", action="store_true", default=None,
920	help="Nested paging is available");
921	parser.add_option("--no-nested-paging",
922	dest="fHasNestedPaging", action="store_false", default=None,
923	help="Nested paging is not available");
924	parser.add_option("--64-bit-guest",
925	dest="fCan64BitGuest", action="store_true", default=None,
926	help="Host can execute 64-bit guests");
927	parser.add_option("--no-64-bit-guest",
928	dest="fCan64BitGuest", action="store_false", default=None,
929	help="Host cannot execute 64-bit guests");
930	parser.add_option("--io-mmu",
931	dest="fHasIoMmu", action="store_true", default=None,
932	help="I/O MMU available");
933	parser.add_option("--no-io-mmu",
934	dest="fHasIoMmu", action="store_false", default=None,
935	help="No I/O MMU available");
936	parser.add_option("--pidfile",
937	dest="sPidFile", default=None,
938	help="For the parent script, ignored.");
939	parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
940	dest = "asEnvVars", default = [],
941	help = "Sets an environment variable. Can be repeated.");
942
943	(oOptions, args) = parser.parse_args()
944	# Check command line
945	if args != []:
946	parser.print_help();
947	return TBS_EXITCODE_SYNTAX;
948
949	if oOptions.sSystemUuid is not None:
950	uuid.UUID(oOptions.sSystemUuid);
951	if not oOptions.sTestManagerUrl.startswith('http://') \
952	and not oOptions.sTestManagerUrl.startswith('https://'):
953	print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
954	return TBS_EXITCODE_SYNTAX;
955
956	for sPrefix in ['sBuilds', 'sTestRsrc']:
957	sType = getattr(oOptions, sPrefix + 'ServerType');
958	if sType is None or len(sType.strip()) == 0:
959	setattr(oOptions, sPrefix + 'ServerType', None);
960	elif sType not in ['cifs', 'nfs']:
961	print('Syntax error: Invalid server type "%s"' % (sType,));
962	return TBS_EXITCODE_SYNTAX;
963
964
965	#
966	# Instantiate the testbox script and start dispatching work.
967	#
968	try:
969	oTestBoxScript = TestBoxScript(oOptions);
970	except TestBoxScriptException, oXcpt:
971	print('Error: %s' % (oXcpt,));
972	return TBS_EXITCODE_SYNTAX;
973	oTestBoxScript.dispatch();
974
975	# Not supposed to get here...
976	return TBS_EXITCODE_FAILURE;
977
978
979
980	if __name__ == '__main__':
981	sys.exit(TestBoxScript.main());
982

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 56292

Download in other formats: