VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 70693

Last change on this file since 70693 was 70660, checked in by vboxsync, 7 years ago

ValidationKit: Python 3 and pylint 1.8.1 adjustments/fixes.

  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 45.6 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: testboxscript_real.py 70660 2018-01-21 16:18:58Z vboxsync $
4
5"""
6TestBox Script - main().
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2017 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 70660 $"
31
32
33# Standard python imports.
34import math
35import multiprocessing
36import os
37from optparse import OptionParser # pylint: disable=deprecated-module
38import platform
39import random
40import shutil
41import sys
42import tempfile
43import time
44import uuid
45
46# Only the main script needs to modify the path.
47try: __file__
48except: __file__ = sys.argv[0];
49g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
50g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
51sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
52
53# Validation Kit imports.
54from common import constants;
55from common import utils;
56import testboxcommons;
57from testboxcommons import TestBoxException;
58from testboxcommand import TestBoxCommand;
59from testboxconnection import TestBoxConnection;
60from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
61
62# Python 3 hacks:
63if sys.version_info[0] >= 3:
64 long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67class TestBoxScriptException(Exception):
68 """ For raising exceptions during TestBoxScript.__init__. """
69 pass;
70
71
72class TestBoxScript(object):
73 """
74 Implementation of the test box script.
75 Communicate with test manager and perform offered actions.
76 """
77
78 ## @name Class Constants.
79 # @{
80
81 # Scratch space round value (MB).
82 kcMbScratchSpaceRounding = 64
83 # Memory size round value (MB).
84 kcMbMemoryRounding = 4
85 # A NULL UUID in string form.
86 ksNullUuid = '00000000-0000-0000-0000-000000000000';
87 # The minimum dispatch loop delay.
88 kcSecMinDelay = 12;
89 # The maximum dispatch loop delay (inclusive).
90 kcSecMaxDelay = 24;
91 # The minimum sign-on delay.
92 kcSecMinSignOnDelay = 30;
93 # The maximum sign-on delay (inclusive).
94 kcSecMaxSignOnDelay = 60;
95
96 # Keys for config params
97 VALUE = 'value'
98 FN = 'fn' # pylint: disable=C0103
99
100 ## @}
101
102
103 def __init__(self, oOptions):
104 """
105 Initialize internals
106 """
107 self._oOptions = oOptions;
108 self._sTestBoxHelper = None;
109
110 # Signed-on state
111 self._cSignOnAttempts = 0;
112 self._fSignedOn = False;
113 self._fNeedReSignOn = False;
114 self._fFirstSignOn = True;
115 self._idTestBox = None;
116 self._sTestBoxName = '';
117 self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
118
119 # Command processor.
120 self._oCommand = TestBoxCommand(self);
121
122 #
123 # Scratch dir setup. Use /var/tmp instead of /tmp because we may need
124 # many many GBs for some test scenarios and /tmp can be backed by swap
125 # or be a fast+small disk of some kind, while /var/tmp is normally
126 # larger, if slower. /var/tmp is generally not cleaned up on reboot,
127 # /tmp often is, this would break host panic / triple-fault detection.
128 #
129 if self._oOptions.sScratchRoot is None:
130 if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
131 # We need *lots* of space, so avoid /tmp as it may be a memory
132 # file system backed by the swap file, or worse.
133 self._oOptions.sScratchRoot = tempfile.gettempdir();
134 else:
135 self._oOptions.sScratchRoot = '/var/tmp';
136 sSubDir = 'testbox';
137 try:
138 sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
139 except:
140 pass;
141 self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
142
143 self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
144 self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
145 self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
146
147 for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
148 if not os.path.isdir(sDir):
149 os.makedirs(sDir, 0o700);
150
151 # We count consecutive reinitScratch failures and will reboot the
152 # testbox after a while in the hope that it will correct the issue.
153 self._cReinitScratchErrors = 0;
154
155 #
156 # Mount builds and test resources if requested.
157 #
158 self.mountShares();
159
160 #
161 # Sign-on parameters: Packed into list of records of format:
162 # { <Parameter ID>: { <Current value>, <Check function> } }
163 #
164 self._ddSignOnParams = \
165 {
166 constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
167 constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
168 constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
169 constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
170 constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
171 constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
172 constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
173 constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
174 constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
175 constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
176 constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
177 #constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
178 constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
179 constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
180 constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
181 constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
182 constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
183 constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
184 }
185 for sItem in self._ddSignOnParams:
186 if self._ddSignOnParams[sItem][self.FN] is not None:
187 self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
188
189 testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
190 testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
191 testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
192 for sItem in self._ddSignOnParams:
193 testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
194
195 #
196 # The System UUID is the primary identification of the machine, so
197 # refuse to cooperate if it's NULL.
198 #
199 self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
200 if self._sTestBoxUuid == self.ksNullUuid:
201 raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
202
203 #
204 # Export environment variables, clearing any we don't know yet.
205 #
206 for sEnvVar in self._oOptions.asEnvVars:
207 iEqual = sEnvVar.find('=');
208 if iEqual == -1: # No '=', remove it.
209 if sEnvVar in os.environ:
210 del os.environ[sEnvVar];
211 elif iEqual > 0: # Set it.
212 os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
213 else: # Starts with '=', bad user.
214 raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
215
216 os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
217 os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
218 os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
219 os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
220 os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
221 os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
222 os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
223 os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
224 os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
225 os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
226 os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
227 os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
228 #TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
229 os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
230 os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
231 os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
232 os.environ['TESTBOX_REPORTER'] = 'remote';
233 os.environ['TESTBOX_NAME'] = '';
234 os.environ['TESTBOX_ID'] = '';
235 os.environ['TESTBOX_TEST_SET_ID'] = '';
236 os.environ['TESTBOX_TIMEOUT'] = '0';
237 os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
238
239 if utils.getHostOs() == 'win':
240 os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
241 # Currently omitting any kBuild tools.
242
243 def mountShares(self):
244 """
245 Mounts the shares.
246 Raises exception on failure.
247 """
248 self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
249 self._oOptions.sBuildsServerShare,
250 self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd, 'builds');
251 self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
252 self._oOptions.sTestRsrcServerShare,
253 self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd, 'testrsrc');
254 return True;
255
256 def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sWhat):
257 """
258 Mounts the specified share if needed.
259 Raises exception on failure.
260 """
261 # Only mount if the type is specified.
262 if sType is None:
263 return True;
264
265 # Test if already mounted.
266 sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '-new.txt');
267 if os.path.isfile(sTestFile):
268 return True;
269
270 #
271 # Platform specific mount code.
272 #
273 sHostOs = utils.getHostOs()
274 if sHostOs in ('darwin', 'freebsd'):
275 utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
276 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
277 utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
278 if sType == 'cifs':
279 # Note! no smb://server/share stuff here, 10.6.8 didn't like it.
280 utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
281 '-f', '0555', '-d', '0555',
282 '//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
283 sMountPoint]);
284 else:
285 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
286
287 elif sHostOs == 'linux':
288 utils.sudoProcessCall(['/bin/umount', sMountPoint]);
289 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
290 if sType == 'cifs':
291 utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
292 '-o',
293 'user=' + sUser
294 + ',password=' + sPassword
295 + ',sec=ntlmv2'
296 + ',uid=' + str(os.getuid()) # pylint: disable=E1101
297 + ',gid=' + str(os.getgid()) # pylint: disable=E1101
298 + ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
299 '//%s/%s' % (sServer, sShare),
300 sMountPoint]);
301 elif sType == 'nfs':
302 utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
303 '-o', 'soft,ro',
304 '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
305 sMountPoint]);
306
307 else:
308 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
309
310 elif sHostOs == 'solaris':
311 utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
312 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
313 if sType == 'cifs':
314 ## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
315 oPasswdFile = tempfile.TemporaryFile();
316 oPasswdFile.write(sPassword + '\n');
317 oPasswdFile.flush();
318 utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
319 '-o',
320 'user=' + sUser
321 + ',uid=' + str(os.getuid()) # pylint: disable=E1101
322 + ',gid=' + str(os.getgid()) # pylint: disable=E1101
323 + ',fileperms=0555,dirperms=0555,noxattr,ro',
324 '//%s/%s' % (sServer, sShare),
325 sMountPoint],
326 stdin = oPasswdFile);
327 oPasswdFile.close();
328 elif sType == 'nfs':
329 utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
330 '-o', 'noxattr,ro',
331 '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
332 sMountPoint]);
333
334 else:
335 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
336
337
338 elif sHostOs == 'win':
339 if sType != 'cifs':
340 raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
341 utils.processCall(['net', 'use', sMountPoint, '/d']);
342 utils.processOutputChecked(['net', 'use', sMountPoint,
343 '\\\\' + sServer + '\\' + sShare,
344 sPassword,
345 '/USER:' + sUser,]);
346 else:
347 raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
348
349 #
350 # Re-test.
351 #
352 if not os.path.isfile(sTestFile):
353 raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
354 % (sWhat, sServer, sShare, sMountPoint, sTestFile));
355
356 return True;
357
358 ## @name Signon property releated methods.
359 # @{
360
361 def _getHelperOutput(self, sCmd):
362 """
363 Invokes TestBoxHelper to obtain information hard to access from python.
364 """
365 if self._sTestBoxHelper is None:
366 if not utils.isRunningFromCheckout():
367 # See VBoxTestBoxScript.zip for layout.
368 self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
369 'TestBoxHelper');
370 else: # Only for in-tree testing, so don't bother be too accurate right now.
371 sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
372 self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
373 utils.getHostOsDotArch(), sType, 'testboxscript', \
374 utils.getHostOs(), utils.getHostArch(), \
375 'TestBoxHelper');
376 if utils.getHostOs() in ['win', 'os2']:
377 self._sTestBoxHelper += '.exe';
378
379 return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
380
381 def _getHelperOutputTristate(self, sCmd, fDunnoValue):
382 """
383 Invokes TestBoxHelper to obtain information hard to access from python.
384 """
385 sValue = self._getHelperOutput(sCmd);
386 sValue = sValue.lower();
387 if sValue == 'true':
388 return True;
389 if sValue == 'false':
390 return False;
391 if sValue != 'dunno' and sValue != 'none':
392 raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
393 return fDunnoValue;
394
395
396 @staticmethod
397 def _isUuidGood(sUuid):
398 """
399 Checks if the UUID looks good.
400
401 There are systems with really bad UUIDs, for instance
402 "03000200-0400-0500-0006-000700080009".
403 """
404 if sUuid == TestBoxScript.ksNullUuid:
405 return False;
406 sUuid = sUuid.lower();
407 for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
408 if sUuid.count(sDigit) > 16:
409 return False;
410 return True;
411
412 def _getHostSystemUuid(self):
413 """
414 Get the system UUID string from the System, return null-uuid if
415 unable to get retrieve it.
416 """
417 if self._oOptions.sSystemUuid is not None:
418 return self._oOptions.sSystemUuid;
419
420 sUuid = self.ksNullUuid;
421
422 #
423 # Try get at the firmware UUID.
424 #
425 if utils.getHostOs() == 'linux':
426 # NOTE: This requires to have kernel option enabled:
427 # Firmware Drivers -> Export DMI identification via sysfs to userspace
428 if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
429 try:
430 sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
431 sUuid = str(uuid.UUID(sVar.strip()));
432 except:
433 pass;
434 ## @todo consider dmidecoder? What about EFI systems?
435
436 elif utils.getHostOs() == 'win':
437 # Windows: WMI
438 try:
439 import win32com.client; # pylint: disable=F0401
440 oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
441 oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
442 for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
443 if oItem.UUID != None:
444 sUuid = str(uuid.UUID(oItem.UUID));
445 except:
446 pass;
447
448 elif utils.getHostOs() == 'darwin':
449 try:
450 sVar = utils.processOutputChecked(['/bin/sh', '-c',
451 '/usr/sbin/ioreg -k IOPlatformUUID' \
452 + '| /usr/bin/grep IOPlatformUUID' \
453 + '| /usr/bin/head -1']);
454 sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
455 sUuid = str(uuid.UUID(sVar));
456 except:
457 pass;
458
459 elif utils.getHostOs() == 'solaris':
460 # Solaris: The smbios util.
461 try:
462 sVar = utils.processOutputChecked(['/bin/sh', '-c',
463 '/usr/sbin/smbios ' \
464 + '| /usr/xpg4/bin/sed -ne \'s/^.*UUID: *//p\'' \
465 + '| /usr/bin/head -1']);
466 sUuid = str(uuid.UUID(sVar.strip()));
467 except:
468 pass;
469
470 if self._isUuidGood(sUuid):
471 return sUuid;
472
473 #
474 # Try add the MAC address.
475 # uuid.getnode may provide it, or it may return a random number...
476 #
477 lMacAddr = uuid.getnode();
478 sNode = '%012x' % (lMacAddr,)
479 if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
480 return sUuid[:-12] + sNode;
481
482 return sUuid;
483
484 def _getHostCpuVendor(self):
485 """
486 Get the CPUID vendor string on intel HW.
487 """
488 return self._getHelperOutput('cpuvendor');
489
490 def _getHostCpuName(self):
491 """
492 Get the CPU name/description string.
493 """
494 return self._getHelperOutput('cpuname');
495
496 def _getHostCpuRevision(self):
497 """
498 Get the CPU revision (family/model/stepping) value.
499 """
500 return self._getHelperOutput('cpurevision');
501
502 def _hasHostHwVirt(self):
503 """
504 Check if the host supports AMD-V or VT-x
505 """
506 if self._oOptions.fHasHwVirt is None:
507 self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
508 return self._oOptions.fHasHwVirt;
509
510 def _hasHostNestedPaging(self):
511 """
512 Check if the host supports nested paging.
513 """
514 if not self._hasHostHwVirt():
515 return False;
516 if self._oOptions.fHasNestedPaging is None:
517 self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
518 return self._oOptions.fHasNestedPaging;
519
520 def _can64BitGuest(self):
521 """
522 Check if the we (VBox) can run 64-bit guests.
523 """
524 if not self._hasHostHwVirt():
525 return False;
526 if self._oOptions.fCan64BitGuest is None:
527 self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
528 return self._oOptions.fCan64BitGuest;
529
530 def _hasHostIoMmu(self):
531 """
532 Check if the host has an I/O MMU of the VT-d kind.
533 """
534 if not self._hasHostHwVirt():
535 return False;
536 if self._oOptions.fHasIoMmu is None:
537 ## @todo Any way to figure this one out on any host OS?
538 self._oOptions.fHasIoMmu = False;
539 return self._oOptions.fHasIoMmu;
540
541 def _withRawModeSupport(self):
542 """
543 Check if the testbox is configured with raw-mode support or not.
544 """
545 if self._oOptions.fWithRawMode is None:
546 self._oOptions.fWithRawMode = True;
547 return self._oOptions.fWithRawMode;
548
549 def _getHostReport(self):
550 """
551 Generate a report about the host hardware and software.
552 """
553 return self._getHelperOutput('report');
554
555
556 def _getHostMemSize(self):
557 """
558 Gets the amount of physical memory on the host (and accessible to the
559 OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
560 Unit: MiB.
561 """
562 cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
563
564 # Round it.
565 cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
566 return cMbMemory;
567
568 def _getFreeScratchSpace(self):
569 """
570 Get free space on the volume where scratch directory is located and
571 return it in bytes rounded down to nearest 64MB
572 (currently works on Linux only)
573 Unit: MiB.
574 """
575 if platform.system() == 'Windows':
576 import ctypes
577 cTypeMbFreeSpace = ctypes.c_ulonglong(0)
578 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
579 ctypes.pointer(cTypeMbFreeSpace))
580 cMbFreeSpace = cTypeMbFreeSpace.value
581 else:
582 stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=E1101
583 cMbFreeSpace = stats.f_frsize * stats.f_bfree
584
585 # Convert to MB
586 cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
587
588 # Round free space size
589 cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
590 return cMbFreeSpace;
591
592 def _getScriptRev(self):
593 """
594 The script (subversion) revision number.
595 """
596 sRev = '@VBOX_SVN_REV@';
597 sRev = sRev.strip(); # just in case...
598 try:
599 _ = int(sRev);
600 except:
601 return __version__[11:-1].strip();
602 return sRev;
603
604 def _getPythonHexVersion(self):
605 """
606 The python hex version number.
607 """
608 uHexVersion = getattr(sys, 'hexversion', None);
609 if uHexVersion is None:
610 uHexVersion = (sys.version_info[0] << 24) | (sys.version_info[1] << 16) | (sys.version_info[2] << 8);
611 if sys.version_info[3] == 'final':
612 uHexVersion |= 0xf0;
613 return uHexVersion;
614
615 # @}
616
617 def openTestManagerConnection(self):
618 """
619 Opens up a connection to the test manager.
620
621 Raises exception on failure.
622 """
623 return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
624
625 def getSignOnParam(self, sName):
626 """
627 Returns a sign-on parameter value as string.
628 Raises exception if the name is incorrect.
629 """
630 return str(self._ddSignOnParams[sName][self.VALUE]);
631
632 def getPathState(self):
633 """
634 Get the path to the state dir in the scratch area.
635 """
636 return self._sScratchState;
637
638 def getPathScripts(self):
639 """
640 Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
641 """
642 return self._sScratchScripts;
643
644 def getPathSpill(self):
645 """
646 Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
647 """
648 return self._sScratchSpill;
649
650 def getPathBuilds(self):
651 """
652 Get the path to the builds.
653 """
654 return self._oOptions.sBuildsPath;
655
656 def getTestBoxId(self):
657 """
658 Get the TestBox ID for state saving purposes.
659 """
660 return self._idTestBox;
661
662 def getTestBoxName(self):
663 """
664 Get the TestBox name for state saving purposes.
665 """
666 return self._sTestBoxName;
667
668 def _reinitScratch(self, fnLog, fUseTheForce):
669 """
670 Wipes the scratch directories and re-initializes them.
671
672 No exceptions raise, returns success indicator instead.
673 """
674 if fUseTheForce is None:
675 fUseTheForce = self._fFirstSignOn;
676
677 class ErrorCallback(object): # pylint: disable=R0903
678 """
679 Callbacks + state for the cleanup.
680 """
681 def __init__(self):
682 self.fRc = True;
683 def onErrorCallback(self, sFnName, sPath, aXcptInfo):
684 """ Logs error during shutil.rmtree operation. """
685 fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
686 self.fRc = False;
687 oRc = ErrorCallback();
688
689 #
690 # Cleanup.
691 #
692 for sName in os.listdir(self._oOptions.sScratchRoot):
693 sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
694 try:
695 if os.path.isdir(sFullName):
696 shutil.rmtree(sFullName, False, oRc.onErrorCallback);
697 else:
698 os.remove(sFullName);
699 if os.path.exists(sFullName):
700 raise Exception('Still exists after deletion, weird.');
701 except Exception as oXcpt:
702 if fUseTheForce is True \
703 and utils.getHostOs() not in ['win', 'os2'] \
704 and len(sFullName) >= 8 \
705 and sFullName[0] == '/' \
706 and sFullName[1] != '/' \
707 and sFullName.find('/../') < 0:
708 fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
709 try:
710 if os.path.isdir(sFullName):
711 iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
712 else:
713 iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
714 if iRc != 0:
715 raise Exception('exit code %s' % iRc);
716 if os.path.exists(sFullName):
717 raise Exception('Still exists after forced deletion, weird^2.');
718 except:
719 fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
720 oRc.fRc = False;
721 else:
722 fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
723 oRc.fRc = False;
724
725 # Display files left behind.
726 def dirEnumCallback(sName, oStat):
727 """ callback for dirEnumerateTree """
728 fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName));
729 utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback);
730
731 #
732 # Re-create the directories.
733 #
734 for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
735 if not os.path.isdir(sDir):
736 try:
737 os.makedirs(sDir, 0o700);
738 except Exception as oXcpt:
739 fnLog('Error creating "%s": %s' % (sDir, oXcpt));
740 oRc.fRc = False;
741
742 if oRc.fRc is True:
743 self._cReinitScratchErrors = 0;
744 else:
745 self._cReinitScratchErrors += 1;
746 return oRc.fRc;
747
748 def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000):
749 """
750 Wipes the scratch directories and re-initializes them.
751
752 Will retry according to the cRetries and cMsDelay parameters. Windows
753 forces us to apply this hack as it ships with services asynchronously
754 scanning files after they execute, thus racing us cleaning up after a
755 test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping
756 vts_rm.exe kind of open, somehow preventing us from removing the
757 directory containing it, despite not issuing any errors deleting the
758 file itself. The service is called "Application Experience", which
759 feels like a weird joke here.
760
761 No exceptions raise, returns success indicator instead.
762 """
763 fRc = self._reinitScratch(fnLog, fUseTheForce)
764 while fRc is False and cRetries > 0:
765 time.sleep(cMsDelay / 1000.0);
766 fnLog('reinitScratch: Retrying...');
767 fRc = self._reinitScratch(fnLog, fUseTheForce)
768 cRetries -= 1;
769 return fRc;
770
771
772 def _doSignOn(self):
773 """
774 Worker for _maybeSignOn that does the actual signing on.
775 """
776 assert not self._oCommand.isRunning();
777
778 # Reset the siged-on state.
779 testboxcommons.log('Signing-on...')
780 self._fSignedOn = False
781 self._idTestBox = None
782 self._cSignOnAttempts += 1;
783
784 # Assemble SIGN-ON request parameters and send the request.
785 dParams = {};
786 for sParam in self._ddSignOnParams:
787 dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
788 oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
789
790 # Check response.
791 try:
792 sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
793 if sResult != constants.tbresp.STATUS_ACK:
794 raise TestBoxException('Result is %s' % (sResult,));
795 oResponse.checkParameterCount(3);
796 idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
797 sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
798 except TestBoxException as err:
799 testboxcommons.log('Failed to sign-on: %s' % (str(err),))
800 testboxcommons.log('Server response: %s' % (oResponse.toString(),));
801 return False;
802
803 # Successfully signed on, update the state.
804 self._fSignedOn = True;
805 self._fNeedReSignOn = False;
806 self._cSignOnAttempts = 0;
807 self._idTestBox = idTestBox;
808 self._sTestBoxName = sTestBoxName;
809
810 # Update the environment.
811 os.environ['TESTBOX_ID'] = str(self._idTestBox);
812 os.environ['TESTBOX_NAME'] = sTestBoxName;
813 os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
814 os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
815 os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
816
817 testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
818 % (self._idTestBox, self._sTestBoxName));
819
820 # Set up the scratch area.
821 self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2);
822
823 self._fFirstSignOn = False;
824 return True;
825
826 def _maybeSignOn(self):
827 """
828 Check if Test Box parameters were changed
829 and do sign-in in case of positive result
830 """
831
832 # Skip sign-on check if background command is currently in
833 # running state (avoid infinite signing on).
834 if self._oCommand.isRunning():
835 return None;
836
837 # Refresh sign-on parameters, changes triggers sign-on.
838 fNeedSignOn = (True if not self._fSignedOn or self._fNeedReSignOn else False)
839 for item in self._ddSignOnParams:
840 if self._ddSignOnParams[item][self.FN] is None:
841 continue
842
843 sOldValue = self._ddSignOnParams[item][self.VALUE]
844 self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
845 if sOldValue != self._ddSignOnParams[item][self.VALUE]:
846 fNeedSignOn = True
847 testboxcommons.log('Detected %s parameter change: %s -> %s' %
848 (item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
849
850 if fNeedSignOn:
851 self._doSignOn();
852 return None;
853
854 def dispatch(self):
855 """
856 Receive orders from Test Manager and execute them
857 """
858
859 (self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
860 self._fNeedReSignOn = self._fSignedOn;
861 if self._fSignedOn:
862 os.environ['TESTBOX_ID'] = str(self._idTestBox);
863 os.environ['TESTBOX_NAME'] = self._sTestBoxName;
864
865 while True:
866 # Make sure we're signed on before trying to do anything.
867 self._maybeSignOn();
868 while not self._fSignedOn:
869 iFactor = 1 if self._cSignOnAttempts < 100 else 4;
870 time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
871 self._maybeSignOn();
872
873 # Retrieve and handle command from the TM.
874 (oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
875 self._idTestBox,
876 self._sTestBoxUuid,
877 self._oCommand.isRunning());
878 if oResponse is not None:
879 self._oCommand.handleCommand(oResponse, oConnection);
880 if oConnection is not None:
881 if oConnection.isConnected():
882 self._oCommand.flushLogOnConnection(oConnection);
883 oConnection.close();
884
885 # Automatically reboot if scratch init fails.
886 #if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False:
887 # testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
888 # % ( self._cReinitScratchErrors, ));
889 # self._oCommand.doReboot();
890
891 # delay a wee bit before looping.
892 ## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
893 # with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
894 iFactor = 1;
895 if self._cReinitScratchErrors > 0:
896 iFactor = 4;
897 time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
898
899 # Not reached.
900
901
902 @staticmethod
903 def main():
904 """
905 Main function a la C/C++. Returns exit code.
906 """
907
908 #
909 # Parse arguments.
910 #
911 sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
912 if utils.getHostOs() in ('win', 'os2'):
913 sDefTestRsrc = 'T:';
914 sDefBuilds = 'U:';
915 elif utils.getHostOs() == 'darwin':
916 sDefTestRsrc = '/Volumes/testrsrc';
917 sDefBuilds = '/Volumes/builds';
918 else:
919 sDefTestRsrc = '/mnt/testrsrc';
920 sDefBuilds = '/mnt/builds';
921
922 class MyOptionParser(OptionParser):
923 """ We need to override the exit code on --help, error and so on. """
924 def __init__(self, *args, **kwargs):
925 OptionParser.__init__(self, *args, **kwargs);
926 def exit(self, status = 0, msg = None):
927 OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
928
929 parser = MyOptionParser(version=__version__[11:-1].strip());
930 for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
931 sLower = sMixed.lower();
932 sPrefix = 's' + sMixed;
933 parser.add_option('--' + sLower + '-path',
934 dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
935 help='Where ' + sDesc + ' can be found');
936 parser.add_option('--' + sLower + '-server-type',
937 dest=sPrefix + 'ServerType', metavar='<nfs|cifs>', default=sDefShareType,
938 help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
939 parser.add_option('--' + sLower + '-server-name',
940 dest=sPrefix + 'ServerName', metavar='<server>',
941 default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
942 help='The name of the server with the builds.');
943 parser.add_option('--' + sLower + '-server-share',
944 dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
945 help='The name of the builds share.');
946 parser.add_option('--' + sLower + '-server-user',
947 dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
948 help='The user name to use when accessing the ' + sDesc + ' share.');
949 parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
950 dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
951 help='The password to use when accessing the ' + sDesc + ' share.');
952
953 parser.add_option("--test-manager", metavar="<url>",
954 dest="sTestManagerUrl",
955 help="Test Manager URL",
956 default="http://tindertux.de.oracle.com/testmanager")
957 parser.add_option("--scratch-root", metavar="<abs-path>",
958 dest="sScratchRoot",
959 help="Path to the scratch directory",
960 default=None)
961 parser.add_option("--system-uuid", metavar="<uuid>",
962 dest="sSystemUuid",
963 help="The system UUID of the testbox, used for uniquely identifiying the machine",
964 default=None)
965 parser.add_option("--hwvirt",
966 dest="fHasHwVirt", action="store_true", default=None,
967 help="Hardware virtualization available in the CPU");
968 parser.add_option("--no-hwvirt",
969 dest="fHasHwVirt", action="store_false", default=None,
970 help="Hardware virtualization not available in the CPU");
971 parser.add_option("--nested-paging",
972 dest="fHasNestedPaging", action="store_true", default=None,
973 help="Nested paging is available");
974 parser.add_option("--no-nested-paging",
975 dest="fHasNestedPaging", action="store_false", default=None,
976 help="Nested paging is not available");
977 parser.add_option("--64-bit-guest",
978 dest="fCan64BitGuest", action="store_true", default=None,
979 help="Host can execute 64-bit guests");
980 parser.add_option("--no-64-bit-guest",
981 dest="fCan64BitGuest", action="store_false", default=None,
982 help="Host cannot execute 64-bit guests");
983 parser.add_option("--io-mmu",
984 dest="fHasIoMmu", action="store_true", default=None,
985 help="I/O MMU available");
986 parser.add_option("--no-io-mmu",
987 dest="fHasIoMmu", action="store_false", default=None,
988 help="No I/O MMU available");
989 parser.add_option("--raw-mode",
990 dest="fWithRawMode", action="store_true", default=None,
991 help="Use raw-mode on this host.");
992 parser.add_option("--no-raw-mode",
993 dest="fWithRawMode", action="store_false", default=None,
994 help="Disables raw-mode tests on this host.");
995 parser.add_option("--pidfile",
996 dest="sPidFile", default=None,
997 help="For the parent script, ignored.");
998 parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
999 dest = "asEnvVars", default = [],
1000 help = "Sets an environment variable. Can be repeated.");
1001
1002 (oOptions, args) = parser.parse_args()
1003 # Check command line
1004 if args != []:
1005 parser.print_help();
1006 return TBS_EXITCODE_SYNTAX;
1007
1008 if oOptions.sSystemUuid is not None:
1009 uuid.UUID(oOptions.sSystemUuid);
1010 if not oOptions.sTestManagerUrl.startswith('http://') \
1011 and not oOptions.sTestManagerUrl.startswith('https://'):
1012 print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
1013 return TBS_EXITCODE_SYNTAX;
1014
1015 for sPrefix in ['sBuilds', 'sTestRsrc']:
1016 sType = getattr(oOptions, sPrefix + 'ServerType');
1017 if sType is None or not sType.strip():
1018 setattr(oOptions, sPrefix + 'ServerType', None);
1019 elif sType not in ['cifs', 'nfs']:
1020 print('Syntax error: Invalid server type "%s"' % (sType,));
1021 return TBS_EXITCODE_SYNTAX;
1022
1023
1024 #
1025 # Instantiate the testbox script and start dispatching work.
1026 #
1027 try:
1028 oTestBoxScript = TestBoxScript(oOptions);
1029 except TestBoxScriptException as oXcpt:
1030 print('Error: %s' % (oXcpt,));
1031 return TBS_EXITCODE_SYNTAX;
1032 oTestBoxScript.dispatch();
1033
1034 # Not supposed to get here...
1035 return TBS_EXITCODE_FAILURE;
1036
1037
1038
1039if __name__ == '__main__':
1040 sys.exit(TestBoxScript.main());
1041
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette