VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 97271

Last change on this file since 97271 was 96407, checked in by vboxsync, 2 years ago

scm copyright and license note update

  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 47.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: testboxscript_real.py 96407 2022-08-22 17:43:14Z vboxsync $
4
5"""
6TestBox Script - main().
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2022 Oracle and/or its affiliates.
12
13This file is part of VirtualBox base platform packages, as
14available from https://www.virtualbox.org.
15
16This program is free software; you can redistribute it and/or
17modify it under the terms of the GNU General Public License
18as published by the Free Software Foundation, in version 3 of the
19License.
20
21This program is distributed in the hope that it will be useful, but
22WITHOUT ANY WARRANTY; without even the implied warranty of
23MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24General Public License for more details.
25
26You should have received a copy of the GNU General Public License
27along with this program; if not, see <https://www.gnu.org/licenses>.
28
29The contents of this file may alternatively be used under the terms
30of the Common Development and Distribution License Version 1.0
31(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32in the VirtualBox distribution, in which case the provisions of the
33CDDL are applicable instead of those of the GPL.
34
35You may elect to license modified versions of this file under the
36terms and conditions of either the GPL or the CDDL or both.
37
38SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39"""
40__version__ = "$Revision: 96407 $"
41
42
43# Standard python imports.
44import math
45import os
46from optparse import OptionParser # pylint: disable=deprecated-module
47import platform
48import random
49import shutil
50import sys
51import tempfile
52import time
53import uuid
54
55# Only the main script needs to modify the path.
56try: __file__
57except: __file__ = sys.argv[0];
58g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
59g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
60sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
61
62# Validation Kit imports.
63from common import constants;
64from common import utils;
65import testboxcommons;
66from testboxcommons import TestBoxException;
67from testboxcommand import TestBoxCommand;
68from testboxconnection import TestBoxConnection;
69from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
70
71# Python 3 hacks:
72if sys.version_info[0] >= 3:
73 long = int; # pylint: disable=redefined-builtin,invalid-name
74
75
76class TestBoxScriptException(Exception):
77 """ For raising exceptions during TestBoxScript.__init__. """
78 pass; # pylint: disable=unnecessary-pass
79
80
81class TestBoxScript(object):
82 """
83 Implementation of the test box script.
84 Communicate with test manager and perform offered actions.
85 """
86
87 ## @name Class Constants.
88 # @{
89
90 # Scratch space round value (MB).
91 kcMbScratchSpaceRounding = 64
92 # Memory size round value (MB).
93 kcMbMemoryRounding = 4
94 # A NULL UUID in string form.
95 ksNullUuid = '00000000-0000-0000-0000-000000000000';
96 # The minimum dispatch loop delay.
97 kcSecMinDelay = 12;
98 # The maximum dispatch loop delay (inclusive).
99 kcSecMaxDelay = 24;
100 # The minimum sign-on delay.
101 kcSecMinSignOnDelay = 30;
102 # The maximum sign-on delay (inclusive).
103 kcSecMaxSignOnDelay = 60;
104
105 # Keys for config params
106 VALUE = 'value'
107 FN = 'fn' # pylint: disable=invalid-name
108
109 ## @}
110
111
112 def __init__(self, oOptions):
113 """
114 Initialize internals
115 """
116 self._oOptions = oOptions;
117 self._sTestBoxHelper = None;
118
119 # Signed-on state
120 self._cSignOnAttempts = 0;
121 self._fSignedOn = False;
122 self._fNeedReSignOn = False;
123 self._fFirstSignOn = True;
124 self._idTestBox = None;
125 self._sTestBoxName = '';
126 self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
127
128 # Command processor.
129 self._oCommand = TestBoxCommand(self);
130
131 #
132 # Scratch dir setup. Use /var/tmp instead of /tmp because we may need
133 # many many GBs for some test scenarios and /tmp can be backed by swap
134 # or be a fast+small disk of some kind, while /var/tmp is normally
135 # larger, if slower. /var/tmp is generally not cleaned up on reboot,
136 # /tmp often is, this would break host panic / triple-fault detection.
137 #
138 if self._oOptions.sScratchRoot is None:
139 if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
140 # We need *lots* of space, so avoid /tmp as it may be a memory
141 # file system backed by the swap file, or worse.
142 self._oOptions.sScratchRoot = tempfile.gettempdir();
143 else:
144 self._oOptions.sScratchRoot = '/var/tmp';
145 sSubDir = 'testbox';
146 try:
147 sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=no-member
148 except:
149 pass;
150 self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
151
152 self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
153 self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
154 self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
155
156 for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
157 if not os.path.isdir(sDir):
158 os.makedirs(sDir, 0o700);
159
160 # We count consecutive reinitScratch failures and will reboot the
161 # testbox after a while in the hope that it will correct the issue.
162 self._cReinitScratchErrors = 0;
163
164 #
165 # Mount builds and test resources if requested.
166 #
167 self.mountShares();
168
169 #
170 # Sign-on parameters: Packed into list of records of format:
171 # { <Parameter ID>: { <Current value>, <Check function> } }
172 #
173 self._ddSignOnParams = \
174 {
175 constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
176 constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
177 constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
178 constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
179 constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
180 constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
181 constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
182 constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
183 constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
184 constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
185 constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
186 #constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
187 constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
188 constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
189 constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
190 constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: utils.getPresentCpuCount },
191 constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
192 constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
193 }
194 for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
195 if self._ddSignOnParams[sItem][self.FN] is not None:
196 self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
197
198 testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
199 testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
200 testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
201 for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
202 testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
203
204 #
205 # The System UUID is the primary identification of the machine, so
206 # refuse to cooperate if it's NULL.
207 #
208 self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
209 if self._sTestBoxUuid == self.ksNullUuid:
210 raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
211
212 #
213 # Export environment variables, clearing any we don't know yet.
214 #
215 for sEnvVar in self._oOptions.asEnvVars:
216 iEqual = sEnvVar.find('=');
217 if iEqual == -1: # No '=', remove it.
218 if sEnvVar in os.environ:
219 del os.environ[sEnvVar];
220 elif iEqual > 0: # Set it.
221 os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
222 else: # Starts with '=', bad user.
223 raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
224
225 os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
226 os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
227 os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
228 os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
229 os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
230 os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
231 os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
232 os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
233 os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
234 os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
235 os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
236 os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
237 #TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
238 os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
239 os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
240 os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
241 os.environ['TESTBOX_REPORTER'] = 'remote';
242 os.environ['TESTBOX_NAME'] = '';
243 os.environ['TESTBOX_ID'] = '';
244 os.environ['TESTBOX_TEST_SET_ID'] = '';
245 os.environ['TESTBOX_TIMEOUT'] = '0';
246 os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
247
248 if utils.getHostOs() == 'win':
249 os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
250 # Currently omitting any kBuild tools.
251
252 def mountShares(self):
253 """
254 Mounts the shares.
255 Raises exception on failure.
256 """
257 self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
258 self._oOptions.sBuildsServerShare,
259 self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd,
260 self._oOptions.sBuildsServerMountOpt, 'builds');
261 self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
262 self._oOptions.sTestRsrcServerShare,
263 self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd,
264 self._oOptions.sTestRsrcServerMountOpt, 'testrsrc');
265 return True;
266
267 def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sMountOpt, sWhat):
268 """
269 Mounts the specified share if needed.
270 Raises exception on failure.
271 """
272 # Only mount if the type is specified.
273 if sType is None:
274 return True;
275
276 # Test if already mounted.
277 sTestFile = os.path.join(sMountPoint + os.path.sep, os.path.basename(sShare) + '-new.txt');
278 if os.path.isfile(sTestFile):
279 return True;
280
281 #
282 # Platform specific mount code.
283 #
284 sHostOs = utils.getHostOs()
285 if sHostOs in ('darwin', 'freebsd'):
286 if sMountOpt != '':
287 sMountOpt = ',' + sMountOpt
288 utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
289 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
290 utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=no-member
291 if sType == 'cifs':
292 # Note! no smb://server/share stuff here, 10.6.8 didn't like it.
293 utils.processOutputChecked(['/sbin/mount_smbfs',
294 '-o',
295 'automounted,nostreams,soft,noowners,noatime,rdonly' + sMountOpt,
296 '-f', '0555', '-d', '0555',
297 '//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
298 sMountPoint]);
299 else:
300 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
301
302 elif sHostOs == 'linux':
303 if sMountOpt != '':
304 sMountOpt = ',' + sMountOpt
305 utils.sudoProcessCall(['/bin/umount', sMountPoint]);
306 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
307 if sType == 'cifs':
308 utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
309 '-o',
310 'user=' + sUser
311 + ',password=' + sPassword
312 + ',sec=ntlmv2'
313 + ',uid=' + str(os.getuid()) # pylint: disable=no-member
314 + ',gid=' + str(os.getgid()) # pylint: disable=no-member
315 + ',nounix,file_mode=0555,dir_mode=0555,soft,ro'
316 + sMountOpt,
317 '//%s/%s' % (sServer, sShare),
318 sMountPoint]);
319 elif sType == 'nfs':
320 utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
321 '-o', 'soft,ro' + sMountOpt,
322 '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
323 sMountPoint]);
324
325 else:
326 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
327
328 elif sHostOs == 'solaris':
329 if sMountOpt != '':
330 sMountOpt = ',' + sMountOpt
331 utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
332 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
333 if sType == 'cifs':
334 ## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
335 oPasswdFile = tempfile.TemporaryFile(); # pylint: disable=consider-using-with
336 oPasswdFile.write(sPassword + '\n');
337 oPasswdFile.flush();
338 utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
339 '-o',
340 'user=' + sUser
341 + ',uid=' + str(os.getuid()) # pylint: disable=no-member
342 + ',gid=' + str(os.getgid()) # pylint: disable=no-member
343 + ',fileperms=0555,dirperms=0555,noxattr,ro'
344 + sMountOpt,
345 '//%s/%s' % (sServer, sShare),
346 sMountPoint],
347 stdin = oPasswdFile);
348 oPasswdFile.close();
349 elif sType == 'nfs':
350 utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
351 '-o', 'noxattr,ro' + sMountOpt,
352 '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
353 sMountPoint]);
354
355 else:
356 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
357
358
359 elif sHostOs == 'win':
360 if sType != 'cifs':
361 raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
362 utils.processCall(['net', 'use', sMountPoint, '/d']);
363 utils.processOutputChecked(['net', 'use', sMountPoint,
364 '\\\\' + sServer + '\\' + sShare,
365 sPassword,
366 '/USER:' + sUser,]);
367 else:
368 raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
369
370 #
371 # Re-test.
372 #
373 if not os.path.isfile(sTestFile):
374 raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
375 % (sWhat, sServer, sShare, sMountPoint, sTestFile));
376
377 return True;
378
379 ## @name Signon property releated methods.
380 # @{
381
382 def _getHelperOutput(self, sCmd):
383 """
384 Invokes TestBoxHelper to obtain information hard to access from python.
385 """
386 if self._sTestBoxHelper is None:
387 if not utils.isRunningFromCheckout():
388 # See VBoxTestBoxScript.zip for layout.
389 self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
390 'TestBoxHelper');
391 else: # Only for in-tree testing, so don't bother be too accurate right now.
392 sType = os.environ.get('KBUILD_TYPE', 'debug');
393 self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
394 utils.getHostOsDotArch(), sType, 'testboxscript', \
395 utils.getHostOs(), utils.getHostArch(), \
396 'TestBoxHelper');
397 if utils.getHostOs() in ['win', 'os2']:
398 self._sTestBoxHelper += '.exe';
399
400 return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
401
402 def _getHelperOutputTristate(self, sCmd, fDunnoValue):
403 """
404 Invokes TestBoxHelper to obtain information hard to access from python.
405 """
406 sValue = self._getHelperOutput(sCmd);
407 sValue = sValue.lower();
408 if sValue == 'true':
409 return True;
410 if sValue == 'false':
411 return False;
412 if sValue not in ('dunno', 'none',):
413 raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
414 return fDunnoValue;
415
416
417 @staticmethod
418 def _isUuidGood(sUuid):
419 """
420 Checks if the UUID looks good.
421
422 There are systems with really bad UUIDs, for instance
423 "03000200-0400-0500-0006-000700080009".
424 """
425 if sUuid == TestBoxScript.ksNullUuid:
426 return False;
427 sUuid = sUuid.lower();
428 for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
429 if sUuid.count(sDigit) > 16:
430 return False;
431 return True;
432
433 def _getHostSystemUuid(self):
434 """
435 Get the system UUID string from the System, return null-uuid if
436 unable to get retrieve it.
437 """
438 if self._oOptions.sSystemUuid is not None:
439 return self._oOptions.sSystemUuid;
440
441 sUuid = self.ksNullUuid;
442
443 #
444 # Try get at the firmware UUID.
445 #
446 if utils.getHostOs() == 'linux':
447 # NOTE: This requires to have kernel option enabled:
448 # Firmware Drivers -> Export DMI identification via sysfs to userspace
449 if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
450 try:
451 sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
452 sUuid = str(uuid.UUID(sVar.strip()));
453 except:
454 pass;
455 ## @todo consider dmidecoder? What about EFI systems?
456
457 elif utils.getHostOs() == 'win':
458 # Windows: WMI
459 try:
460 import win32com.client; # pylint: disable=import-error
461 oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
462 oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
463 for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
464 if oItem.UUID is not None:
465 sUuid = str(uuid.UUID(oItem.UUID));
466 except:
467 pass;
468
469 elif utils.getHostOs() == 'darwin':
470 try:
471 sVar = utils.processOutputChecked(['/bin/sh', '-c',
472 '/usr/sbin/ioreg -k IOPlatformUUID' \
473 + '| /usr/bin/grep IOPlatformUUID' \
474 + '| /usr/bin/head -1']);
475 sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
476 sUuid = str(uuid.UUID(sVar));
477 except:
478 pass;
479
480 elif utils.getHostOs() == 'solaris':
481 # Solaris: The smbios util.
482 try:
483 sVar = utils.processOutputChecked(['/bin/sh', '-c',
484 '/usr/sbin/smbios ' \
485 + '| /usr/xpg4/bin/sed -ne \'s/^.*UUID: *//p\'' \
486 + '| /usr/bin/head -1']);
487 sUuid = str(uuid.UUID(sVar.strip()));
488 except:
489 pass;
490
491 if self._isUuidGood(sUuid):
492 return sUuid;
493
494 #
495 # Try add the MAC address.
496 # uuid.getnode may provide it, or it may return a random number...
497 #
498 lMacAddr = uuid.getnode();
499 sNode = '%012x' % (lMacAddr,)
500 if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
501 return sUuid[:-12] + sNode;
502
503 return sUuid;
504
505 def _getHostCpuVendor(self):
506 """
507 Get the CPUID vendor string on intel HW.
508 """
509 return self._getHelperOutput('cpuvendor');
510
511 def _getHostCpuName(self):
512 """
513 Get the CPU name/description string.
514 """
515 return self._getHelperOutput('cpuname');
516
517 def _getHostCpuRevision(self):
518 """
519 Get the CPU revision (family/model/stepping) value.
520 """
521 return self._getHelperOutput('cpurevision');
522
523 def _hasHostHwVirt(self):
524 """
525 Check if the host supports AMD-V or VT-x
526 """
527 if self._oOptions.fHasHwVirt is None:
528 self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
529 return self._oOptions.fHasHwVirt;
530
531 def _hasHostNestedPaging(self):
532 """
533 Check if the host supports nested paging.
534 """
535 if not self._hasHostHwVirt():
536 return False;
537 if self._oOptions.fHasNestedPaging is None:
538 self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
539 return self._oOptions.fHasNestedPaging;
540
541 def _can64BitGuest(self):
542 """
543 Check if the we (VBox) can run 64-bit guests.
544 """
545 if not self._hasHostHwVirt():
546 return False;
547 if self._oOptions.fCan64BitGuest is None:
548 self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
549 return self._oOptions.fCan64BitGuest;
550
551 def _hasHostIoMmu(self):
552 """
553 Check if the host has an I/O MMU of the VT-d kind.
554 """
555 if not self._hasHostHwVirt():
556 return False;
557 if self._oOptions.fHasIoMmu is None:
558 ## @todo Any way to figure this one out on any host OS?
559 self._oOptions.fHasIoMmu = False;
560 return self._oOptions.fHasIoMmu;
561
562 def _withRawModeSupport(self):
563 """
564 Check if the testbox is configured with raw-mode support or not.
565 """
566 if self._oOptions.fWithRawMode is None:
567 self._oOptions.fWithRawMode = True;
568 return self._oOptions.fWithRawMode;
569
570 def _getHostReport(self):
571 """
572 Generate a report about the host hardware and software.
573 """
574 return self._getHelperOutput('report');
575
576
577 def _getHostMemSize(self):
578 """
579 Gets the amount of physical memory on the host (and accessible to the
580 OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
581 Unit: MiB.
582 """
583 cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
584
585 # Round it.
586 cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
587 return cMbMemory;
588
589 def _getFreeScratchSpace(self):
590 """
591 Get free space on the volume where scratch directory is located and
592 return it in bytes rounded down to nearest 64MB
593 (currently works on Linux only)
594 Unit: MiB.
595 """
596 if platform.system() == 'Windows':
597 import ctypes
598 cTypeMbFreeSpace = ctypes.c_ulonglong(0)
599 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
600 ctypes.pointer(cTypeMbFreeSpace))
601 cMbFreeSpace = cTypeMbFreeSpace.value
602 else:
603 stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=no-member
604 cMbFreeSpace = stats.f_frsize * stats.f_bfree
605
606 # Convert to MB
607 cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
608
609 # Round free space size
610 cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
611 return cMbFreeSpace;
612
613 def _getScriptRev(self):
614 """
615 The script (subversion) revision number.
616 """
617 sRev = '@VBOX_SVN_REV@';
618 sRev = sRev.strip(); # just in case...
619 try:
620 _ = int(sRev);
621 except:
622 return __version__[11:-1].strip();
623 return sRev;
624
625 def _getPythonHexVersion(self):
626 """
627 The python hex version number.
628 """
629 uHexVersion = getattr(sys, 'hexversion', None);
630 if uHexVersion is None:
631 uHexVersion = (sys.version_info[0] << 24) | (sys.version_info[1] << 16) | (sys.version_info[2] << 8);
632 if sys.version_info[3] == 'final':
633 uHexVersion |= 0xf0;
634 return uHexVersion;
635
636 # @}
637
638 def openTestManagerConnection(self):
639 """
640 Opens up a connection to the test manager.
641
642 Raises exception on failure.
643 """
644 return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
645
646 def getSignOnParam(self, sName):
647 """
648 Returns a sign-on parameter value as string.
649 Raises exception if the name is incorrect.
650 """
651 return str(self._ddSignOnParams[sName][self.VALUE]);
652
653 def getPathState(self):
654 """
655 Get the path to the state dir in the scratch area.
656 """
657 return self._sScratchState;
658
659 def getPathScripts(self):
660 """
661 Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
662 """
663 return self._sScratchScripts;
664
665 def getPathSpill(self):
666 """
667 Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
668 """
669 return self._sScratchSpill;
670
671 def getPathBuilds(self):
672 """
673 Get the path to the builds.
674 """
675 return self._oOptions.sBuildsPath;
676
677 def getTestBoxId(self):
678 """
679 Get the TestBox ID for state saving purposes.
680 """
681 return self._idTestBox;
682
683 def getTestBoxName(self):
684 """
685 Get the TestBox name for state saving purposes.
686 """
687 return self._sTestBoxName;
688
689 def _reinitScratch(self, fnLog, fUseTheForce):
690 """
691 Wipes the scratch directories and re-initializes them.
692
693 No exceptions raise, returns success indicator instead.
694 """
695 if fUseTheForce is None:
696 fUseTheForce = self._fFirstSignOn;
697
698 class ErrorCallback(object): # pylint: disable=too-few-public-methods
699 """
700 Callbacks + state for the cleanup.
701 """
702 def __init__(self):
703 self.fRc = True;
704 def onErrorCallback(self, sFnName, sPath, aXcptInfo):
705 """ Logs error during shutil.rmtree operation. """
706 fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
707 self.fRc = False;
708 oRc = ErrorCallback();
709
710 #
711 # Cleanup.
712 #
713 for sName in os.listdir(self._oOptions.sScratchRoot):
714 sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
715 try:
716 if os.path.isdir(sFullName):
717 shutil.rmtree(sFullName, False, oRc.onErrorCallback);
718 else:
719 os.remove(sFullName);
720 if os.path.exists(sFullName):
721 raise Exception('Still exists after deletion, weird.');
722 except Exception as oXcpt:
723 if fUseTheForce is True \
724 and utils.getHostOs() not in ['win', 'os2'] \
725 and len(sFullName) >= 8 \
726 and sFullName[0] == '/' \
727 and sFullName[1] != '/' \
728 and sFullName.find('/../') < 0:
729 fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
730 try:
731 if os.path.isdir(sFullName):
732 iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
733 else:
734 iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
735 if iRc != 0:
736 raise Exception('exit code %s' % iRc);
737 if os.path.exists(sFullName):
738 raise Exception('Still exists after forced deletion, weird^2.');
739 except:
740 fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
741 oRc.fRc = False;
742 else:
743 fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
744 oRc.fRc = False;
745
746 # Display files left behind.
747 def dirEnumCallback(sName, oStat):
748 """ callback for dirEnumerateTree """
749 fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName));
750 utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback);
751
752 #
753 # Re-create the directories.
754 #
755 for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
756 if not os.path.isdir(sDir):
757 try:
758 os.makedirs(sDir, 0o700);
759 except Exception as oXcpt:
760 fnLog('Error creating "%s": %s' % (sDir, oXcpt));
761 oRc.fRc = False;
762
763 if oRc.fRc is True:
764 self._cReinitScratchErrors = 0;
765 else:
766 self._cReinitScratchErrors += 1;
767 return oRc.fRc;
768
769 def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000):
770 """
771 Wipes the scratch directories and re-initializes them.
772
773 Will retry according to the cRetries and cMsDelay parameters. Windows
774 forces us to apply this hack as it ships with services asynchronously
775 scanning files after they execute, thus racing us cleaning up after a
776 test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping
777 vts_rm.exe kind of open, somehow preventing us from removing the
778 directory containing it, despite not issuing any errors deleting the
779 file itself. The service is called "Application Experience", which
780 feels like a weird joke here.
781
782 No exceptions raise, returns success indicator instead.
783 """
784 fRc = self._reinitScratch(fnLog, fUseTheForce)
785 while fRc is False and cRetries > 0:
786 time.sleep(cMsDelay / 1000.0);
787 fnLog('reinitScratch: Retrying...');
788 fRc = self._reinitScratch(fnLog, fUseTheForce)
789 cRetries -= 1;
790 return fRc;
791
792
793 def _doSignOn(self):
794 """
795 Worker for _maybeSignOn that does the actual signing on.
796 """
797 assert not self._oCommand.isRunning();
798
799 # Reset the siged-on state.
800 testboxcommons.log('Signing-on...')
801 self._fSignedOn = False
802 self._idTestBox = None
803 self._cSignOnAttempts += 1;
804
805 # Assemble SIGN-ON request parameters and send the request.
806 dParams = {};
807 for sParam in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
808 dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
809 oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
810
811 # Check response.
812 try:
813 sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
814 if sResult != constants.tbresp.STATUS_ACK:
815 raise TestBoxException('Result is %s' % (sResult,));
816 oResponse.checkParameterCount(3);
817 idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
818 sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
819 except TestBoxException as err:
820 testboxcommons.log('Failed to sign-on: %s' % (str(err),))
821 testboxcommons.log('Server response: %s' % (oResponse.toString(),));
822 return False;
823
824 # Successfully signed on, update the state.
825 self._fSignedOn = True;
826 self._fNeedReSignOn = False;
827 self._cSignOnAttempts = 0;
828 self._idTestBox = idTestBox;
829 self._sTestBoxName = sTestBoxName;
830
831 # Update the environment.
832 os.environ['TESTBOX_ID'] = str(self._idTestBox);
833 os.environ['TESTBOX_NAME'] = sTestBoxName;
834 os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
835 os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
836 os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
837
838 testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
839 % (self._idTestBox, self._sTestBoxName));
840
841 # Set up the scratch area.
842 self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2);
843
844 self._fFirstSignOn = False;
845 return True;
846
847 def _maybeSignOn(self):
848 """
849 Check if Test Box parameters were changed
850 and do sign-in in case of positive result
851 """
852
853 # Skip sign-on check if background command is currently in
854 # running state (avoid infinite signing on).
855 if self._oCommand.isRunning():
856 return None;
857
858 # Refresh sign-on parameters, changes triggers sign-on.
859 fNeedSignOn = not self._fSignedOn or self._fNeedReSignOn;
860 for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items
861 if self._ddSignOnParams[sItem][self.FN] is None:
862 continue
863
864 sOldValue = self._ddSignOnParams[sItem][self.VALUE]
865 self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
866 if sOldValue != self._ddSignOnParams[sItem][self.VALUE]:
867 fNeedSignOn = True
868 testboxcommons.log('Detected %s parameter change: %s -> %s'
869 % (sItem, sOldValue, self._ddSignOnParams[sItem][self.VALUE],))
870
871 if fNeedSignOn:
872 self._doSignOn();
873 return None;
874
875 def dispatch(self):
876 """
877 Receive orders from Test Manager and execute them
878 """
879
880 (self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
881 self._fNeedReSignOn = self._fSignedOn;
882 if self._fSignedOn:
883 os.environ['TESTBOX_ID'] = str(self._idTestBox);
884 os.environ['TESTBOX_NAME'] = self._sTestBoxName;
885
886 while True:
887 # Make sure we're signed on before trying to do anything.
888 self._maybeSignOn();
889 while not self._fSignedOn:
890 iFactor = 1 if self._cSignOnAttempts < 100 else 4;
891 time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
892 self._maybeSignOn();
893
894 # Retrieve and handle command from the TM.
895 (oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
896 self._idTestBox,
897 self._sTestBoxUuid,
898 self._oCommand.isRunning());
899 if oResponse is not None:
900 self._oCommand.handleCommand(oResponse, oConnection);
901 if oConnection is not None:
902 if oConnection.isConnected():
903 self._oCommand.flushLogOnConnection(oConnection);
904 oConnection.close();
905
906 # Automatically reboot if scratch init fails.
907 #if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False:
908 # testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
909 # % ( self._cReinitScratchErrors, ));
910 # self._oCommand.doReboot();
911
912 # delay a wee bit before looping.
913 ## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
914 # with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
915 iFactor = 1;
916 if self._cReinitScratchErrors > 0:
917 iFactor = 4;
918 time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
919
920 # Not reached.
921
922
923 @staticmethod
924 def main():
925 """
926 Main function a la C/C++. Returns exit code.
927 """
928
929 #
930 # Parse arguments.
931 #
932 sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
933 if utils.getHostOs() in ('win', 'os2'):
934 sDefTestRsrc = 'T:';
935 sDefBuilds = 'U:';
936 elif utils.getHostOs() == 'darwin':
937 sDefTestRsrc = '/Volumes/testrsrc';
938 sDefBuilds = '/Volumes/builds';
939 else:
940 sDefTestRsrc = '/mnt/testrsrc';
941 sDefBuilds = '/mnt/builds';
942
943 class MyOptionParser(OptionParser):
944 """ We need to override the exit code on --help, error and so on. """
945 def __init__(self, *args, **kwargs):
946 OptionParser.__init__(self, *args, **kwargs);
947 def exit(self, status = 0, msg = None):
948 OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
949
950 parser = MyOptionParser(version=__version__[11:-1].strip());
951 for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
952 sLower = sMixed.lower();
953 sPrefix = 's' + sMixed;
954 parser.add_option('--' + sLower + '-path',
955 dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
956 help='Where ' + sDesc + ' can be found');
957 parser.add_option('--' + sLower + '-server-type',
958 dest=sPrefix + 'ServerType', metavar='<nfs|cifs>', default=sDefShareType,
959 help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
960 parser.add_option('--' + sLower + '-server-name',
961 dest=sPrefix + 'ServerName', metavar='<server>',
962 default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
963 help='The name of the server with the builds.');
964 parser.add_option('--' + sLower + '-server-share',
965 dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
966 help='The name of the builds share.');
967 parser.add_option('--' + sLower + '-server-user',
968 dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
969 help='The user name to use when accessing the ' + sDesc + ' share.');
970 parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
971 dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
972 help='The password to use when accessing the ' + sDesc + ' share.');
973 parser.add_option('--' + sLower + '-server-mountopt',
974 dest=sPrefix + 'ServerMountOpt', metavar='<mountopt>', default='',
975 help='The mount options to use when accessing the ' + sDesc + ' share.');
976
977 parser.add_option("--test-manager", metavar="<url>",
978 dest="sTestManagerUrl",
979 help="Test Manager URL",
980 default="http://tindertux.de.oracle.com/testmanager")
981 parser.add_option("--scratch-root", metavar="<abs-path>",
982 dest="sScratchRoot",
983 help="Path to the scratch directory",
984 default=None)
985 parser.add_option("--system-uuid", metavar="<uuid>",
986 dest="sSystemUuid",
987 help="The system UUID of the testbox, used for uniquely identifiying the machine",
988 default=None)
989 parser.add_option("--hwvirt",
990 dest="fHasHwVirt", action="store_true", default=None,
991 help="Hardware virtualization available in the CPU");
992 parser.add_option("--no-hwvirt",
993 dest="fHasHwVirt", action="store_false", default=None,
994 help="Hardware virtualization not available in the CPU");
995 parser.add_option("--nested-paging",
996 dest="fHasNestedPaging", action="store_true", default=None,
997 help="Nested paging is available");
998 parser.add_option("--no-nested-paging",
999 dest="fHasNestedPaging", action="store_false", default=None,
1000 help="Nested paging is not available");
1001 parser.add_option("--64-bit-guest",
1002 dest="fCan64BitGuest", action="store_true", default=None,
1003 help="Host can execute 64-bit guests");
1004 parser.add_option("--no-64-bit-guest",
1005 dest="fCan64BitGuest", action="store_false", default=None,
1006 help="Host cannot execute 64-bit guests");
1007 parser.add_option("--io-mmu",
1008 dest="fHasIoMmu", action="store_true", default=None,
1009 help="I/O MMU available");
1010 parser.add_option("--no-io-mmu",
1011 dest="fHasIoMmu", action="store_false", default=None,
1012 help="No I/O MMU available");
1013 parser.add_option("--raw-mode",
1014 dest="fWithRawMode", action="store_true", default=None,
1015 help="Use raw-mode on this host.");
1016 parser.add_option("--no-raw-mode",
1017 dest="fWithRawMode", action="store_false", default=None,
1018 help="Disables raw-mode tests on this host.");
1019 parser.add_option("--pidfile",
1020 dest="sPidFile", default=None,
1021 help="For the parent script, ignored.");
1022 parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
1023 dest = "asEnvVars", default = [],
1024 help = "Sets an environment variable. Can be repeated.");
1025 def sbp_callback(option, opt_str, value, parser):
1026 _, _, _ = opt_str, value, option
1027 parser.values.sTestManagerUrl = 'http://10.162.100.8/testmanager/'
1028 parser.values.sBuildsServerName = 'vbox-st02.ru.oracle.com'
1029 parser.values.sTestRsrcServerName = 'vbox-st02.ru.oracle.com'
1030 parser.values.sTestRsrcServerShare = 'scratch/data/testrsrc'
1031 parser.add_option("--spb", "--load-sbp-defaults", action="callback", callback=sbp_callback,
1032 help="Load defaults for the sbp setup.")
1033
1034 (oOptions, args) = parser.parse_args()
1035 # Check command line
1036 if args != []:
1037 parser.print_help();
1038 return TBS_EXITCODE_SYNTAX;
1039
1040 if oOptions.sSystemUuid is not None:
1041 uuid.UUID(oOptions.sSystemUuid);
1042 if not oOptions.sTestManagerUrl.startswith('http://') \
1043 and not oOptions.sTestManagerUrl.startswith('https://'):
1044 print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
1045 return TBS_EXITCODE_SYNTAX;
1046
1047 for sPrefix in ['sBuilds', 'sTestRsrc']:
1048 sType = getattr(oOptions, sPrefix + 'ServerType');
1049 if sType is None or not sType.strip():
1050 setattr(oOptions, sPrefix + 'ServerType', None);
1051 elif sType not in ['cifs', 'nfs']:
1052 print('Syntax error: Invalid server type "%s"' % (sType,));
1053 return TBS_EXITCODE_SYNTAX;
1054
1055
1056 #
1057 # Instantiate the testbox script and start dispatching work.
1058 #
1059 try:
1060 oTestBoxScript = TestBoxScript(oOptions);
1061 except TestBoxScriptException as oXcpt:
1062 print('Error: %s' % (oXcpt,));
1063 return TBS_EXITCODE_SYNTAX;
1064 oTestBoxScript.dispatch();
1065
1066 # Not supposed to get here...
1067 return TBS_EXITCODE_FAILURE;
1068
1069
1070
1071if __name__ == '__main__':
1072 sys.exit(TestBoxScript.main());
1073
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette