VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/testdriver/vboxtestfileset.py@ 79182

Last change on this file since 79182 was 79182, checked in by vboxsync, 6 years ago

ValidationKit/testdriver/vboxtestfileset.py: SCM fix.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 17.5 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: vboxtestfileset.py 79182 2019-06-17 12:48:28Z vboxsync $
3# pylint: disable=too-many-lines
4
5"""
6VirtualBox Test File Set
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2010-2019 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 79182 $"
31
32
33# Standard Python imports.
34import os;
35import random;
36import string;
37import sys;
38import tarfile;
39
40# Validation Kit imports.
41from common import utils;
42from testdriver import reporter;
43
44# Python 3 hacks:
45if sys.version_info[0] >= 3:
46 xrange = range; # pylint: disable=redefined-builtin,invalid-name
47
48
49
50class GstFsObj(object):
51 """ A file system object we created in the guest for test purposes. """
52 def __init__(self, oParent, sPath):
53 self.oParent = oParent # type: GstDir
54 self.sPath = sPath # type: str
55 self.sName = sPath # type: str
56 if oParent:
57 assert sPath.startswith(oParent.sPath);
58 self.sName = sPath[len(oParent.sPath) + 1:];
59 # Add to parent.
60 oParent.aoChildren.append(self);
61 oParent.dChildrenUpper[self.sName.upper()] = self;
62
63class GstFile(GstFsObj):
64 """ A file object in the guest. """
65 def __init__(self, oParent, sPath, abContent):
66 GstFsObj.__init__(self, oParent, sPath);
67 self.abContent = abContent # type: bytearray
68 self.cbContent = len(abContent);
69 self.off = 0;
70
71 def read(self, cbToRead):
72 assert self.off <= self.cbContent;
73 cbLeft = self.cbContent - self.off;
74 if cbLeft < cbToRead:
75 cbToRead = cbLeft;
76 abRet = self.abContent[self.off:(self.off + cbToRead)];
77 assert len(abRet) == cbToRead;
78 self.off += cbToRead;
79 if sys.version_info[0] < 3:
80 return bytes(abRet);
81 return abRet;
82
83class GstDir(GstFsObj):
84 """ A file object in the guest. """
85 def __init__(self, oParent, sPath):
86 GstFsObj.__init__(self, oParent, sPath);
87 self.aoChildren = [] # type: list(GsFsObj)
88 self.dChildrenUpper = {} # type: dict(str,GsFsObj)
89
90 def contains(self, sName):
91 """ Checks if the directory contains the given name. """
92 return sName.upper() in self.dChildrenUpper
93
94
95class TestFileSet(object):
96 """
97 Generates a set of files and directories for use in a test.
98
99 Uploaded as a tarball and expanded via TXS (if new enough) or uploaded vts_tar
100 utility from the validation kit.
101 """
102
103 ksReservedWinOS2 = '/\\"*:<>?|\t\v\n\r\f\a\b';
104 ksReservedUnix = '/';
105 ksReservedTrailingWinOS2 = ' .';
106 ksReservedTrailingUnix = '';
107
108 def __init__(self, oTestVm, sBasePath, sSubDir, # pylint: disable=too-many-arguments
109 oRngFileSizes = xrange(0, 16384),
110 oRngManyFiles = xrange(128, 512),
111 oRngTreeFiles = xrange(128, 384),
112 oRngTreeDepth = xrange(92, 256),
113 oRngTreeDirs = xrange(2, 16),
114 cchMaxPath = 230,
115 cchMaxName = 230):
116 ## @name Parameters
117 ## @{
118 self.oTestVm = oTestVm;
119 self.sBasePath = sBasePath;
120 self.sSubDir = sSubDir;
121 self.oRngFileSizes = oRngFileSizes;
122 self.oRngManyFiles = oRngManyFiles;
123 self.oRngTreeFiles = oRngTreeFiles;
124 self.oRngTreeDepth = oRngTreeDepth;
125 self.oRngTreeDirs = oRngTreeDirs;
126 self.cchMaxPath = cchMaxPath;
127 self.cchMaxName = cchMaxName
128 ## @}
129
130 ## @name Charset stuff
131 ## @todo allow more chars for unix hosts + guests.
132 ## @{
133 ## The filename charset.
134 self.sFileCharset = string.printable;
135 ## The filename charset common to host and guest.
136 self.sFileCharsetCommon = string.printable;
137 ## Set of characters that should not trail a guest filename.
138 self.sReservedTrailing = self.ksReservedTrailingWinOS2;
139 ## Set of characters that should not trail a host or guest filename.
140 self.sReservedTrailingCommon = self.ksReservedTrailingWinOS2;
141 if oTestVm.isWindows() or oTestVm.isOS2():
142 for ch in self.ksReservedWinOS2:
143 self.sFileCharset = self.sFileCharset.replace(ch, '');
144 else:
145 self.sReservedTrailing = self.ksReservedTrailingUnix;
146 for ch in self.ksReservedUnix:
147 self.sFileCharset = self.sFileCharset.replace(ch, '');
148 self.sFileCharset += ' ...';
149
150 for ch in self.ksReservedWinOS2:
151 self.sFileCharsetCommon = self.sFileCharset.replace(ch, '');
152 self.sFileCharsetCommon += ' ...';
153 ## @}
154
155 ## The root directory.
156 self.oRoot = None # type: GstDir;
157 ## An empty directory (under root).
158 self.oEmptyDir = None # type: GstDir;
159
160 ## A directory with a lot of files in it.
161 self.oManyDir = None # type: GstDir;
162
163 ## A directory with a mixed tree structure under it.
164 self.oTreeDir = None # type: GstDir;
165 ## Number of files in oTreeDir.
166 self.cTreeFiles = 0;
167 ## Number of directories under oTreeDir.
168 self.cTreeDirs = 0;
169 ## Number of other file types under oTreeDir.
170 self.cTreeOthers = 0;
171
172 ## All directories in creation order.
173 self.aoDirs = [] # type: list(GstDir);
174 ## All files in creation order.
175 self.aoFiles = [] # type: list(GstFile);
176 ## Path to object lookup.
177 self.dPaths = {} # type: dict(str, GstFsObj);
178
179 #
180 # Do the creating.
181 #
182 self.uSeed = utils.timestampMilli();
183 self.oRandom = random.Random();
184 self.oRandom.seed(self.uSeed);
185 reporter.log('prepareGuestForTesting: random seed %s' % (self.uSeed,));
186
187 self.__createTestStuff();
188
189 def __createFilename(self, oParent, sCharset, sReservedTrailing):
190 """
191 Creates a filename contains random characters from sCharset and together
192 with oParent.sPath doesn't exceed the given max chars in length.
193 """
194 ## @todo Consider extending this to take UTF-8 and UTF-16 encoding so we
195 ## can safely use the full unicode range. Need to check how
196 ## RTZipTarCmd handles file name encoding in general...
197
198 if oParent:
199 cchMaxName = self.cchMaxPath - len(oParent.sPath) - 1;
200 else:
201 cchMaxName = self.cchMaxPath - 4;
202 if cchMaxName > self.cchMaxName:
203 cchMaxName = self.cchMaxName;
204 if cchMaxName <= 1:
205 cchMaxName = 2;
206
207 while True:
208 cchName = self.oRandom.randrange(1, cchMaxName);
209 sName = ''.join(self.oRandom.choice(sCharset) for _ in xrange(cchName));
210 if oParent is None or not oParent.contains(sName):
211 if sName[-1] not in sReservedTrailing:
212 if sName not in ('.', '..',):
213 return sName;
214 return ''; # never reached, but makes pylint happy.
215
216 def generateFilenameEx(self, cchMax = -1, cchMin = -1, fCommonCharset = True):
217 """
218 Generates a filename according to the given specs.
219
220 This is for external use, whereas __createFilename is for internal.
221
222 Returns generated filename.
223 """
224 assert cchMax == -1 or (cchMax >= 1 and cchMax > cchMin);
225 if cchMin <= 0:
226 cchMin = 1;
227 if cchMax < cchMin:
228 cchMax = self.cchMaxName;
229
230 if fCommonCharset:
231 sCharset = self.sFileCharsetCommon;
232 sReservedTrailing = self.sReservedTrailingCommon;
233 else:
234 sCharset = self.sFileCharset;
235 sReservedTrailing = self.sReservedTrailing;
236
237 while True:
238 cchName = self.oRandom.randrange(cchMin, cchMax + 1);
239 sName = ''.join(self.oRandom.choice(sCharset) for _ in xrange(cchName));
240 if sName[-1] not in sReservedTrailing:
241 if sName not in ('.', '..',):
242 return sName;
243 return ''; # never reached, but makes pylint happy.
244
245 def __createTestDir(self, oParent, sDir):
246 """
247 Creates a test directory.
248 """
249 oDir = GstDir(oParent, sDir);
250 self.aoDirs.append(oDir);
251 self.dPaths[sDir] = oDir;
252 return oDir;
253
254 def __createTestFile(self, oParent, sFile):
255 """
256 Creates a test file with random size up to cbMaxContent and random content.
257 """
258 cbFile = self.oRandom.choice(self.oRngFileSizes);
259 abContent = bytearray(self.oRandom.getrandbits(8) for _ in xrange(cbFile));
260
261 oFile = GstFile(oParent, sFile, abContent);
262 self.aoFiles.append(oFile);
263 self.dPaths[sFile] = oFile;
264 return oFile;
265
266 def __createTestStuff(self):
267 """
268 Create a random file set that we can work on in the tests.
269 Returns True/False.
270 """
271
272 #
273 # Create the root test dir.
274 #
275 sRoot = self.oTestVm.pathJoin(self.sBasePath, self.sSubDir);
276 self.oRoot = self.__createTestDir(None, sRoot);
277 self.oEmptyDir = self.__createTestDir(self.oRoot, self.oTestVm.pathJoin(sRoot, 'empty'));
278
279 #
280 # Create a directory with about files in it using the guest specific charset:
281 #
282 oDir = self.__createTestDir(self.oRoot, self.oTestVm.pathJoin(sRoot, 'many'));
283 self.oManyDir = oDir;
284 cManyFiles = self.oRandom.choice(self.oRngManyFiles);
285 for _ in xrange(cManyFiles):
286 sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
287 self.__createTestFile(oDir, self.oTestVm.pathJoin(oDir.sPath, sName));
288
289 #
290 # Generate a tree of files and dirs. Portable character set.
291 #
292 oDir = self.__createTestDir(self.oRoot, self.oTestVm.pathJoin(sRoot, 'tree'));
293 uMaxDepth = self.oRandom.choice(self.oRngTreeDepth);
294 cMaxFiles = self.oRandom.choice(self.oRngTreeFiles);
295 cMaxDirs = self.oRandom.choice(self.oRngTreeDirs);
296 self.oTreeDir = oDir;
297 self.cTreeFiles = 0;
298 self.cTreeDirs = 0;
299 uDepth = 0;
300 while self.cTreeFiles < cMaxFiles and self.cTreeDirs < cMaxDirs:
301 iAction = self.oRandom.randrange(0, 2+1);
302 # 0: Add a file:
303 if iAction == 0 and self.cTreeFiles < cMaxFiles and len(oDir.sPath) < 230 - 2:
304 sName = self.__createFilename(oDir, self.sFileCharsetCommon, self.sReservedTrailingCommon);
305 self.__createTestFile(oDir, self.oTestVm.pathJoin(oDir.sPath, sName));
306 self.cTreeFiles += 1;
307 # 1: Add a subdirector and descend into it:
308 elif iAction == 1 and self.cTreeDirs < cMaxDirs and uDepth < uMaxDepth and len(oDir.sPath) < 220:
309 sName = self.__createFilename(oDir, self.sFileCharsetCommon, self.sReservedTrailingCommon);
310 oDir = self.__createTestDir(oDir, self.oTestVm.pathJoin(oDir.sPath, sName));
311 self.cTreeDirs += 1;
312 uDepth += 1;
313 # 2: Ascend to parent dir:
314 elif iAction == 2 and uDepth > 0:
315 oDir = oDir.oParent;
316 uDepth -= 1;
317
318 return True;
319
320 def createTarball(self, sTarFileHst):
321 """
322 Creates a tarball on the host.
323 Returns success indicator.
324 """
325 reporter.log('Creating tarball "%s" with test files for the guest...' % (sTarFileHst,));
326
327 chOtherSlash = '\\' if self.oTestVm.isWindows() or self.oTestVm.isOS2() else '/';
328 cchSkip = len(self.sBasePath) + 1;
329
330 # Open the tarball:
331 try:
332 oTarFile = tarfile.open(sTarFileHst, 'w:gz');
333 except:
334 return reporter.errorXcpt('Failed to open new tar file: %s' % (sTarFileHst,));
335
336 # Directories:
337 for oDir in self.aoDirs:
338 oTarInfo = tarfile.TarInfo(oDir.sPath[cchSkip:].replace(chOtherSlash, '/') + '/');
339 oTarInfo.mode = 0o777;
340 oTarInfo.type = tarfile.DIRTYPE;
341 try:
342 oTarFile.addfile(oTarInfo);
343 except:
344 return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oDir.sPath,));
345
346 # Files:
347 for oFile in self.aoFiles:
348 oTarInfo = tarfile.TarInfo(oFile.sPath[cchSkip:].replace(chOtherSlash, '/'));
349 oTarInfo.size = len(oFile.abContent);
350 oFile.off = 0;
351 try:
352 oTarFile.addfile(oTarInfo, oFile);
353 except:
354 return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oFile.sPath,));
355
356 # Complete the tarball.
357 try:
358 oTarFile.close();
359 except:
360 return reporter.errorXcpt('Error closing new tar file: %s' % (sTarFileHst,));
361 return True;
362
363 def __uploadFallback(self, oTxsSession, sTarFileGst, oTstDrv):
364 """
365 Fallback upload method.
366 """
367
368 ## Directories:
369 #for oDir in self.aoDirs:
370 # if oTxsSession.syncMkDirPath(oDir.sPath, 0o777) is not True:
371 # return reporter.error('Failed to create directory "%s"!' % (oDir.sPath,));
372 #
373 ## Files:
374 #for oFile in self.aoFiles:
375 # if oTxsSession.syncUploadString(oFile.abContent, oFile.sPath) is not True:
376 # return reporter.error('Failed to create file "%s" with %s content bytes!' % (oFile.sPath, oFile.cbContent));
377
378 sVtsTarExe = 'vts_tar' + self.oTestVm.getGuestExeSuff();
379 sVtsTarHst = os.path.join(oTstDrv.sVBoxValidationKit, self.oTestVm.getGuestOs(),
380 self.oTestVm.getGuestArch(), sVtsTarExe);
381 sVtsTarGst = self.oTestVm.pathJoin(self.sBasePath, sVtsTarExe);
382
383 if oTxsSession.syncUploadFile(sVtsTarHst, sVtsTarGst) is not True:
384 return reporter.error('Failed to upload "%s" to the guest as "%s"!' % (sVtsTarHst, sVtsTarGst,));
385
386 fRc = oTxsSession.syncExec(sVtsTarGst, [sVtsTarGst, '-xzf', sTarFileGst, '-C', self.sBasePath,], fWithTestPipe = False);
387 if fRc is not True:
388 return reporter.error('vts_tar failed!');
389 return True;
390
391 def upload(self, oTxsSession, oTstDrv):
392 """
393 Uploads the files into the guest via the given TXS session.
394
395 Returns True / False.
396 """
397
398 #
399 # Create a tarball.
400 #
401 sTarFileHst = os.path.join(oTstDrv.sScratchPath, 'tdAddGuestCtrl-1-Stuff.tar.gz');
402 sTarFileGst = self.oTestVm.pathJoin(self.sBasePath, 'tdAddGuestCtrl-1-Stuff.tar.gz');
403 if self.createTarball(sTarFileHst) is not True:
404 return False;
405
406 #
407 # Upload it.
408 #
409 reporter.log('Uploading tarball "%s" to the guest as "%s"...' % (sTarFileHst, sTarFileGst));
410 if oTxsSession.syncUploadFile(sTarFileHst, sTarFileGst) is not True:
411 return reporter.error('Failed upload tarball "%s" as "%s"!' % (sTarFileHst, sTarFileGst,));
412
413 #
414 # Try unpack it.
415 #
416 reporter.log('Unpacking "%s" into "%s"...' % (sTarFileGst, self.sBasePath));
417 if oTxsSession.syncUnpackFile(sTarFileGst, self.sBasePath, fIgnoreErrors = True) is not True:
418 reporter.log('Failed to expand tarball "%s" into "%s", falling back on individual directory and file creation...'
419 % (sTarFileGst, self.sBasePath,));
420 if self.__uploadFallback(oTxsSession, sTarFileGst, oTstDrv) is not True:
421 return False;
422 reporter.log('Successfully placed test files and directories in the VM.');
423 return True;
424
425 def chooseRandomFileFromTree(self):
426 """
427 Returns a random file from the tree (self.oTreeDir).
428 """
429 while True:
430 oFile = self.aoFiles[self.oRandom.choice(xrange(len(self.aoFiles)))];
431 oParent = oFile.oParent;
432 while oParent is not None:
433 if oParent is self.oTreeDir:
434 return oFile;
435 oParent = oParent.oParent;
436
437 def chooseRandomDirFromTree(self, fLeaf = False, fNonEmpty = False):
438 """
439 Returns a random directoryr from the tree (self.oTreeDir).
440 """
441 while True:
442 oDir = self.aoDirs[self.oRandom.choice(xrange(len(self.aoDirs)))];
443 # Check fNonEmpty requirement:
444 if not fNonEmpty or oDir.aoChildren:
445 # Check leaf requirement:
446 if not fLeaf:
447 for oChild in oDir.aoChildren:
448 if isinstance(oChild, GstDir):
449 continue; # skip it.
450
451 # Return if in the tree:
452 oParent = oDir.oParent;
453 while oParent is not None:
454 if oParent is self.oTreeDir:
455 return oDir;
456 oParent = oParent.oParent;
457
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette