| 1 | # -*- coding: utf-8 -*-
|
|---|
| 2 | # $Id: testfileset.py 96407 2022-08-22 17:43:14Z vboxsync $
|
|---|
| 3 | # pylint: disable=too-many-lines
|
|---|
| 4 |
|
|---|
| 5 | """
|
|---|
| 6 | Test File Set
|
|---|
| 7 | """
|
|---|
| 8 |
|
|---|
| 9 | __copyright__ = \
|
|---|
| 10 | """
|
|---|
| 11 | Copyright (C) 2010-2022 Oracle and/or its affiliates.
|
|---|
| 12 |
|
|---|
| 13 | This file is part of VirtualBox base platform packages, as
|
|---|
| 14 | available from https://www.virtualbox.org.
|
|---|
| 15 |
|
|---|
| 16 | This program is free software; you can redistribute it and/or
|
|---|
| 17 | modify it under the terms of the GNU General Public License
|
|---|
| 18 | as published by the Free Software Foundation, in version 3 of the
|
|---|
| 19 | License.
|
|---|
| 20 |
|
|---|
| 21 | This program is distributed in the hope that it will be useful, but
|
|---|
| 22 | WITHOUT ANY WARRANTY; without even the implied warranty of
|
|---|
| 23 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|---|
| 24 | General Public License for more details.
|
|---|
| 25 |
|
|---|
| 26 | You should have received a copy of the GNU General Public License
|
|---|
| 27 | along with this program; if not, see <https://www.gnu.org/licenses>.
|
|---|
| 28 |
|
|---|
| 29 | The contents of this file may alternatively be used under the terms
|
|---|
| 30 | of the Common Development and Distribution License Version 1.0
|
|---|
| 31 | (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
|
|---|
| 32 | in the VirtualBox distribution, in which case the provisions of the
|
|---|
| 33 | CDDL are applicable instead of those of the GPL.
|
|---|
| 34 |
|
|---|
| 35 | You may elect to license modified versions of this file under the
|
|---|
| 36 | terms and conditions of either the GPL or the CDDL or both.
|
|---|
| 37 |
|
|---|
| 38 | SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
|
|---|
| 39 | """
|
|---|
| 40 | __version__ = "$Revision: 96407 $"
|
|---|
| 41 |
|
|---|
| 42 |
|
|---|
| 43 | # Standard Python imports.
|
|---|
| 44 | import os;
|
|---|
| 45 | import random;
|
|---|
| 46 | import string;
|
|---|
| 47 | import sys;
|
|---|
| 48 | import tarfile;
|
|---|
| 49 | import unittest;
|
|---|
| 50 |
|
|---|
| 51 | # Validation Kit imports.
|
|---|
| 52 | from common import utils;
|
|---|
| 53 | from common import pathutils;
|
|---|
| 54 | from testdriver import reporter;
|
|---|
| 55 |
|
|---|
| 56 | # Python 3 hacks:
|
|---|
| 57 | if sys.version_info[0] >= 3:
|
|---|
| 58 | xrange = range; # pylint: disable=redefined-builtin,invalid-name
|
|---|
| 59 |
|
|---|
| 60 |
|
|---|
| 61 |
|
|---|
| 62 | class TestFsObj(object):
|
|---|
| 63 | """ A file system object we created in for test purposes. """
|
|---|
| 64 | def __init__(self, oParent, sPath, sName = None):
|
|---|
| 65 | self.oParent = oParent # type: TestDir
|
|---|
| 66 | self.sPath = sPath # type: str
|
|---|
| 67 | self.sName = sName # type: str
|
|---|
| 68 | if oParent:
|
|---|
| 69 | assert sPath.startswith(oParent.sPath);
|
|---|
| 70 | assert sName is None;
|
|---|
| 71 | self.sName = sPath[len(oParent.sPath) + 1:];
|
|---|
| 72 | # Add to parent.
|
|---|
| 73 | oParent.aoChildren.append(self);
|
|---|
| 74 | oParent.dChildrenUpper[self.sName.upper()] = self;
|
|---|
| 75 |
|
|---|
| 76 | def buildPath(self, sRoot, sSep):
|
|---|
| 77 | """
|
|---|
| 78 | Build the path from sRoot using sSep.
|
|---|
| 79 |
|
|---|
| 80 | This is handy for getting the path to an object in a different context
|
|---|
| 81 | (OS, path) than what it was generated for.
|
|---|
| 82 | """
|
|---|
| 83 | if self.oParent:
|
|---|
| 84 | return self.oParent.buildPath(sRoot, sSep) + sSep + self.sName;
|
|---|
| 85 | return sRoot + sSep + self.sName;
|
|---|
| 86 |
|
|---|
| 87 |
|
|---|
| 88 | class TestFile(TestFsObj):
|
|---|
| 89 | """ A file object in the guest. """
|
|---|
| 90 | def __init__(self, oParent, sPath, abContent):
|
|---|
| 91 | TestFsObj.__init__(self, oParent, sPath);
|
|---|
| 92 | self.abContent = abContent # type: bytearray
|
|---|
| 93 | self.cbContent = len(abContent);
|
|---|
| 94 | self.off = 0;
|
|---|
| 95 |
|
|---|
| 96 | def read(self, cbToRead):
|
|---|
| 97 | """ read() emulation. """
|
|---|
| 98 | assert self.off <= self.cbContent;
|
|---|
| 99 | cbLeft = self.cbContent - self.off;
|
|---|
| 100 | if cbLeft < cbToRead:
|
|---|
| 101 | cbToRead = cbLeft;
|
|---|
| 102 | abRet = self.abContent[self.off:(self.off + cbToRead)];
|
|---|
| 103 | assert len(abRet) == cbToRead;
|
|---|
| 104 | self.off += cbToRead;
|
|---|
| 105 | if sys.version_info[0] < 3:
|
|---|
| 106 | return bytes(abRet);
|
|---|
| 107 | return abRet;
|
|---|
| 108 |
|
|---|
| 109 | def equalFile(self, oFile):
|
|---|
| 110 | """ Compares the content of oFile with self.abContent. """
|
|---|
| 111 |
|
|---|
| 112 | # Check the size first.
|
|---|
| 113 | try:
|
|---|
| 114 | cbFile = os.fstat(oFile.fileno()).st_size;
|
|---|
| 115 | except:
|
|---|
| 116 | return reporter.errorXcpt();
|
|---|
| 117 | if cbFile != self.cbContent:
|
|---|
| 118 | return reporter.error('file size differs: %s, cbContent=%s' % (cbFile, self.cbContent));
|
|---|
| 119 |
|
|---|
| 120 | # Compare the bytes next.
|
|---|
| 121 | offFile = 0;
|
|---|
| 122 | try:
|
|---|
| 123 | oFile.seek(offFile);
|
|---|
| 124 | except:
|
|---|
| 125 | return reporter.error('seek error');
|
|---|
| 126 | while offFile < self.cbContent:
|
|---|
| 127 | cbToRead = self.cbContent - offFile;
|
|---|
| 128 | if cbToRead > 256*1024:
|
|---|
| 129 | cbToRead = 256*1024;
|
|---|
| 130 | try:
|
|---|
| 131 | abRead = oFile.read(cbToRead);
|
|---|
| 132 | except:
|
|---|
| 133 | return reporter.error('read error at offset %s' % (offFile,));
|
|---|
| 134 | cbRead = len(abRead);
|
|---|
| 135 | if cbRead == 0:
|
|---|
| 136 | return reporter.error('premature end of file at offset %s' % (offFile,));
|
|---|
| 137 | if not utils.areBytesEqual(abRead, self.abContent[offFile:(offFile + cbRead)]):
|
|---|
| 138 | return reporter.error('%s byte block at offset %s differs' % (cbRead, offFile,));
|
|---|
| 139 | # Advance:
|
|---|
| 140 | offFile += cbRead;
|
|---|
| 141 |
|
|---|
| 142 | return True;
|
|---|
| 143 |
|
|---|
| 144 | @staticmethod
|
|---|
| 145 | def hexFormatBytes(abBuf):
|
|---|
| 146 | """ Formats a buffer/string/whatever as a string of hex bytes """
|
|---|
| 147 | if sys.version_info[0] >= 3:
|
|---|
| 148 | if utils.isString(abBuf):
|
|---|
| 149 | try: abBuf = bytes(abBuf, 'utf-8');
|
|---|
| 150 | except: pass;
|
|---|
| 151 | else:
|
|---|
| 152 | if utils.isString(abBuf):
|
|---|
| 153 | try: abBuf = bytearray(abBuf, 'utf-8'); # pylint: disable=redefined-variable-type
|
|---|
| 154 | except: pass;
|
|---|
| 155 | sRet = '';
|
|---|
| 156 | off = 0;
|
|---|
| 157 | for off, bByte in enumerate(abBuf):
|
|---|
| 158 | if off > 0:
|
|---|
| 159 | sRet += ' ' if off & 7 else '-';
|
|---|
| 160 | if isinstance(bByte, int):
|
|---|
| 161 | sRet += '%02x' % (bByte,);
|
|---|
| 162 | else:
|
|---|
| 163 | sRet += '%02x' % (ord(bByte),);
|
|---|
| 164 | return sRet;
|
|---|
| 165 |
|
|---|
| 166 | def checkRange(self, cbRange, offFile = 0):
|
|---|
| 167 | """ Check if the specified range is entirely within the file or not. """
|
|---|
| 168 | if offFile >= self.cbContent:
|
|---|
| 169 | return reporter.error('buffer @ %s LB %s is beyond the end of the file (%s bytes)!'
|
|---|
| 170 | % (offFile, cbRange, self.cbContent,));
|
|---|
| 171 | if offFile + cbRange > self.cbContent:
|
|---|
| 172 | return reporter.error('buffer @ %s LB %s is partially beyond the end of the file (%s bytes)!'
|
|---|
| 173 | % (offFile, cbRange, self.cbContent,));
|
|---|
| 174 | return True;
|
|---|
| 175 |
|
|---|
| 176 | def equalMemory(self, abBuf, offFile = 0):
|
|---|
| 177 | """
|
|---|
| 178 | Compares the content of the given buffer with the file content at that
|
|---|
| 179 | file offset.
|
|---|
| 180 |
|
|---|
| 181 | Returns True if it matches, False + error logging if it does not match.
|
|---|
| 182 | """
|
|---|
| 183 | if not abBuf:
|
|---|
| 184 | return True;
|
|---|
| 185 |
|
|---|
| 186 | if not self.checkRange(len(abBuf), offFile):
|
|---|
| 187 | return False;
|
|---|
| 188 |
|
|---|
| 189 | if sys.version_info[0] >= 3:
|
|---|
| 190 | if utils.areBytesEqual(abBuf, self.abContent[offFile:(offFile + len(abBuf))]):
|
|---|
| 191 | return True;
|
|---|
| 192 | else:
|
|---|
| 193 | if utils.areBytesEqual(abBuf, buffer(self.abContent, offFile, len(abBuf))): # pylint: disable=undefined-variable
|
|---|
| 194 | return True;
|
|---|
| 195 |
|
|---|
| 196 | reporter.error('mismatch with buffer @ %s LB %s (cbContent=%s)!' % (offFile, len(abBuf), self.cbContent,));
|
|---|
| 197 | reporter.error(' type(abBuf): %s' % (type(abBuf),));
|
|---|
| 198 | #if isinstance(abBuf, memoryview):
|
|---|
| 199 | # reporter.error(' nbytes=%s len=%s itemsize=%s type(obj)=%s'
|
|---|
| 200 | # % (abBuf.nbytes, len(abBuf), abBuf.itemsize, type(abBuf.obj),));
|
|---|
| 201 | reporter.error('type(abContent): %s' % (type(self.abContent),));
|
|---|
| 202 |
|
|---|
| 203 | offBuf = 0;
|
|---|
| 204 | cbLeft = len(abBuf);
|
|---|
| 205 | while cbLeft > 0:
|
|---|
| 206 | cbLine = min(16, cbLeft);
|
|---|
| 207 | abBuf1 = abBuf[offBuf:(offBuf + cbLine)];
|
|---|
| 208 | abBuf2 = self.abContent[offFile:(offFile + cbLine)];
|
|---|
| 209 | if not utils.areBytesEqual(abBuf1, abBuf2):
|
|---|
| 210 | try: sStr1 = self.hexFormatBytes(abBuf1);
|
|---|
| 211 | except: sStr1 = 'oops';
|
|---|
| 212 | try: sStr2 = self.hexFormatBytes(abBuf2);
|
|---|
| 213 | except: sStr2 = 'oops';
|
|---|
| 214 | reporter.log('%#10x: %s' % (offBuf, sStr1,));
|
|---|
| 215 | reporter.log('%#10x: %s' % (offFile, sStr2,));
|
|---|
| 216 |
|
|---|
| 217 | # Advance.
|
|---|
| 218 | offBuf += 16;
|
|---|
| 219 | offFile += 16;
|
|---|
| 220 | cbLeft -= 16;
|
|---|
| 221 |
|
|---|
| 222 | return False;
|
|---|
| 223 |
|
|---|
| 224 |
|
|---|
| 225 | class TestFileZeroFilled(TestFile):
|
|---|
| 226 | """
|
|---|
| 227 | Zero filled test file.
|
|---|
| 228 | """
|
|---|
| 229 |
|
|---|
| 230 | def __init__(self, oParent, sPath, cbContent):
|
|---|
| 231 | TestFile.__init__(self, oParent, sPath, bytearray(1));
|
|---|
| 232 | self.cbContent = cbContent;
|
|---|
| 233 |
|
|---|
| 234 | def read(self, cbToRead):
|
|---|
| 235 | """ read() emulation. """
|
|---|
| 236 | assert self.off <= self.cbContent;
|
|---|
| 237 | cbLeft = self.cbContent - self.off;
|
|---|
| 238 | if cbLeft < cbToRead:
|
|---|
| 239 | cbToRead = cbLeft;
|
|---|
| 240 | abRet = bytearray(cbToRead);
|
|---|
| 241 | assert len(abRet) == cbToRead;
|
|---|
| 242 | self.off += cbToRead;
|
|---|
| 243 | if sys.version_info[0] < 3:
|
|---|
| 244 | return bytes(abRet);
|
|---|
| 245 | return abRet;
|
|---|
| 246 |
|
|---|
| 247 | def equalFile(self, oFile):
|
|---|
| 248 | _ = oFile;
|
|---|
| 249 | assert False, "not implemented";
|
|---|
| 250 | return False;
|
|---|
| 251 |
|
|---|
| 252 | def equalMemory(self, abBuf, offFile = 0):
|
|---|
| 253 | if not abBuf:
|
|---|
| 254 | return True;
|
|---|
| 255 |
|
|---|
| 256 | if not self.checkRange(len(abBuf), offFile):
|
|---|
| 257 | return False;
|
|---|
| 258 |
|
|---|
| 259 | if utils.areBytesEqual(abBuf, bytearray(len(abBuf))):
|
|---|
| 260 | return True;
|
|---|
| 261 |
|
|---|
| 262 | cErrors = 0;
|
|---|
| 263 | offBuf = 0
|
|---|
| 264 | while offBuf < len(abBuf):
|
|---|
| 265 | bByte = abBuf[offBuf];
|
|---|
| 266 | if not isinstance(bByte, int):
|
|---|
| 267 | bByte = ord(bByte);
|
|---|
| 268 | if bByte != 0:
|
|---|
| 269 | reporter.error('Mismatch @ %s/%s: %#x, expected 0!' % (offFile, offBuf, bByte,));
|
|---|
| 270 | cErrors += 1;
|
|---|
| 271 | if cErrors > 32:
|
|---|
| 272 | return False;
|
|---|
| 273 | offBuf += 1;
|
|---|
| 274 | return cErrors == 0;
|
|---|
| 275 |
|
|---|
| 276 |
|
|---|
| 277 | class TestDir(TestFsObj):
|
|---|
| 278 | """ A file object in the guest. """
|
|---|
| 279 | def __init__(self, oParent, sPath, sName = None):
|
|---|
| 280 | TestFsObj.__init__(self, oParent, sPath, sName);
|
|---|
| 281 | self.aoChildren = [] # type: list(TestFsObj)
|
|---|
| 282 | self.dChildrenUpper = {} # type: dict(str, TestFsObj)
|
|---|
| 283 |
|
|---|
| 284 | def contains(self, sName):
|
|---|
| 285 | """ Checks if the directory contains the given name. """
|
|---|
| 286 | return sName.upper() in self.dChildrenUpper
|
|---|
| 287 |
|
|---|
| 288 |
|
|---|
| 289 | class TestFileSet(object):
|
|---|
| 290 | """
|
|---|
| 291 | A generated set of files and directories for use in a test.
|
|---|
| 292 |
|
|---|
| 293 | Can be wrapped up into a tarball or written directly to the file system.
|
|---|
| 294 | """
|
|---|
| 295 |
|
|---|
| 296 | ksReservedWinOS2 = '/\\"*:<>?|\t\v\n\r\f\a\b';
|
|---|
| 297 | ksReservedUnix = '/';
|
|---|
| 298 | ksReservedTrailingWinOS2 = ' .';
|
|---|
| 299 | ksReservedTrailingUnix = '';
|
|---|
| 300 |
|
|---|
| 301 | ## @name Path style.
|
|---|
| 302 | ## @{
|
|---|
| 303 |
|
|---|
| 304 | ## @}
|
|---|
| 305 |
|
|---|
| 306 | def __init__(self, fDosStyle, sBasePath, sSubDir, # pylint: disable=too-many-arguments
|
|---|
| 307 | asCompatibleWith = None, # List of getHostOs values to the names must be compatible with.
|
|---|
| 308 | oRngFileSizes = xrange(0, 16384),
|
|---|
| 309 | oRngManyFiles = xrange(128, 512),
|
|---|
| 310 | oRngTreeFiles = xrange(128, 384),
|
|---|
| 311 | oRngTreeDepth = xrange(92, 256),
|
|---|
| 312 | oRngTreeDirs = xrange(2, 16),
|
|---|
| 313 | cchMaxPath = 230,
|
|---|
| 314 | cchMaxName = 230,
|
|---|
| 315 | uSeed = None):
|
|---|
| 316 | ## @name Parameters
|
|---|
| 317 | ## @{
|
|---|
| 318 | self.fDosStyle = fDosStyle;
|
|---|
| 319 | self.sMinStyle = 'win' if fDosStyle else 'linux';
|
|---|
| 320 | if asCompatibleWith is not None:
|
|---|
| 321 | for sOs in asCompatibleWith:
|
|---|
| 322 | assert sOs in ('win', 'os2', 'darwin', 'linux', 'solaris',), sOs;
|
|---|
| 323 | if 'os2' in asCompatibleWith:
|
|---|
| 324 | self.sMinStyle = 'os2';
|
|---|
| 325 | elif 'win' in asCompatibleWith:
|
|---|
| 326 | self.sMinStyle = 'win';
|
|---|
| 327 | self.sBasePath = sBasePath;
|
|---|
| 328 | self.sSubDir = sSubDir;
|
|---|
| 329 | self.oRngFileSizes = oRngFileSizes;
|
|---|
| 330 | self.oRngManyFiles = oRngManyFiles;
|
|---|
| 331 | self.oRngTreeFiles = oRngTreeFiles;
|
|---|
| 332 | self.oRngTreeDepth = oRngTreeDepth;
|
|---|
| 333 | self.oRngTreeDirs = oRngTreeDirs;
|
|---|
| 334 | self.cchMaxPath = cchMaxPath;
|
|---|
| 335 | self.cchMaxName = cchMaxName
|
|---|
| 336 | ## @}
|
|---|
| 337 |
|
|---|
| 338 | ## @name Charset stuff
|
|---|
| 339 | ## @todo allow more chars for unix hosts + guests.
|
|---|
| 340 | ## @todo include unicode stuff, except on OS/2 and DOS.
|
|---|
| 341 | ## @{
|
|---|
| 342 | ## The filename charset.
|
|---|
| 343 | self.sFileCharset = string.printable;
|
|---|
| 344 | ## Set of characters that should not trail a guest filename.
|
|---|
| 345 | self.sReservedTrailing = self.ksReservedTrailingWinOS2;
|
|---|
| 346 | if self.sMinStyle in ('win', 'os2'):
|
|---|
| 347 | for ch in self.ksReservedWinOS2:
|
|---|
| 348 | self.sFileCharset = self.sFileCharset.replace(ch, '');
|
|---|
| 349 | else:
|
|---|
| 350 | self.sReservedTrailing = self.ksReservedTrailingUnix;
|
|---|
| 351 | for ch in self.ksReservedUnix:
|
|---|
| 352 | self.sFileCharset = self.sFileCharset.replace(ch, '');
|
|---|
| 353 | # More spaces and dot:
|
|---|
| 354 | self.sFileCharset += ' ...';
|
|---|
| 355 | ## @}
|
|---|
| 356 |
|
|---|
| 357 | ## The root directory.
|
|---|
| 358 | self.oRoot = None # type: TestDir;
|
|---|
| 359 | ## An empty directory (under root).
|
|---|
| 360 | self.oEmptyDir = None # type: TestDir;
|
|---|
| 361 |
|
|---|
| 362 | ## A directory with a lot of files in it.
|
|---|
| 363 | self.oManyDir = None # type: TestDir;
|
|---|
| 364 |
|
|---|
| 365 | ## A directory with a mixed tree structure under it.
|
|---|
| 366 | self.oTreeDir = None # type: TestDir;
|
|---|
| 367 | ## Number of files in oTreeDir.
|
|---|
| 368 | self.cTreeFiles = 0;
|
|---|
| 369 | ## Number of directories under oTreeDir.
|
|---|
| 370 | self.cTreeDirs = 0;
|
|---|
| 371 | ## Number of other file types under oTreeDir.
|
|---|
| 372 | self.cTreeOthers = 0;
|
|---|
| 373 |
|
|---|
| 374 | ## All directories in creation order.
|
|---|
| 375 | self.aoDirs = [] # type: list(TestDir);
|
|---|
| 376 | ## All files in creation order.
|
|---|
| 377 | self.aoFiles = [] # type: list(TestFile);
|
|---|
| 378 | ## Path to object lookup.
|
|---|
| 379 | self.dPaths = {} # type: dict(str, TestFsObj);
|
|---|
| 380 |
|
|---|
| 381 | #
|
|---|
| 382 | # Do the creating.
|
|---|
| 383 | #
|
|---|
| 384 | self.uSeed = uSeed if uSeed is not None else utils.timestampMilli();
|
|---|
| 385 | self.oRandom = random.Random();
|
|---|
| 386 | self.oRandom.seed(self.uSeed);
|
|---|
| 387 | reporter.log('prepareGuestForTesting: random seed %s' % (self.uSeed,));
|
|---|
| 388 |
|
|---|
| 389 | self.__createTestStuff();
|
|---|
| 390 |
|
|---|
| 391 | def __createFilename(self, oParent, sCharset, sReservedTrailing):
|
|---|
| 392 | """
|
|---|
| 393 | Creates a filename contains random characters from sCharset and together
|
|---|
| 394 | with oParent.sPath doesn't exceed the given max chars in length.
|
|---|
| 395 | """
|
|---|
| 396 | ## @todo Consider extending this to take UTF-8 and UTF-16 encoding so we
|
|---|
| 397 | ## can safely use the full unicode range. Need to check how
|
|---|
| 398 | ## RTZipTarCmd handles file name encoding in general...
|
|---|
| 399 |
|
|---|
| 400 | if oParent:
|
|---|
| 401 | cchMaxName = self.cchMaxPath - len(oParent.sPath) - 1;
|
|---|
| 402 | else:
|
|---|
| 403 | cchMaxName = self.cchMaxPath - 4;
|
|---|
| 404 | if cchMaxName > self.cchMaxName:
|
|---|
| 405 | cchMaxName = self.cchMaxName;
|
|---|
| 406 | if cchMaxName <= 1:
|
|---|
| 407 | cchMaxName = 2;
|
|---|
| 408 |
|
|---|
| 409 | while True:
|
|---|
| 410 | cchName = self.oRandom.randrange(1, cchMaxName);
|
|---|
| 411 | sName = ''.join(self.oRandom.choice(sCharset) for _ in xrange(cchName));
|
|---|
| 412 | if oParent is None or not oParent.contains(sName):
|
|---|
| 413 | if sName[-1] not in sReservedTrailing:
|
|---|
| 414 | if sName not in ('.', '..',):
|
|---|
| 415 | return sName;
|
|---|
| 416 | return ''; # never reached, but makes pylint happy.
|
|---|
| 417 |
|
|---|
| 418 | def generateFilenameEx(self, cchMax = -1, cchMin = -1):
|
|---|
| 419 | """
|
|---|
| 420 | Generates a filename according to the given specs.
|
|---|
| 421 |
|
|---|
| 422 | This is for external use, whereas __createFilename is for internal.
|
|---|
| 423 |
|
|---|
| 424 | Returns generated filename.
|
|---|
| 425 | """
|
|---|
| 426 | assert cchMax == -1 or (cchMax >= 1 and cchMax > cchMin);
|
|---|
| 427 | if cchMin <= 0:
|
|---|
| 428 | cchMin = 1;
|
|---|
| 429 | if cchMax < cchMin:
|
|---|
| 430 | cchMax = self.cchMaxName;
|
|---|
| 431 |
|
|---|
| 432 | while True:
|
|---|
| 433 | cchName = self.oRandom.randrange(cchMin, cchMax + 1);
|
|---|
| 434 | sName = ''.join(self.oRandom.choice(self.sFileCharset) for _ in xrange(cchName));
|
|---|
| 435 | if sName[-1] not in self.sReservedTrailing:
|
|---|
| 436 | if sName not in ('.', '..',):
|
|---|
| 437 | return sName;
|
|---|
| 438 | return ''; # never reached, but makes pylint happy.
|
|---|
| 439 |
|
|---|
| 440 | def __createTestDir(self, oParent, sDir, sName = None):
|
|---|
| 441 | """
|
|---|
| 442 | Creates a test directory.
|
|---|
| 443 | """
|
|---|
| 444 | oDir = TestDir(oParent, sDir, sName);
|
|---|
| 445 | self.aoDirs.append(oDir);
|
|---|
| 446 | self.dPaths[sDir] = oDir;
|
|---|
| 447 | return oDir;
|
|---|
| 448 |
|
|---|
| 449 | def __createTestFile(self, oParent, sFile):
|
|---|
| 450 | """
|
|---|
| 451 | Creates a test file with random size up to cbMaxContent and random content.
|
|---|
| 452 | """
|
|---|
| 453 | cbFile = self.oRandom.choice(self.oRngFileSizes);
|
|---|
| 454 | abContent = bytearray(self.oRandom.getrandbits(8) for _ in xrange(cbFile));
|
|---|
| 455 |
|
|---|
| 456 | oFile = TestFile(oParent, sFile, abContent);
|
|---|
| 457 | self.aoFiles.append(oFile);
|
|---|
| 458 | self.dPaths[sFile] = oFile;
|
|---|
| 459 | return oFile;
|
|---|
| 460 |
|
|---|
| 461 | def __createTestStuff(self):
|
|---|
| 462 | """
|
|---|
| 463 | Create a random file set that we can work on in the tests.
|
|---|
| 464 | Returns True/False.
|
|---|
| 465 | """
|
|---|
| 466 |
|
|---|
| 467 | #
|
|---|
| 468 | # Create the root test dir.
|
|---|
| 469 | #
|
|---|
| 470 | sRoot = pathutils.joinEx(self.fDosStyle, self.sBasePath, self.sSubDir);
|
|---|
| 471 | self.oRoot = self.__createTestDir(None, sRoot, self.sSubDir);
|
|---|
| 472 | self.oEmptyDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'empty'));
|
|---|
| 473 |
|
|---|
| 474 | #
|
|---|
| 475 | # Create a directory with lots of files in it:
|
|---|
| 476 | #
|
|---|
| 477 | oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'many'));
|
|---|
| 478 | self.oManyDir = oDir;
|
|---|
| 479 | cManyFiles = self.oRandom.choice(self.oRngManyFiles);
|
|---|
| 480 | for _ in xrange(cManyFiles):
|
|---|
| 481 | sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
|
|---|
| 482 | self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
|
|---|
| 483 |
|
|---|
| 484 | #
|
|---|
| 485 | # Generate a tree of files and dirs.
|
|---|
| 486 | #
|
|---|
| 487 | oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'tree'));
|
|---|
| 488 | uMaxDepth = self.oRandom.choice(self.oRngTreeDepth);
|
|---|
| 489 | cMaxFiles = self.oRandom.choice(self.oRngTreeFiles);
|
|---|
| 490 | cMaxDirs = self.oRandom.choice(self.oRngTreeDirs);
|
|---|
| 491 | self.oTreeDir = oDir;
|
|---|
| 492 | self.cTreeFiles = 0;
|
|---|
| 493 | self.cTreeDirs = 0;
|
|---|
| 494 | uDepth = 0;
|
|---|
| 495 | while self.cTreeFiles < cMaxFiles and self.cTreeDirs < cMaxDirs:
|
|---|
| 496 | iAction = self.oRandom.randrange(0, 2+1);
|
|---|
| 497 | # 0: Add a file:
|
|---|
| 498 | if iAction == 0 and self.cTreeFiles < cMaxFiles and len(oDir.sPath) < 230 - 2:
|
|---|
| 499 | sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
|
|---|
| 500 | self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
|
|---|
| 501 | self.cTreeFiles += 1;
|
|---|
| 502 | # 1: Add a subdirector and descend into it:
|
|---|
| 503 | elif iAction == 1 and self.cTreeDirs < cMaxDirs and uDepth < uMaxDepth and len(oDir.sPath) < 220:
|
|---|
| 504 | sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
|
|---|
| 505 | oDir = self.__createTestDir(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
|
|---|
| 506 | self.cTreeDirs += 1;
|
|---|
| 507 | uDepth += 1;
|
|---|
| 508 | # 2: Ascend to parent dir:
|
|---|
| 509 | elif iAction == 2 and uDepth > 0:
|
|---|
| 510 | oDir = oDir.oParent;
|
|---|
| 511 | uDepth -= 1;
|
|---|
| 512 |
|
|---|
| 513 | return True;
|
|---|
| 514 |
|
|---|
| 515 | def createTarball(self, sTarFileHst):
|
|---|
| 516 | """
|
|---|
| 517 | Creates a tarball on the host.
|
|---|
| 518 | Returns success indicator.
|
|---|
| 519 | """
|
|---|
| 520 | reporter.log('Creating tarball "%s" with test files for the guest...' % (sTarFileHst,));
|
|---|
| 521 |
|
|---|
| 522 | cchSkip = len(self.sBasePath) + 1;
|
|---|
| 523 |
|
|---|
| 524 | # Open the tarball:
|
|---|
| 525 | try:
|
|---|
| 526 | # Make sure to explicitly set GNU_FORMAT here, as with Python 3.8 the default format (tarfile.DEFAULT_FORMAT)
|
|---|
| 527 | # has been changed to tarfile.PAX_FORMAT, which our extraction code (vts_tar) currently can't handle.
|
|---|
| 528 | ## @todo Remove tarfile.GNU_FORMAT and use tarfile.PAX_FORMAT as soon as we have PAX support.
|
|---|
| 529 | oTarFile = tarfile.open(sTarFileHst, 'w:gz', format = tarfile.GNU_FORMAT); # pylint: disable=consider-using-with
|
|---|
| 530 | except:
|
|---|
| 531 | return reporter.errorXcpt('Failed to open new tar file: %s' % (sTarFileHst,));
|
|---|
| 532 |
|
|---|
| 533 | # Directories:
|
|---|
| 534 | for oDir in self.aoDirs:
|
|---|
| 535 | sPath = oDir.sPath[cchSkip:];
|
|---|
| 536 | if self.fDosStyle:
|
|---|
| 537 | sPath = sPath.replace('\\', '/');
|
|---|
| 538 | oTarInfo = tarfile.TarInfo(sPath + '/');
|
|---|
| 539 | oTarInfo.mode = 0o777;
|
|---|
| 540 | oTarInfo.type = tarfile.DIRTYPE;
|
|---|
| 541 | try:
|
|---|
| 542 | oTarFile.addfile(oTarInfo);
|
|---|
| 543 | except:
|
|---|
| 544 | return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oDir.sPath,));
|
|---|
| 545 |
|
|---|
| 546 | # Files:
|
|---|
| 547 | for oFile in self.aoFiles:
|
|---|
| 548 | sPath = oFile.sPath[cchSkip:];
|
|---|
| 549 | if self.fDosStyle:
|
|---|
| 550 | sPath = sPath.replace('\\', '/');
|
|---|
| 551 | oTarInfo = tarfile.TarInfo(sPath);
|
|---|
| 552 | oTarInfo.mode = 0o666;
|
|---|
| 553 | oTarInfo.size = len(oFile.abContent);
|
|---|
| 554 | oFile.off = 0;
|
|---|
| 555 | try:
|
|---|
| 556 | oTarFile.addfile(oTarInfo, oFile);
|
|---|
| 557 | except:
|
|---|
| 558 | return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oFile.sPath,));
|
|---|
| 559 |
|
|---|
| 560 | # Complete the tarball.
|
|---|
| 561 | try:
|
|---|
| 562 | oTarFile.close();
|
|---|
| 563 | except:
|
|---|
| 564 | return reporter.errorXcpt('Error closing new tar file: %s' % (sTarFileHst,));
|
|---|
| 565 | return True;
|
|---|
| 566 |
|
|---|
| 567 | def writeToDisk(self, sAltBase = None):
|
|---|
| 568 | """
|
|---|
| 569 | Writes out the files to disk.
|
|---|
| 570 | Returns True on success, False + error logging on failure.
|
|---|
| 571 | """
|
|---|
| 572 |
|
|---|
| 573 | # We only need to flip DOS slashes to unix ones, since windows & OS/2 can handle unix slashes.
|
|---|
| 574 | fDosToUnix = self.fDosStyle and os.path.sep != '\\';
|
|---|
| 575 |
|
|---|
| 576 | # The directories:
|
|---|
| 577 | for oDir in self.aoDirs:
|
|---|
| 578 | sPath = oDir.sPath;
|
|---|
| 579 | if sAltBase:
|
|---|
| 580 | if fDosToUnix:
|
|---|
| 581 | sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep);
|
|---|
| 582 | else:
|
|---|
| 583 | sPath = sAltBase + sPath[len(self.sBasePath):];
|
|---|
| 584 | elif fDosToUnix:
|
|---|
| 585 | sPath = sPath.replace('\\', os.path.sep);
|
|---|
| 586 |
|
|---|
| 587 | try:
|
|---|
| 588 | os.mkdir(sPath, 0o770);
|
|---|
| 589 | except:
|
|---|
| 590 | return reporter.errorXcpt('mkdir(%s) failed' % (sPath,));
|
|---|
| 591 |
|
|---|
| 592 | # The files:
|
|---|
| 593 | for oFile in self.aoFiles:
|
|---|
| 594 | sPath = oFile.sPath;
|
|---|
| 595 | if sAltBase:
|
|---|
| 596 | if fDosToUnix:
|
|---|
| 597 | sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep);
|
|---|
| 598 | else:
|
|---|
| 599 | sPath = sAltBase + sPath[len(self.sBasePath):];
|
|---|
| 600 | elif fDosToUnix:
|
|---|
| 601 | sPath = sPath.replace('\\', os.path.sep);
|
|---|
| 602 |
|
|---|
| 603 | try:
|
|---|
| 604 | oOutFile = open(sPath, 'wb'); # pylint: disable=consider-using-with
|
|---|
| 605 | except:
|
|---|
| 606 | return reporter.errorXcpt('open(%s, "wb") failed' % (sPath,));
|
|---|
| 607 | try:
|
|---|
| 608 | if sys.version_info[0] < 3:
|
|---|
| 609 | oOutFile.write(bytes(oFile.abContent));
|
|---|
| 610 | else:
|
|---|
| 611 | oOutFile.write(oFile.abContent);
|
|---|
| 612 | except:
|
|---|
| 613 | try: oOutFile.close();
|
|---|
| 614 | except: pass;
|
|---|
| 615 | return reporter.errorXcpt('%s: write(%s bytes) failed' % (sPath, oFile.cbContent,));
|
|---|
| 616 | try:
|
|---|
| 617 | oOutFile.close();
|
|---|
| 618 | except:
|
|---|
| 619 | return reporter.errorXcpt('%s: close() failed' % (sPath,));
|
|---|
| 620 |
|
|---|
| 621 | return True;
|
|---|
| 622 |
|
|---|
| 623 |
|
|---|
| 624 | def chooseRandomFile(self):
|
|---|
| 625 | """
|
|---|
| 626 | Returns a random file.
|
|---|
| 627 | """
|
|---|
| 628 | return self.aoFiles[self.oRandom.choice(xrange(len(self.aoFiles)))];
|
|---|
| 629 |
|
|---|
| 630 | def chooseRandomDirFromTree(self, fLeaf = False, fNonEmpty = False, cMaxRetries = 1024):
|
|---|
| 631 | """
|
|---|
| 632 | Returns a random directory from the tree (self.oTreeDir).
|
|---|
| 633 | Will return None if no directory with given parameters was found.
|
|---|
| 634 | """
|
|---|
| 635 | cRetries = 0;
|
|---|
| 636 | while cRetries < cMaxRetries:
|
|---|
| 637 | oDir = self.aoDirs[self.oRandom.choice(xrange(len(self.aoDirs)))];
|
|---|
| 638 | # Check fNonEmpty requirement:
|
|---|
| 639 | if not fNonEmpty or oDir.aoChildren:
|
|---|
| 640 | # Check leaf requirement:
|
|---|
| 641 | if not fLeaf:
|
|---|
| 642 | for oChild in oDir.aoChildren:
|
|---|
| 643 | if isinstance(oChild, TestDir):
|
|---|
| 644 | continue; # skip it.
|
|---|
| 645 |
|
|---|
| 646 | # Return if in the tree:
|
|---|
| 647 | oParent = oDir.oParent;
|
|---|
| 648 | while oParent is not None:
|
|---|
| 649 | if oParent is self.oTreeDir:
|
|---|
| 650 | return oDir;
|
|---|
| 651 | oParent = oParent.oParent;
|
|---|
| 652 | cRetries += 1;
|
|---|
| 653 |
|
|---|
| 654 | return None; # make pylint happy
|
|---|
| 655 |
|
|---|
| 656 | #
|
|---|
| 657 | # Unit testing.
|
|---|
| 658 | #
|
|---|
| 659 |
|
|---|
| 660 | # pylint: disable=missing-docstring
|
|---|
| 661 | # pylint: disable=undefined-variable
|
|---|
| 662 | class TestFileSetUnitTests(unittest.TestCase):
|
|---|
| 663 | def testGeneral(self):
|
|---|
| 664 | oSet = TestFileSet(False, '/tmp', 'unittest');
|
|---|
| 665 | self.assertTrue(isinstance(oSet.chooseRandomDirFromTree(), TestDir));
|
|---|
| 666 | self.assertTrue(isinstance(oSet.chooseRandomFile(), TestFile));
|
|---|
| 667 |
|
|---|
| 668 | def testHexFormatBytes(self):
|
|---|
| 669 | self.assertEqual(TestFile.hexFormatBytes(bytearray([0,1,2,3,4,5,6,7,8,9])),
|
|---|
| 670 | '00 01 02 03 04 05 06 07-08 09');
|
|---|
| 671 | self.assertEqual(TestFile.hexFormatBytes(memoryview(bytearray([0,1,2,3,4,5,6,7,8,9,10, 16]))),
|
|---|
| 672 | '00 01 02 03 04 05 06 07-08 09 0a 10');
|
|---|
| 673 |
|
|---|
| 674 |
|
|---|
| 675 | if __name__ == '__main__':
|
|---|
| 676 | unittest.main();
|
|---|
| 677 | # not reached.
|
|---|
| 678 |
|
|---|
