[52776] | 1 | # -*- coding: utf-8 -*-
|
---|
| 2 | # $Id: reader.py 98103 2023-01-17 14:15:46Z vboxsync $
|
---|
| 3 |
|
---|
| 4 | """
|
---|
| 5 | XML reader module.
|
---|
| 6 |
|
---|
| 7 | This produces a test result tree that can be processed and passed to
|
---|
| 8 | reporting.
|
---|
| 9 | """
|
---|
| 10 |
|
---|
| 11 | __copyright__ = \
|
---|
| 12 | """
|
---|
[98103] | 13 | Copyright (C) 2010-2023 Oracle and/or its affiliates.
|
---|
[52776] | 14 |
|
---|
[96407] | 15 | This file is part of VirtualBox base platform packages, as
|
---|
| 16 | available from https://www.virtualbox.org.
|
---|
[52776] | 17 |
|
---|
[96407] | 18 | This program is free software; you can redistribute it and/or
|
---|
| 19 | modify it under the terms of the GNU General Public License
|
---|
| 20 | as published by the Free Software Foundation, in version 3 of the
|
---|
| 21 | License.
|
---|
| 22 |
|
---|
| 23 | This program is distributed in the hope that it will be useful, but
|
---|
| 24 | WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 25 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
---|
| 26 | General Public License for more details.
|
---|
| 27 |
|
---|
| 28 | You should have received a copy of the GNU General Public License
|
---|
| 29 | along with this program; if not, see <https://www.gnu.org/licenses>.
|
---|
| 30 |
|
---|
[52776] | 31 | The contents of this file may alternatively be used under the terms
|
---|
| 32 | of the Common Development and Distribution License Version 1.0
|
---|
[96407] | 33 | (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
|
---|
| 34 | in the VirtualBox distribution, in which case the provisions of the
|
---|
[52776] | 35 | CDDL are applicable instead of those of the GPL.
|
---|
| 36 |
|
---|
| 37 | You may elect to license modified versions of this file under the
|
---|
| 38 | terms and conditions of either the GPL or the CDDL or both.
|
---|
[96407] | 39 |
|
---|
| 40 | SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
|
---|
[52776] | 41 | """
|
---|
| 42 | __version__ = "$Revision: 98103 $"
|
---|
[79092] | 43 | __all__ = [ 'parseTestResult', ]
|
---|
[52776] | 44 |
|
---|
| 45 | # Standard python imports.
|
---|
[97266] | 46 | import datetime;
|
---|
| 47 | import os;
|
---|
[97267] | 48 | import re;
|
---|
[97266] | 49 | import sys;
|
---|
| 50 | import traceback;
|
---|
[52776] | 51 |
|
---|
[97266] | 52 | # Only the main script needs to modify the path.
|
---|
| 53 | try: __file__;
|
---|
| 54 | except: __file__ = sys.argv[0];
|
---|
| 55 | g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)));
|
---|
| 56 | sys.path.append(g_ksValidationKitDir);
|
---|
| 57 |
|
---|
| 58 | # ValidationKit imports.
|
---|
| 59 | from common import utils;
|
---|
| 60 |
|
---|
| 61 | # Python 3 hacks:
|
---|
| 62 | if sys.version_info[0] >= 3:
|
---|
| 63 | long = int; # pylint: disable=redefined-builtin,invalid-name
|
---|
| 64 |
|
---|
[79087] | 65 | # pylint: disable=missing-docstring
|
---|
[52776] | 66 |
|
---|
[97266] | 67 |
|
---|
[52776] | 68 | class Value(object):
|
---|
| 69 | """
|
---|
| 70 | Represents a value. Usually this is benchmark result or parameter.
|
---|
| 71 | """
|
---|
[97266] | 72 |
|
---|
| 73 | kdBestByUnit = {
|
---|
| 74 | "%": +1, # Difficult to say what's best really.
|
---|
| 75 | "bytes": +1, # Difficult to say what's best really.
|
---|
| 76 | "bytes/s": +2,
|
---|
| 77 | "KB": +1,
|
---|
| 78 | "KB/s": +2,
|
---|
| 79 | "MB": +1,
|
---|
| 80 | "MB/s": +2,
|
---|
| 81 | "packets": +2,
|
---|
| 82 | "packets/s": +2,
|
---|
| 83 | "frames": +2,
|
---|
| 84 | "frames/s": +2,
|
---|
| 85 | "occurrences": +1, # Difficult to say what's best really.
|
---|
| 86 | "occurrences/s": +2,
|
---|
| 87 | "roundtrips": +2,
|
---|
| 88 | "calls": +1, # Difficult to say what's best really.
|
---|
| 89 | "calls/s": +2,
|
---|
| 90 | "s": -2,
|
---|
| 91 | "ms": -2,
|
---|
| 92 | "ns": -2,
|
---|
| 93 | "ns/call": -2,
|
---|
| 94 | "ns/frame": -2,
|
---|
| 95 | "ns/occurrence": -2,
|
---|
| 96 | "ns/packet": -2,
|
---|
| 97 | "ns/roundtrip": -2,
|
---|
| 98 | "ins": +2,
|
---|
[97279] | 99 | "ins/sec": +2,
|
---|
[97266] | 100 | "": +1, # Difficult to say what's best really.
|
---|
| 101 | "pp1k": -2,
|
---|
| 102 | "pp10k": -2,
|
---|
| 103 | "ppm": -2,
|
---|
| 104 | "ppb": -2,
|
---|
| 105 | "ticks": -1, # Difficult to say what's best really.
|
---|
| 106 | "ticks/call": -2,
|
---|
| 107 | "ticks/occ": -2,
|
---|
| 108 | "pages": +1, # Difficult to say what's best really.
|
---|
| 109 | "pages/s": +2,
|
---|
| 110 | "ticks/page": -2,
|
---|
| 111 | "ns/page": -2,
|
---|
| 112 | "ps": -1, # Difficult to say what's best really.
|
---|
| 113 | "ps/call": -2,
|
---|
| 114 | "ps/frame": -2,
|
---|
| 115 | "ps/occurrence": -2,
|
---|
| 116 | "ps/packet": -2,
|
---|
| 117 | "ps/roundtrip": -2,
|
---|
| 118 | "ps/page": -2,
|
---|
| 119 | };
|
---|
| 120 |
|
---|
| 121 | def __init__(self, oTest, sName = None, sUnit = None, sTimestamp = None, lValue = None):
|
---|
[52776] | 122 | self.oTest = oTest;
|
---|
[97266] | 123 | self.sName = sName;
|
---|
| 124 | self.sUnit = sUnit;
|
---|
| 125 | self.sTimestamp = sTimestamp;
|
---|
| 126 | self.lValue = self.valueToInteger(lValue);
|
---|
| 127 | assert self.lValue is None or isinstance(self.lValue, (int, long)), "lValue=%s %s" % (self.lValue, type(self.lValue),);
|
---|
| 128 |
|
---|
| 129 | def clone(self, oParentTest):
|
---|
| 130 | """
|
---|
| 131 | Clones the value.
|
---|
| 132 | """
|
---|
| 133 | return Value(oParentTest, self.sName, self.sUnit, self.sTimestamp, self.lValue);
|
---|
[52776] | 134 |
|
---|
[97267] | 135 | def matchFilters(self, sPrefix, aoFilters):
|
---|
| 136 | """
|
---|
| 137 | Checks for any substring match between aoFilters (str or re.Pattern)
|
---|
| 138 | and the value name prefixed by sPrefix.
|
---|
[52776] | 139 |
|
---|
[97267] | 140 | Returns True if any of the filters matches.
|
---|
| 141 | Returns False if none of the filters matches.
|
---|
| 142 | """
|
---|
| 143 | sFullName = sPrefix + self.sName;
|
---|
| 144 | for oFilter in aoFilters:
|
---|
| 145 | if oFilter.search(sFullName) is not None if isinstance(oFilter, re.Pattern) else sFullName.find(oFilter) >= 0:
|
---|
| 146 | return True;
|
---|
| 147 | return False;
|
---|
| 148 |
|
---|
[97279] | 149 | def canDoBetterCompare(self):
|
---|
| 150 | """
|
---|
| 151 | Checks whether we can do a confident better-than comparsion of the value.
|
---|
| 152 | """
|
---|
| 153 | return self.sUnit is not None and self.kdBestByUnit[self.sUnit] not in (-1, 0, 1);
|
---|
[97267] | 154 |
|
---|
[97279] | 155 | def getBetterRelation(self):
|
---|
| 156 | """
|
---|
| 157 | Returns +2 if larger values are definintely better.
|
---|
| 158 | Returns +1 if larger values are likely to be better.
|
---|
| 159 | Returns 0 if we have no clue.
|
---|
| 160 | Returns -1 if smaller values are likey to better.
|
---|
| 161 | Returns -2 if smaller values are definitely better.
|
---|
| 162 | """
|
---|
| 163 | if self.sUnit is None:
|
---|
| 164 | return 0;
|
---|
| 165 | return self.kdBestByUnit[self.sUnit];
|
---|
| 166 |
|
---|
[97266] | 167 | @staticmethod
|
---|
| 168 | def valueToInteger(sValue):
|
---|
| 169 | """
|
---|
| 170 | Returns integer (long) represention of lValue.
|
---|
| 171 | Returns None if it cannot be converted to integer.
|
---|
| 172 |
|
---|
| 173 | Raises an exception if sValue isn't an integer.
|
---|
| 174 | """
|
---|
| 175 | if sValue is None or isinstance(sValue, (int, long)):
|
---|
| 176 | return sValue;
|
---|
| 177 | sValue = sValue.strip();
|
---|
| 178 | if not sValue:
|
---|
| 179 | return None;
|
---|
| 180 | return long(sValue);
|
---|
| 181 |
|
---|
| 182 | # Manipluation
|
---|
| 183 |
|
---|
| 184 | def distill(self, aoValues, sMethod):
|
---|
| 185 | """
|
---|
| 186 | Distills the value of the object from values from multiple test runs.
|
---|
| 187 | """
|
---|
| 188 | if not aoValues:
|
---|
| 189 | return self;
|
---|
| 190 |
|
---|
| 191 | # Everything except the value comes from the first run.
|
---|
| 192 | self.sName = aoValues[0].sName;
|
---|
| 193 | self.sTimestamp = aoValues[0].sTimestamp;
|
---|
| 194 | self.sUnit = aoValues[0].sUnit;
|
---|
| 195 |
|
---|
| 196 | # Find the value to use according to sMethod.
|
---|
| 197 | if len(aoValues) == 1:
|
---|
| 198 | self.lValue = aoValues[0].lValue;
|
---|
| 199 | else:
|
---|
| 200 | alValuesXcptInvalid = [oValue.lValue for oValue in aoValues if oValue.lValue is not None];
|
---|
| 201 | if not alValuesXcptInvalid:
|
---|
| 202 | # No integer result, so just pick the first value whatever it is.
|
---|
| 203 | self.lValue = aoValues[0].lValue;
|
---|
| 204 |
|
---|
| 205 | elif sMethod == 'best':
|
---|
| 206 | # Pick the best result out of the whole bunch.
|
---|
| 207 | if self.kdBestByUnit[self.sUnit] >= 0:
|
---|
| 208 | self.lValue = max(alValuesXcptInvalid);
|
---|
| 209 | else:
|
---|
| 210 | self.lValue = min(alValuesXcptInvalid);
|
---|
| 211 |
|
---|
| 212 | elif sMethod == 'avg':
|
---|
| 213 | # Calculate the average.
|
---|
| 214 | self.lValue = (sum(alValuesXcptInvalid) + len(alValuesXcptInvalid) // 2) // len(alValuesXcptInvalid);
|
---|
| 215 |
|
---|
| 216 | else:
|
---|
| 217 | assert False;
|
---|
| 218 | self.lValue = aoValues[0].lValue;
|
---|
| 219 |
|
---|
| 220 | return self;
|
---|
| 221 |
|
---|
| 222 |
|
---|
[52776] | 223 | # debug
|
---|
| 224 |
|
---|
| 225 | def printValue(self, cIndent):
|
---|
[97266] | 226 | print('%sValue: name=%s timestamp=%s unit=%s value=%s'
|
---|
| 227 | % (''.ljust(cIndent*2), self.sName, self.sTimestamp, self.sUnit, self.lValue));
|
---|
[52776] | 228 |
|
---|
| 229 |
|
---|
| 230 | class Test(object):
|
---|
| 231 | """
|
---|
| 232 | Nested test result.
|
---|
| 233 | """
|
---|
[97266] | 234 | def __init__(self, oParent = None, hsAttrs = None):
|
---|
| 235 | self.aoChildren = [] # type: list(Test)
|
---|
[52776] | 236 | self.aoValues = [];
|
---|
| 237 | self.oParent = oParent;
|
---|
[97266] | 238 | self.sName = hsAttrs['name'] if hsAttrs else None;
|
---|
| 239 | self.sStartTS = hsAttrs['timestamp'] if hsAttrs else None;
|
---|
[52776] | 240 | self.sEndTS = None;
|
---|
| 241 | self.sStatus = None;
|
---|
| 242 | self.cErrors = -1;
|
---|
[97266] | 243 |
|
---|
| 244 | def clone(self, oParent = None):
|
---|
| 245 | """
|
---|
| 246 | Returns a deep copy.
|
---|
| 247 | """
|
---|
| 248 | oClone = Test(oParent, {'name': self.sName, 'timestamp': self.sStartTS});
|
---|
| 249 |
|
---|
| 250 | for oChild in self.aoChildren:
|
---|
| 251 | oClone.aoChildren.append(oChild.clone(oClone));
|
---|
| 252 |
|
---|
| 253 | for oValue in self.aoValues:
|
---|
| 254 | oClone.aoValues.append(oValue.clone(oClone));
|
---|
| 255 |
|
---|
| 256 | oClone.sEndTS = self.sEndTS;
|
---|
| 257 | oClone.sStatus = self.sStatus;
|
---|
| 258 | oClone.cErrors = self.cErrors;
|
---|
| 259 | return oClone;
|
---|
| 260 |
|
---|
[52776] | 261 | # parsing
|
---|
| 262 |
|
---|
| 263 | def addChild(self, oChild):
|
---|
| 264 | self.aoChildren.append(oChild);
|
---|
| 265 | return oChild;
|
---|
| 266 |
|
---|
[97266] | 267 | def addValue(self, oValue):
|
---|
[52776] | 268 | self.aoValues.append(oValue);
|
---|
| 269 | return oValue;
|
---|
| 270 |
|
---|
[97266] | 271 | def __markCompleted(self, sTimestamp):
|
---|
| 272 | """ Sets sEndTS if not already done. """
|
---|
| 273 | if not self.sEndTS:
|
---|
| 274 | self.sEndTS = sTimestamp;
|
---|
| 275 |
|
---|
| 276 | def markPassed(self, sTimestamp):
|
---|
| 277 | self.__markCompleted(sTimestamp);
|
---|
[52776] | 278 | self.sStatus = 'passed';
|
---|
| 279 | self.cErrors = 0;
|
---|
| 280 |
|
---|
[97266] | 281 | def markSkipped(self, sTimestamp):
|
---|
| 282 | self.__markCompleted(sTimestamp);
|
---|
[52776] | 283 | self.sStatus = 'skipped';
|
---|
| 284 | self.cErrors = 0;
|
---|
| 285 |
|
---|
[97266] | 286 | def markFailed(self, sTimestamp, cErrors):
|
---|
| 287 | self.__markCompleted(sTimestamp);
|
---|
[52776] | 288 | self.sStatus = 'failed';
|
---|
[97266] | 289 | self.cErrors = cErrors;
|
---|
[52776] | 290 |
|
---|
[97266] | 291 | def markEnd(self, sTimestamp, cErrors):
|
---|
| 292 | self.__markCompleted(sTimestamp);
|
---|
[52776] | 293 | if self.sStatus is None:
|
---|
[97266] | 294 | self.sStatus = 'failed' if cErrors != 0 else 'end';
|
---|
[52776] | 295 | self.cErrors = 0;
|
---|
| 296 |
|
---|
| 297 | def mergeInIncludedTest(self, oTest):
|
---|
| 298 | """ oTest will be robbed. """
|
---|
| 299 | if oTest is not None:
|
---|
| 300 | for oChild in oTest.aoChildren:
|
---|
| 301 | oChild.oParent = self;
|
---|
| 302 | self.aoChildren.append(oChild);
|
---|
| 303 | for oValue in oTest.aoValues:
|
---|
| 304 | oValue.oTest = self;
|
---|
| 305 | self.aoValues.append(oValue);
|
---|
| 306 | oTest.aoChildren = [];
|
---|
| 307 | oTest.aoValues = [];
|
---|
| 308 |
|
---|
| 309 | # debug
|
---|
| 310 |
|
---|
| 311 | def printTree(self, iLevel = 0):
|
---|
[79092] | 312 | print('%sTest: name=%s start=%s end=%s' % (''.ljust(iLevel*2), self.sName, self.sStartTS, self.sEndTS));
|
---|
[52776] | 313 | for oChild in self.aoChildren:
|
---|
| 314 | oChild.printTree(iLevel + 1);
|
---|
| 315 | for oValue in self.aoValues:
|
---|
| 316 | oValue.printValue(iLevel + 1);
|
---|
| 317 |
|
---|
| 318 | # getters / queries
|
---|
| 319 |
|
---|
| 320 | def getFullNameWorker(self, cSkipUpper):
|
---|
| 321 | if self.oParent is None:
|
---|
| 322 | return (self.sName, 0);
|
---|
| 323 | sName, iLevel = self.oParent.getFullNameWorker(cSkipUpper);
|
---|
| 324 | if iLevel < cSkipUpper:
|
---|
| 325 | sName = self.sName;
|
---|
| 326 | else:
|
---|
| 327 | sName += ', ' + self.sName;
|
---|
| 328 | return (sName, iLevel + 1);
|
---|
| 329 |
|
---|
| 330 | def getFullName(self, cSkipUpper = 2):
|
---|
| 331 | return self.getFullNameWorker(cSkipUpper)[0];
|
---|
| 332 |
|
---|
[97267] | 333 | def matchFilters(self, aoFilters):
|
---|
[52776] | 334 | """
|
---|
[97267] | 335 | Checks for any substring match between aoFilters (str or re.Pattern)
|
---|
| 336 | and the full test name.
|
---|
| 337 |
|
---|
| 338 | Returns True if any of the filters matches.
|
---|
| 339 | Returns False if none of the filters matches.
|
---|
[52776] | 340 | """
|
---|
[97267] | 341 | sFullName = self.getFullName();
|
---|
| 342 | for oFilter in aoFilters:
|
---|
| 343 | if oFilter.search(sFullName) is not None if isinstance(oFilter, re.Pattern) else sFullName.find(oFilter) >= 0:
|
---|
| 344 | return True;
|
---|
| 345 | return False;
|
---|
[52776] | 346 |
|
---|
| 347 | # manipulation
|
---|
| 348 |
|
---|
[97267] | 349 | def filterTestsWorker(self, asFilters, fReturnOnMatch):
|
---|
[52776] | 350 | # depth first
|
---|
| 351 | i = 0;
|
---|
| 352 | while i < len(self.aoChildren):
|
---|
[97267] | 353 | if self.aoChildren[i].filterTestsWorker(asFilters, fReturnOnMatch):
|
---|
[52776] | 354 | i += 1;
|
---|
| 355 | else:
|
---|
| 356 | self.aoChildren[i].oParent = None;
|
---|
| 357 | del self.aoChildren[i];
|
---|
| 358 |
|
---|
| 359 | # If we have children, they must've matched up.
|
---|
[79092] | 360 | if self.aoChildren:
|
---|
[52776] | 361 | return True;
|
---|
[97267] | 362 | if self.matchFilters(asFilters):
|
---|
| 363 | return fReturnOnMatch;
|
---|
| 364 | return not fReturnOnMatch;
|
---|
[52776] | 365 |
|
---|
| 366 | def filterTests(self, asFilters):
|
---|
[97267] | 367 | """ Keep tests matching asFilters. """
|
---|
[79092] | 368 | if asFilters:
|
---|
[97267] | 369 | self.filterTestsWorker(asFilters, True);
|
---|
[52776] | 370 | return self;
|
---|
| 371 |
|
---|
[97267] | 372 | def filterOutTests(self, asFilters):
|
---|
| 373 | """ Removes tests matching asFilters. """
|
---|
| 374 | if asFilters:
|
---|
| 375 | self.filterTestsWorker(asFilters, False);
|
---|
| 376 | return self;
|
---|
| 377 |
|
---|
| 378 | def filterValuesWorker(self, asFilters, fKeepWhen):
|
---|
| 379 | # Process children recursively.
|
---|
| 380 | for oChild in self.aoChildren:
|
---|
| 381 | oChild.filterValuesWorker(asFilters, fKeepWhen);
|
---|
| 382 |
|
---|
| 383 | # Filter our values.
|
---|
| 384 | iValue = len(self.aoValues);
|
---|
| 385 | if iValue > 0:
|
---|
| 386 | sFullname = self.getFullName() + ': ';
|
---|
| 387 | while iValue > 0:
|
---|
| 388 | iValue -= 1;
|
---|
| 389 | if self.aoValues[iValue].matchFilters(sFullname, asFilters) != fKeepWhen:
|
---|
| 390 | del self.aoValues[iValue];
|
---|
| 391 | return None;
|
---|
| 392 |
|
---|
| 393 | def filterValues(self, asFilters):
|
---|
| 394 | """ Keep values matching asFilters. """
|
---|
| 395 | if asFilters:
|
---|
| 396 | self.filterValuesWorker(asFilters, True);
|
---|
| 397 | return self;
|
---|
| 398 |
|
---|
| 399 | def filterOutValues(self, asFilters):
|
---|
| 400 | """ Removes values matching asFilters. """
|
---|
| 401 | if asFilters:
|
---|
| 402 | self.filterValuesWorker(asFilters, False);
|
---|
| 403 | return self;
|
---|
| 404 |
|
---|
| 405 | def filterOutEmptyLeafTests(self):
|
---|
| 406 | """
|
---|
| 407 | Removes any child tests that has neither values nor sub-tests.
|
---|
| 408 | Returns True if leaf, False if not.
|
---|
| 409 | """
|
---|
| 410 | iChild = len(self.aoChildren);
|
---|
| 411 | while iChild > 0:
|
---|
| 412 | iChild -= 1;
|
---|
| 413 | if self.aoChildren[iChild].filterOutEmptyLeafTests():
|
---|
| 414 | del self.aoChildren[iChild];
|
---|
| 415 | return not self.aoChildren and not self.aoValues;
|
---|
| 416 |
|
---|
[97266] | 417 | @staticmethod
|
---|
| 418 | def calcDurationStatic(sStartTS, sEndTS):
|
---|
| 419 | """
|
---|
| 420 | Returns None the start timestamp is absent or invalid.
|
---|
| 421 | Returns datetime.timedelta otherwise.
|
---|
| 422 | """
|
---|
| 423 | if not sStartTS:
|
---|
| 424 | return None;
|
---|
| 425 | try:
|
---|
| 426 | oStart = utils.parseIsoTimestamp(sStartTS);
|
---|
| 427 | except:
|
---|
| 428 | return None;
|
---|
[52776] | 429 |
|
---|
[97266] | 430 | if not sEndTS:
|
---|
| 431 | return datetime.timedelta.max;
|
---|
| 432 | try:
|
---|
| 433 | oEnd = utils.parseIsoTimestamp(sEndTS);
|
---|
| 434 | except:
|
---|
| 435 | return datetime.timedelta.max;
|
---|
| 436 |
|
---|
| 437 | return oEnd - oStart;
|
---|
| 438 |
|
---|
| 439 | def calcDuration(self):
|
---|
| 440 | """
|
---|
| 441 | Returns the duration as a datetime.timedelta object or None if not available.
|
---|
| 442 | """
|
---|
| 443 | return self.calcDurationStatic(self.sStartTS, self.sEndTS);
|
---|
| 444 |
|
---|
| 445 | def calcDurationAsMicroseconds(self):
|
---|
| 446 | """
|
---|
| 447 | Returns the duration as microseconds or None if not available.
|
---|
| 448 | """
|
---|
| 449 | oDuration = self.calcDuration();
|
---|
| 450 | if not oDuration:
|
---|
| 451 | return None;
|
---|
| 452 | return (oDuration.days * 86400 + oDuration.seconds) * 1000000 + oDuration.microseconds;
|
---|
| 453 |
|
---|
| 454 | @staticmethod
|
---|
| 455 | def distillTimes(aoTestRuns, sMethod, sStatus):
|
---|
| 456 | """
|
---|
| 457 | Destills the error counts of the tests.
|
---|
| 458 | Returns a (sStartTS, sEndTS) pair.
|
---|
| 459 | """
|
---|
| 460 |
|
---|
| 461 | #
|
---|
| 462 | # Start by assembling two list of start and end times for all runs that have a start timestamp.
|
---|
| 463 | # Then sort out the special cases where no run has a start timestamp and only a single one has.
|
---|
| 464 | #
|
---|
| 465 | asStartTS = [oRun.sStartTS for oRun in aoTestRuns if oRun.sStartTS];
|
---|
| 466 | if not asStartTS:
|
---|
| 467 | return (None, None);
|
---|
| 468 | asEndTS = [oRun.sEndTS for oRun in aoTestRuns if oRun.sStartTS]; # parallel to asStartTS, so we don't check sEndTS.
|
---|
| 469 | if len(asStartTS) == 1:
|
---|
| 470 | return (asStartTS[0], asEndTS[0]);
|
---|
| 471 |
|
---|
| 472 | #
|
---|
| 473 | # Calculate durations for all runs.
|
---|
| 474 | #
|
---|
| 475 | if sMethod == 'best':
|
---|
| 476 | aoDurations = [Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS) for oRun in aoTestRuns if oRun.sStatus == sStatus];
|
---|
| 477 | if not aoDurations or aoDurations.count(None) == len(aoDurations):
|
---|
| 478 | aoDurations = [Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS) for oRun in aoTestRuns];
|
---|
| 479 | if aoDurations.count(None) == len(aoDurations):
|
---|
| 480 | return (asStartTS[0], None);
|
---|
| 481 | oDuration = min([oDuration for oDuration in aoDurations if oDuration is not None]);
|
---|
| 482 |
|
---|
| 483 | elif sMethod == 'avg':
|
---|
| 484 | print("dbg: 0: sStatus=%s []=%s"
|
---|
| 485 | % (sStatus, [(Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS),oRun.sStatus) for oRun in aoTestRuns],));
|
---|
| 486 | aoDurations = [Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS) for oRun in aoTestRuns if oRun.sStatus == sStatus];
|
---|
| 487 | print("dbg: 1: aoDurations=%s" % (aoDurations,))
|
---|
| 488 | aoDurations = [oDuration for oDuration in aoDurations if oDuration];
|
---|
| 489 | print("dbg: 2: aoDurations=%s" % (aoDurations,))
|
---|
| 490 | if not aoDurations:
|
---|
| 491 | return (asStartTS[0], None);
|
---|
| 492 | aoDurations = [oDuration for oDuration in aoDurations if oDuration < datetime.timedelta.max];
|
---|
| 493 | print("dbg: 3: aoDurations=%s" % (aoDurations,))
|
---|
| 494 | if not aoDurations:
|
---|
| 495 | return (asStartTS[0], None);
|
---|
| 496 | # sum doesn't work on timedelta, so do it manually.
|
---|
| 497 | oDuration = aoDurations[0];
|
---|
| 498 | for i in range(1, len(aoDurations)):
|
---|
| 499 | oDuration += aoDurations[i];
|
---|
| 500 | print("dbg: 5: oDuration=%s" % (aoDurations,))
|
---|
| 501 | oDuration = oDuration / len(aoDurations);
|
---|
| 502 | print("dbg: 6: oDuration=%s" % (aoDurations,))
|
---|
| 503 |
|
---|
| 504 | else:
|
---|
| 505 | assert False;
|
---|
| 506 | return (asStartTS[0], asEndTS[0]);
|
---|
| 507 |
|
---|
| 508 | # Check unfinished test.
|
---|
| 509 | if oDuration >= datetime.timedelta.max:
|
---|
| 510 | return (asStartTS[0], None);
|
---|
| 511 |
|
---|
| 512 | # Calculate and format the end timestamp string.
|
---|
| 513 | oStartTS = utils.parseIsoTimestamp(asStartTS[0]);
|
---|
| 514 | oEndTS = oStartTS + oDuration;
|
---|
| 515 | return (asStartTS[0], utils.formatIsoTimestamp(oEndTS));
|
---|
| 516 |
|
---|
| 517 | @staticmethod
|
---|
| 518 | def distillStatus(aoTestRuns, sMethod):
|
---|
| 519 | """
|
---|
| 520 | Destills the status of the tests.
|
---|
| 521 | Returns the status.
|
---|
| 522 | """
|
---|
| 523 | asStatuses = [oRun.sStatus for oRun in aoTestRuns];
|
---|
| 524 |
|
---|
| 525 | if sMethod == 'best':
|
---|
| 526 | for sStatus in ('passed', 'failed', 'skipped'):
|
---|
| 527 | if sStatus in asStatuses:
|
---|
| 528 | return sStatus;
|
---|
| 529 | return asStatuses[0];
|
---|
| 530 |
|
---|
| 531 | if sMethod == 'avg':
|
---|
| 532 | cPassed = asStatuses.count('passed');
|
---|
| 533 | cFailed = asStatuses.count('failed');
|
---|
| 534 | cSkipped = asStatuses.count('skipped');
|
---|
| 535 | cEnd = asStatuses.count('end');
|
---|
| 536 | cNone = asStatuses.count(None);
|
---|
| 537 | if cPassed >= cFailed and cPassed >= cSkipped and cPassed >= cNone and cPassed >= cEnd:
|
---|
| 538 | return 'passed';
|
---|
| 539 | if cFailed >= cPassed and cFailed >= cSkipped and cFailed >= cNone and cFailed >= cEnd:
|
---|
| 540 | return 'failed';
|
---|
| 541 | if cSkipped >= cPassed and cSkipped >= cFailed and cSkipped >= cNone and cSkipped >= cEnd:
|
---|
| 542 | return 'skipped';
|
---|
| 543 | if cEnd >= cPassed and cEnd >= cFailed and cEnd >= cNone and cEnd >= cSkipped:
|
---|
| 544 | return 'end';
|
---|
| 545 | return None;
|
---|
| 546 |
|
---|
| 547 | assert False;
|
---|
| 548 | return asStatuses[0];
|
---|
| 549 |
|
---|
| 550 | @staticmethod
|
---|
| 551 | def distillErrors(aoTestRuns, sMethod):
|
---|
| 552 | """
|
---|
| 553 | Destills the error counts of the tests.
|
---|
| 554 | Returns the status.
|
---|
| 555 | """
|
---|
| 556 | acErrorsXcptNeg = [oRun.cErrors for oRun in aoTestRuns if oRun.cErrors >= 0];
|
---|
| 557 |
|
---|
| 558 | if sMethod == 'best':
|
---|
| 559 | if acErrorsXcptNeg:
|
---|
| 560 | return min(acErrorsXcptNeg);
|
---|
| 561 | elif sMethod == 'avg':
|
---|
| 562 | if acErrorsXcptNeg:
|
---|
| 563 | return sum(acErrorsXcptNeg) // len(acErrorsXcptNeg);
|
---|
| 564 | else:
|
---|
| 565 | assert False;
|
---|
| 566 | return -1;
|
---|
| 567 |
|
---|
| 568 | def distill(self, aoTestRuns, sMethod, fDropLoners):
|
---|
| 569 | """
|
---|
| 570 | Distills the test runs into this test.
|
---|
| 571 | """
|
---|
| 572 | #
|
---|
| 573 | # Recurse first (before we create too much state in the stack
|
---|
| 574 | # frame) and do child tests.
|
---|
| 575 | #
|
---|
| 576 | # We copy the child lists of each test run so we can remove tests we've
|
---|
| 577 | # processed from each run and thus make sure we include tests in
|
---|
| 578 | #
|
---|
| 579 | #
|
---|
| 580 | aaoChildren = [list(oRun.aoChildren) for oRun in aoTestRuns];
|
---|
| 581 |
|
---|
| 582 | # Process the tests for each run.
|
---|
| 583 | for i, _ in enumerate(aaoChildren):
|
---|
| 584 | # Process all tests for the current run.
|
---|
| 585 | while len(aaoChildren[i]) > 0:
|
---|
| 586 | oFirst = aaoChildren[i].pop(0);
|
---|
| 587 |
|
---|
| 588 | # Build a list of sub-test runs by searching remaining runs by test name.
|
---|
| 589 | aoSameSubTests = [oFirst,];
|
---|
| 590 | for j in range(i + 1, len(aaoChildren)):
|
---|
| 591 | aoThis = aaoChildren[j];
|
---|
| 592 | for iThis, oThis in enumerate(aoThis):
|
---|
| 593 | if oThis.sName == oFirst.sName:
|
---|
| 594 | del aoThis[iThis];
|
---|
| 595 | aoSameSubTests.append(oThis);
|
---|
| 596 | break;
|
---|
| 597 |
|
---|
| 598 | # Apply fDropLoners.
|
---|
| 599 | if not fDropLoners or len(aoSameSubTests) > 1 or len(aaoChildren) == 1:
|
---|
| 600 | # Create an empty test and call distill on it with the subtest array, unless
|
---|
| 601 | # of course that the array only has one member and we can simply clone it.
|
---|
| 602 | if len(aoSameSubTests) == 1:
|
---|
| 603 | self.addChild(oFirst.clone(self));
|
---|
| 604 | else:
|
---|
| 605 | oSubTest = Test(self);
|
---|
| 606 | oSubTest.sName = oFirst.sName;
|
---|
| 607 | oSubTest.distill(aoSameSubTests, sMethod, fDropLoners);
|
---|
| 608 | self.addChild(oSubTest);
|
---|
| 609 | del aaoChildren;
|
---|
| 610 |
|
---|
| 611 | #
|
---|
| 612 | # Do values. Similar approch as for the sub-tests.
|
---|
| 613 | #
|
---|
| 614 | aaoValues = [list(oRun.aoValues) for oRun in aoTestRuns];
|
---|
| 615 |
|
---|
| 616 | # Process the values for each run.
|
---|
| 617 | for i,_ in enumerate(aaoValues):
|
---|
| 618 | # Process all values for the current run.
|
---|
| 619 | while len(aaoValues[i]) > 0:
|
---|
| 620 | oFirst = aaoValues[i].pop(0);
|
---|
| 621 |
|
---|
| 622 | # Build a list of values runs by searching remaining runs by value name and unit.
|
---|
| 623 | aoSameValues = [oFirst,];
|
---|
| 624 | for j in range(i + 1, len(aaoValues)):
|
---|
| 625 | aoThis = aaoValues[j];
|
---|
| 626 | for iThis, oThis in enumerate(aoThis):
|
---|
| 627 | if oThis.sName == oFirst.sName and oThis.sUnit == oFirst.sUnit:
|
---|
| 628 | del aoThis[iThis];
|
---|
| 629 | aoSameValues.append(oThis);
|
---|
| 630 | break;
|
---|
| 631 |
|
---|
| 632 | # Apply fDropLoners.
|
---|
| 633 | if not fDropLoners or len(aoSameValues) > 1 or len(aaoValues) == 1:
|
---|
| 634 | # Create an empty test and call distill on it with the subtest array, unless
|
---|
| 635 | # of course that the array only has one member and we can simply clone it.
|
---|
| 636 | if len(aoSameValues) == 1:
|
---|
| 637 | self.aoValues.append(oFirst.clone(self));
|
---|
| 638 | else:
|
---|
| 639 | oValue = Value(self);
|
---|
| 640 | oValue.distill(aoSameValues, sMethod);
|
---|
| 641 | self.aoValues.append(oValue);
|
---|
| 642 | del aaoValues;
|
---|
| 643 |
|
---|
| 644 | #
|
---|
| 645 | # Distill test properties.
|
---|
| 646 | #
|
---|
| 647 | self.sStatus = self.distillStatus(aoTestRuns, sMethod);
|
---|
| 648 | self.cErrors = self.distillErrors(aoTestRuns, sMethod);
|
---|
| 649 | (self.sStartTS, self.sEndTS) = self.distillTimes(aoTestRuns, sMethod, self.sStatus);
|
---|
| 650 | print("dbg: %s: sStartTS=%s, sEndTS=%s" % (self.sName, self.sStartTS, self.sEndTS));
|
---|
| 651 |
|
---|
| 652 | return self;
|
---|
| 653 |
|
---|
| 654 |
|
---|
[52776] | 655 | class XmlLogReader(object):
|
---|
| 656 | """
|
---|
| 657 | XML log reader class.
|
---|
| 658 | """
|
---|
| 659 |
|
---|
| 660 | def __init__(self, sXmlFile):
|
---|
| 661 | self.sXmlFile = sXmlFile;
|
---|
| 662 | self.oRoot = Test(None, {'name': 'root', 'timestamp': ''});
|
---|
| 663 | self.oTest = self.oRoot;
|
---|
| 664 | self.iLevel = 0;
|
---|
| 665 | self.oValue = None;
|
---|
| 666 |
|
---|
| 667 | def parse(self):
|
---|
| 668 | try:
|
---|
[97266] | 669 | oFile = open(self.sXmlFile, 'rb'); # pylint: disable=consider-using-with
|
---|
[52776] | 670 | except:
|
---|
| 671 | traceback.print_exc();
|
---|
| 672 | return False;
|
---|
| 673 |
|
---|
| 674 | from xml.parsers.expat import ParserCreate
|
---|
| 675 | oParser = ParserCreate();
|
---|
| 676 | oParser.StartElementHandler = self.handleElementStart;
|
---|
| 677 | oParser.CharacterDataHandler = self.handleElementData;
|
---|
| 678 | oParser.EndElementHandler = self.handleElementEnd;
|
---|
| 679 | try:
|
---|
| 680 | oParser.ParseFile(oFile);
|
---|
| 681 | except:
|
---|
| 682 | traceback.print_exc();
|
---|
| 683 | oFile.close();
|
---|
| 684 | return False;
|
---|
| 685 | oFile.close();
|
---|
| 686 | return True;
|
---|
| 687 |
|
---|
| 688 | def handleElementStart(self, sName, hsAttrs):
|
---|
[97266] | 689 | #print('%s%s: %s' % (''.ljust(self.iLevel * 2), sName, str(hsAttrs)));
|
---|
[79092] | 690 | if sName in ('Test', 'SubTest',):
|
---|
[52776] | 691 | self.iLevel += 1;
|
---|
| 692 | self.oTest = self.oTest.addChild(Test(self.oTest, hsAttrs));
|
---|
| 693 | elif sName == 'Value':
|
---|
[97266] | 694 | self.oValue = self.oTest.addValue(Value(self.oTest, hsAttrs.get('name'), hsAttrs.get('unit'),
|
---|
| 695 | hsAttrs.get('timestamp'), hsAttrs.get('value')));
|
---|
[52776] | 696 | elif sName == 'End':
|
---|
[97266] | 697 | self.oTest.markEnd(hsAttrs.get('timestamp'), int(hsAttrs.get('errors', '0')));
|
---|
[52776] | 698 | elif sName == 'Passed':
|
---|
[97266] | 699 | self.oTest.markPassed(hsAttrs.get('timestamp'));
|
---|
[52776] | 700 | elif sName == 'Skipped':
|
---|
[97266] | 701 | self.oTest.markSkipped(hsAttrs.get('timestamp'));
|
---|
[52776] | 702 | elif sName == 'Failed':
|
---|
[97266] | 703 | self.oTest.markFailed(hsAttrs.get('timestamp'), int(hsAttrs['errors']));
|
---|
[52776] | 704 | elif sName == 'Include':
|
---|
| 705 | self.handleInclude(hsAttrs);
|
---|
| 706 | else:
|
---|
[97266] | 707 | print('Unknown element "%s"' % (sName,));
|
---|
[52776] | 708 |
|
---|
| 709 | def handleElementData(self, sData):
|
---|
| 710 | if self.oValue is not None:
|
---|
| 711 | self.oValue.addData(sData);
|
---|
| 712 | elif sData.strip() != '':
|
---|
[79092] | 713 | print('Unexpected data "%s"' % (sData,));
|
---|
[52776] | 714 | return True;
|
---|
| 715 |
|
---|
| 716 | def handleElementEnd(self, sName):
|
---|
[79092] | 717 | if sName in ('Test', 'Subtest',):
|
---|
[52776] | 718 | self.iLevel -= 1;
|
---|
| 719 | self.oTest = self.oTest.oParent;
|
---|
| 720 | elif sName == 'Value':
|
---|
| 721 | self.oValue = None;
|
---|
| 722 | return True;
|
---|
| 723 |
|
---|
| 724 | def handleInclude(self, hsAttrs):
|
---|
| 725 | # relative or absolute path.
|
---|
| 726 | sXmlFile = hsAttrs['filename'];
|
---|
| 727 | if not os.path.isabs(sXmlFile):
|
---|
| 728 | sXmlFile = os.path.join(os.path.dirname(self.sXmlFile), sXmlFile);
|
---|
| 729 |
|
---|
| 730 | # Try parse it.
|
---|
| 731 | oSub = parseTestResult(sXmlFile);
|
---|
| 732 | if oSub is None:
|
---|
[79092] | 733 | print('error: failed to parse include "%s"' % (sXmlFile,));
|
---|
[52776] | 734 | else:
|
---|
| 735 | # Skip the root and the next level before merging it the subtest and
|
---|
| 736 | # values in to the current test. The reason for this is that the
|
---|
| 737 | # include is the output of some sub-program we've run and we don't need
|
---|
| 738 | # the extra test level it automatically adds.
|
---|
| 739 | #
|
---|
| 740 | # More benchmark heuristics: Walk down until we find more than one
|
---|
| 741 | # test or values.
|
---|
| 742 | oSub2 = oSub;
|
---|
[79092] | 743 | while len(oSub2.aoChildren) == 1 and not oSub2.aoValues:
|
---|
[52776] | 744 | oSub2 = oSub2.aoChildren[0];
|
---|
[79092] | 745 | if not oSub2.aoValues:
|
---|
[52776] | 746 | oSub2 = oSub;
|
---|
| 747 | self.oTest.mergeInIncludedTest(oSub2);
|
---|
| 748 | return True;
|
---|
| 749 |
|
---|
| 750 | def parseTestResult(sXmlFile):
|
---|
| 751 | """
|
---|
| 752 | Parses the test results in the XML.
|
---|
| 753 | Returns result tree.
|
---|
| 754 | Returns None on failure.
|
---|
| 755 | """
|
---|
| 756 | oXlr = XmlLogReader(sXmlFile);
|
---|
| 757 | if oXlr.parse():
|
---|
[97266] | 758 | if len(oXlr.oRoot.aoChildren) == 1 and not oXlr.oRoot.aoValues:
|
---|
| 759 | return oXlr.oRoot.aoChildren[0];
|
---|
[52776] | 760 | return oXlr.oRoot;
|
---|
| 761 | return None;
|
---|
| 762 |
|
---|