btresolver.py@ 66995

Last change on this file since 66995 was 65967, checked in by vboxsync, 8 years ago
ValidationKit/testdriver: pylint 2.0.0 fixes.
Property svn:executable set to ``*
File size: 20.5 KB

Line
1	# -- coding: utf-8 --
2	# $Id$
3	# pylint: disable=C0302
4
5	"""
6	Backtrace resolver using external debugging symbols and RTLdrFlt.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2016 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision$"
31
32
33	# Standard Python imports.
34	import os;
35	import re;
36	import shutil;
37	import subprocess;
38
39	# Validation Kit imports.
40	from common import utils;
41
42	def getRTLdrFltPath(asPaths):
43	"""
44	Returns the path to the RTLdrFlt tool looking in the provided paths
45	or None if not found.
46	"""
47
48	for sPath in asPaths:
49	for sDirPath, _, asFiles in os.walk(sPath):
50	if 'RTLdrFlt' in asFiles:
51	return os.path.join(sDirPath, 'RTLdrFlt');
52
53	return None;
54
55
56
57	class BacktraceResolverOs(object):
58	"""
59	Base class for all OS specific resolvers.
60	"""
61
62	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
63	self.sScratchPath = sScratchPath;
64	self.sBuildRoot = sBuildRoot;
65	self.fnLog = fnLog;
66
67	def log(self, sText):
68	"""
69	Internal logger callback.
70	"""
71	if self.fnLog is not None:
72	self.fnLog(sText);
73
74
75
76	class BacktraceResolverOsLinux(BacktraceResolverOs):
77	"""
78	Linux specific backtrace resolver.
79	"""
80
81	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
82	"""
83	Constructs a Linux host specific backtrace resolver.
84	"""
85	BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog);
86
87	self.asDbgFiles = {};
88
89	def prepareEnv(self):
90	"""
91	Prepares the environment for annotating Linux reports.
92	"""
93	fRc = False;
94	try:
95	sDbgArchive = os.path.join(self.sBuildRoot, 'bin', 'VirtualBox-dbg.tar.bz2');
96
97	# Extract debug symbol archive if it was found.
98	if os.path.exists(sDbgArchive):
99	asMembers = utils.unpackFile(sDbgArchive, self.sScratchPath, self.fnLog,
100	self.fnLog);
101	if asMembers:
102	# Populate the list of debug files.
103	for sMember in asMembers:
104	if os.path.isfile(sMember):
105	self.asDbgFiles[os.path.basename(sMember)] = sMember;
106	fRc = True;
107	except:
108	self.log('Failed to setup debug symbols');
109
110	return fRc;
111
112	def cleanupEnv(self):
113	"""
114	Cleans up the environment.
115	"""
116	fRc = False;
117	try:
118	shutil.rmtree(self.sScratchPath, True);
119	fRc = True;
120	except:
121	pass;
122
123	return fRc;
124
125	def getDbgSymPathFromBinary(self, sBinary, sArch):
126	"""
127	Returns the path to file containing the debug symbols for the specified binary.
128	"""
129	_ = sArch;
130	sDbgFilePath = None;
131	try:
132	sDbgFilePath = self.asDbgFiles[sBinary];
133	except:
134	pass;
135
136	return sDbgFilePath;
137
138	def getBinaryListWithLoadAddrFromReport(self, asReport):
139	"""
140	Parses the given VM state report and returns a list of binaries and their
141	load address.
142
143	Returns a list if tuples containing the binary and load addres or an empty
144	list on failure.
145	"""
146	asListBinaries = [];
147
148	# Look for the line "Mapped address spaces:"
149	iLine = 0;
150	while iLine < len(asReport):
151	if asReport[iLine].startswith('Mapped address spaces:'):
152	break;
153	iLine += 1;
154
155	for sLine in asReport[iLine:]:
156	asCandidate = sLine.split();
157	if len(asCandidate) == 5 \
158	and asCandidate[0].startswith('0x') \
159	and asCandidate[1].startswith('0x') \
160	and asCandidate[2].startswith('0x') \
161	and (asCandidate[3] == '0x0' or asCandidate[3] == '0')\
162	and 'VirtualBox' in asCandidate[4]:
163	asListBinaries.append((asCandidate[0], os.path.basename(asCandidate[4])));
164
165	return asListBinaries;
166
167
168
169	class BacktraceResolverOsDarwin(BacktraceResolverOs):
170	"""
171	Darwin specific backtrace resolver.
172	"""
173
174	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
175	"""
176	Constructs a Linux host specific backtrace resolver.
177	"""
178	BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog);
179
180	self.asDbgFiles = {};
181
182	def prepareEnv(self):
183	"""
184	Prepares the environment for annotating Darwin reports.
185	"""
186	fRc = False;
187	try:
188	#
189	# Walk the scratch path directory and look for .dSYM directories, building a
190	# list of them.
191	#
192	asDSymPaths = [];
193
194	for sDirPath, asDirs, _ in os.walk(self.sBuildRoot):
195	for sDir in asDirs:
196	if sDir.endswith('.dSYM'):
197	asDSymPaths.append(os.path.join(sDirPath, sDir));
198
199	# Expand the dSYM paths to full DWARF debug files in the next step
200	# and add them to the debug files dictionary.
201	for sDSymPath in asDSymPaths:
202	sBinary = os.path.basename(sDSymPath).strip('.dSYM');
203	self.asDbgFiles[sBinary] = os.path.join(sDSymPath, 'Contents', 'Resources',
204	'DWARF', sBinary);
205
206	fRc = True;
207	except:
208	self.log('Failed to setup debug symbols');
209
210	return fRc;
211
212	def cleanupEnv(self):
213	"""
214	Cleans up the environment.
215	"""
216	fRc = False;
217	try:
218	shutil.rmtree(self.sScratchPath, True);
219	fRc = True;
220	except:
221	pass;
222
223	return fRc;
224
225	def getDbgSymPathFromBinary(self, sBinary, sArch):
226	"""
227	Returns the path to file containing the debug symbols for the specified binary.
228	"""
229	# Hack to exclude executables as RTLdrFlt has some problems with it currently.
230	_ = sArch;
231	sDbgSym = None;
232	try:
233	sDbgSym = self.asDbgFiles[sBinary];
234	except:
235	pass;
236
237	if sDbgSym is not None and sDbgSym.endswith('.dylib'):
238	return sDbgSym;
239
240	return None;
241
242	def _getReportVersion(self, asReport):
243	"""
244	Returns the version of the darwin report.
245	"""
246	# Find the line starting with "Report Version:"
247	iLine = 0;
248	iVersion = 0;
249	while iLine < len(asReport):
250	if asReport[iLine].startswith('Report Version:'):
251	break;
252	iLine += 1;
253
254	if iLine < len(asReport):
255	# Look for the start of the number
256	sVersion = asReport[iLine];
257	iStartVersion = len('Report Version:');
258	iEndVersion = len(sVersion);
259
260	while iStartVersion < len(sVersion) \
261	and not sVersion[iStartVersion:iStartVersion+1].isdigit():
262	iStartVersion += 1;
263
264	while iEndVersion > 0 \
265	and not sVersion[iEndVersion-1:iEndVersion].isdigit():
266	iEndVersion -= 1;
267
268	iVersion = int(sVersion[iStartVersion:iEndVersion]);
269	else:
270	self.log('Couldn\'t find the report version');
271
272	return iVersion;
273
274	def _getListOfBinariesFromReportPreSierra(self, asReport):
275	"""
276	Returns a list of loaded binaries with their load address obtained from
277	a pre Sierra report.
278	"""
279	asListBinaries = [];
280
281	# Find the line starting with "Binary Images:"
282	iLine = 0;
283	while iLine < len(asReport):
284	if asReport[iLine].startswith('Binary Images:'):
285	break;
286	iLine += 1;
287
288	if iLine < len(asReport):
289	# List starts after that
290	iLine += 1;
291
292	# A line for a loaded binary looks like the following:
293	# 0x100042000 - 0x100095fff +VBoxDDU.dylib (4.3.15) <EB19C44D-F882-0803-DBDD-9995723111B7> /Application...
294	# We need the start address and the library name.
295	# To distinguish between our own libraries and ones from Apple we check whether the path at the end starts with
296	# /Applications/VirtualBox.app/Contents/MacOS
297	oRegExpPath = re.compile(r'/VirtualBox.app/Contents/MacOS');
298	oRegExpAddr = re.compile(r'0x\w+');
299	oRegExpBinPath = re.compile(r'VirtualBox.app/Contents/MacOS/\S*');
300	while iLine < len(asReport):
301	asMatches = oRegExpPath.findall(asReport[iLine]);
302	if asMatches:
303	# Line contains the path, extract start address and path to binary
304	sAddr = oRegExpAddr.findall(asReport[iLine]);
305	sPath = oRegExpBinPath.findall(asReport[iLine]);
306
307	if sAddr and sPath:
308	# Construct the path in into the build cache containing the debug symbols
309	oRegExp = re.compile(r'\w+\.{0,1}\w*$');
310	sFilename = oRegExp.findall(sPath[0]);
311
312	asListBinaries.append((sAddr[0], sFilename[0]));
313	else:
314	break; # End of image list
315	iLine += 1;
316	else:
317	self.log('Couldn\'t find the list of loaded binaries in the given report');
318
319	return asListBinaries;
320
321	def _getListOfBinariesFromReportSierra(self, asReport):
322	"""
323	Returns a list of loaded binaries with their load address obtained from
324	a Sierra+ report.
325	"""
326	asListBinaries = [];
327
328	# A line for a loaded binary looks like the following:
329	# 4 VBoxXPCOMIPCC.dylib 0x00000001139f17ea 0x1139e4000 + 55274
330	# We need the start address and the library name.
331	# To distinguish between our own libraries and ones from Apple we check whether the library
332	# name contains VBox or VirtualBox
333	iLine = 0;
334	while iLine < len(asReport):
335	asStackTrace = asReport[iLine].split();
336
337	# Check whether the line is made up of 6 elements separated by whitespace
338	# and the first one is a number.
339	if len(asStackTrace) == 6 and asStackTrace[0].isdigit() \
340	and (asStackTrace[1].find('VBox') != -1 or asStackTrace[1].find('VirtualBox') != -1) \
341	and asStackTrace[3].startswith('0x'):
342
343	# Check whether the library is already in our list an only add new ones
344	fFound = False;
345	for _, sLibrary in asListBinaries:
346	if asStackTrace[1] == sLibrary:
347	fFound = True;
348	break;
349
350	if not fFound:
351	asListBinaries.append((asStackTrace[3], asStackTrace[1]));
352	iLine += 1;
353
354	return asListBinaries;
355
356	def getBinaryListWithLoadAddrFromReport(self, asReport):
357	"""
358	Parses the given VM state report and returns a list of binaries and their
359	load address.
360
361	Returns a list if tuples containing the binary and load addres or an empty
362	list on failure.
363	"""
364	asListBinaries = [];
365
366	iVersion = self._getReportVersion(asReport);
367	if iVersion > 0:
368	if iVersion <= 11:
369	self.log('Pre Sierra Report');
370	asListBinaries = self._getListOfBinariesFromReportPreSierra(asReport);
371	elif iVersion == 12:
372	self.log('Sierra report');
373	asListBinaries = self._getListOfBinariesFromReportSierra(asReport);
374	else:
375	self.log('Unsupported report version %s' % (iVersion, ));
376
377	return asListBinaries;
378
379
380
381	class BacktraceResolverOsSolaris(BacktraceResolverOs):
382	"""
383	Solaris specific backtrace resolver.
384	"""
385
386	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
387	"""
388	Constructs a Linux host specific backtrace resolver.
389	"""
390	BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog);
391
392	self.asDbgFiles = {};
393
394	def prepareEnv(self):
395	"""
396	Prepares the environment for annotating Linux reports.
397	"""
398	fRc = False;
399	try:
400	sDbgArchive = os.path.join(self.sBuildRoot, 'bin', 'VirtualBoxDebug.tar.bz2');
401
402	# Extract debug symbol archive if it was found.
403	if os.path.exists(sDbgArchive):
404	asMembers = utils.unpackFile(sDbgArchive, self.sScratchPath, self.fnLog,
405	self.fnLog);
406	if asMembers:
407	# Populate the list of debug files.
408	for sMember in asMembers:
409	if os.path.isfile(sMember):
410	sArch = '';
411	if 'amd64' in sMember:
412	sArch = 'amd64';
413	else:
414	sArch = 'x86';
415	self.asDbgFiles[os.path.basename(sMember) + '/' + sArch] = sMember;
416	fRc = True;
417	else:
418	self.log('Unpacking the debug archive failed');
419	except:
420	self.log('Failed to setup debug symbols');
421
422	return fRc;
423
424	def cleanupEnv(self):
425	"""
426	Cleans up the environment.
427	"""
428	fRc = False;
429	try:
430	shutil.rmtree(self.sScratchPath, True);
431	fRc = True;
432	except:
433	pass;
434
435	return fRc;
436
437	def getDbgSymPathFromBinary(self, sBinary, sArch):
438	"""
439	Returns the path to file containing the debug symbols for the specified binary.
440	"""
441	sDbgFilePath = None;
442	try:
443	sDbgFilePath = self.asDbgFiles[sBinary + '/' + sArch];
444	except:
445	pass;
446
447	return sDbgFilePath;
448
449	def getBinaryListWithLoadAddrFromReport(self, asReport):
450	"""
451	Parses the given VM state report and returns a list of binaries and their
452	load address.
453
454	Returns a list if tuples containing the binary and load addres or an empty
455	list on failure.
456	"""
457	asListBinaries = [];
458
459	# Look for the beginning of the process address space mappings"
460	for sLine in asReport:
461	asItems = sLine.split();
462	if len(asItems) == 4 \
463	and asItems[3].startswith('/opt/VirtualBox') \
464	and asItems[2] == 'r-x--':
465	fFound = False;
466	sBinaryFile = os.path.basename(asItems[3]);
467	for _, sBinary in asListBinaries:
468	if sBinary == sBinaryFile:
469	fFound = True;
470	break;
471	if not fFound:
472	asListBinaries.append(('0x' + asItems[0], sBinaryFile));
473
474	return asListBinaries;
475
476
477
478	class BacktraceResolver(object):
479	"""
480	A backtrace resolving class.
481	"""
482
483	def __init__(self, sScratchPath, sBuildRoot, sTargetOs, sArch, sRTLdrFltPath = None, fnLog = None):
484	"""
485	Constructs a backtrace resolver object for the given target OS,
486	architecture and path to the directory containing the debug symbols and tools
487	we need.
488	"""
489	# Initialize all members first.
490	self.sScratchPath = sScratchPath;
491	self.sBuildRoot = sBuildRoot;
492	self.sTargetOs = sTargetOs;
493	self.sArch = sArch;
494	self.sRTLdrFltPath = sRTLdrFltPath;
495	self.fnLog = fnLog;
496	self.sDbgSymPath = None;
497	self.oResolverOs = None;
498	self.sScratchDbgPath = os.path.join(self.sScratchPath, 'dbgsymbols');
499
500	if self.fnLog is None:
501	self.fnLog = self.logStub;
502
503	if self.sRTLdrFltPath is None:
504	self.sRTLdrFltPath = getRTLdrFltPath([self.sScratchPath, self.sBuildRoot]);
505	if self.sRTLdrFltPath is not None:
506	self.log('Found RTLdrFlt in %s' % (self.sRTLdrFltPath,));
507	else:
508	self.log('Couldn\'t find RTLdrFlt in either %s or %s' % (self.sScratchPath, self.sBuildRoot));
509
510	def log(self, sText):
511	"""
512	Internal logger callback.
513	"""
514	if self.fnLog is not None:
515	self.fnLog(sText);
516
517	def logStub(self, sText):
518	"""
519	Logging stub doing nothing.
520	"""
521	_ = sText;
522
523	def prepareEnv(self):
524	"""
525	Prepares the environment to annotate backtraces, finding the required tools
526	and retrieving the debug symbols depending on the host OS.
527
528	Returns True on success and False on error or if not supported.
529	"""
530
531	# No access to the RTLdrFlt tool means no symbols so no point in trying
532	# to set something up.
533	if self.sRTLdrFltPath is None:
534	return False;
535
536	# Create a directory containing the scratch space for the OS resolver backends.
537	fRc = True;
538	if not os.path.exists(self.sScratchDbgPath):
539	try:
540	os.makedirs(self.sScratchDbgPath, 0750);
541	except:
542	fRc = False;
543	self.log('Failed to create scratch directory for debug symbols');
544
545	if fRc:
546	if self.sTargetOs == 'linux':
547	self.oResolverOs = BacktraceResolverOsLinux(self.sScratchDbgPath, self.sScratchPath, self.fnLog);
548	elif self.sTargetOs == 'darwin':
549	self.oResolverOs = BacktraceResolverOsDarwin(self.sScratchDbgPath, self.sScratchPath, self.fnLog); # pylint: disable=R0204
550	elif self.sTargetOs == 'solaris':
551	self.oResolverOs = BacktraceResolverOsSolaris(self.sScratchDbgPath, self.sScratchPath, self.fnLog); # pylint: disable=R0204
552	else:
553	self.log('The backtrace resolver is not supported on %s' % (self.sTargetOs,));
554	fRc = False;
555
556	if fRc:
557	fRc = self.oResolverOs.prepareEnv();
558	if not fRc:
559	self.oResolverOs = None;
560
561	if not fRc:
562	shutil.rmtree(self.sScratchDbgPath, True)
563
564	return fRc;
565
566	def cleanupEnv(self):
567	"""
568	Prepares the environment to annotate backtraces, finding the required tools
569	and retrieving the debug symbols depending on the host OS.
570
571	Returns True on success and False on error or if not supported.
572	"""
573	fRc = False;
574	if self.oResolverOs is not None:
575	fRc = self.oResolverOs.cleanupEnv();
576
577	shutil.rmtree(self.sScratchDbgPath, True);
578	return fRc;
579
580	def annotateReport(self, sReport):
581	"""
582	Annotates the given report with the previously prepared environment.
583
584	Returns the annotated report on success or None on failure.
585	"""
586	sReportAn = None;
587
588	if self.oResolverOs is not None:
589	asListBinaries = self.oResolverOs.getBinaryListWithLoadAddrFromReport(sReport.split('\n'));
590
591	if asListBinaries:
592	asArgs = [self.sRTLdrFltPath, ];
593
594	for sLoadAddr, sBinary in asListBinaries:
595	sDbgSymPath = self.oResolverOs.getDbgSymPathFromBinary(sBinary, self.sArch);
596	if sDbgSymPath is not None:
597	asArgs.append(sDbgSymPath);
598	asArgs.append(sLoadAddr);
599
600	oRTLdrFltProc = subprocess.Popen(asArgs, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0);
601	if oRTLdrFltProc is not None:
602	sReportAn, _ = oRTLdrFltProc.communicate(sReport);
603	else:
604	self.log('Error spawning RTLdrFlt process');
605	else:
606	self.log('Getting list of loaded binaries failed');
607	else:
608	self.log('Backtrace resolver not fully initialized, not possible to annotate');
609
610	return sReportAn;
611

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testdriver/btresolver.py@ 66995

Download in other formats: