utils.py@ 106199

Last change on this file since 106199 was 106061, checked in by vboxsync, 2 months ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 89.9 KB

Line
1	# -- coding: utf-8 --
2	# $Id: utils.py 106061 2024-09-16 14:03:52Z vboxsync $
3	# pylint: disable=too-many-lines
4
5	"""
6	Common Utility Functions.
7	"""
8
9	from __future__ import print_function;
10
11	__copyright__ = \
12	"""
13	Copyright (C) 2012-2024 Oracle and/or its affiliates.
14
15	This file is part of VirtualBox base platform packages, as
16	available from https://www.virtualbox.org.
17
18	This program is free software; you can redistribute it and/or
19	modify it under the terms of the GNU General Public License
20	as published by the Free Software Foundation, in version 3 of the
21	License.
22
23	This program is distributed in the hope that it will be useful, but
24	WITHOUT ANY WARRANTY; without even the implied warranty of
25	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
26	General Public License for more details.
27
28	You should have received a copy of the GNU General Public License
29	along with this program; if not, see <https://www.gnu.org/licenses>.
30
31	The contents of this file may alternatively be used under the terms
32	of the Common Development and Distribution License Version 1.0
33	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
34	in the VirtualBox distribution, in which case the provisions of the
35	CDDL are applicable instead of those of the GPL.
36
37	You may elect to license modified versions of this file under the
38	terms and conditions of either the GPL or the CDDL or both.
39
40	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
41	"""
42	__version__ = "$Revision: 106061 $"
43
44
45	# Standard Python imports.
46	import datetime;
47	import errno;
48	import os;
49	import platform;
50	import re;
51	import stat;
52	import subprocess;
53	import sys;
54	import time;
55	import traceback;
56	import unittest;
57
58	if sys.platform == 'win32':
59	import ctypes;
60	import msvcrt; # pylint: disable=import-error
61	import win32api; # pylint: disable=import-error
62	import win32con; # pylint: disable=import-error
63	import win32console; # pylint: disable=import-error
64	import win32file; # pylint: disable=import-error
65	import win32process; # pylint: disable=import-error
66	import winerror; # pylint: disable=import-error
67	import pywintypes; # pylint: disable=import-error
68	else:
69	import signal;
70
71	# Python 3 hacks:
72	if sys.version_info[0] >= 3:
73	unicode = str; # pylint: disable=redefined-builtin,invalid-name
74	xrange = range; # pylint: disable=redefined-builtin,invalid-name
75	long = int; # pylint: disable=redefined-builtin,invalid-name
76
77
78	#
79	# Python 2/3 glue.
80	#
81
82	if sys.version_info[0] >= 3:
83	def iteritems(dDict):
84	"""
85	Wrapper around dict.items() / dict.iteritems().
86	"""
87	return iter(dDict.items());
88	else:
89	def iteritems(dDict):
90	"""
91	Wrapper around dict.items() / dict.iteritems().
92	"""
93	return dDict.iteritems();
94
95
96	#
97	# Strings.
98	#
99
100	def toUnicode(sString, encoding = None, errors = 'strict'):
101	"""
102	A little like the python 2 unicode() function.
103	"""
104	if sys.version_info[0] >= 3:
105	if isinstance(sString, bytes):
106	return str(sString, encoding if encoding else 'utf-8', errors);
107	else:
108	if not isinstance(sString, unicode):
109	return unicode(sString, encoding if encoding else 'utf-8', errors);
110	return sString;
111
112
113
114	#
115	# Output.
116	#
117
118	def printOut(sString):
119	"""
120	Outputs a string to standard output, dealing with python 2.x encoding stupidity.
121	"""
122	sStreamEncoding = sys.stdout.encoding;
123	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
124	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
125	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
126	print(sString);
127	else:
128	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
129
130	def printErr(sString):
131	"""
132	Outputs a string to standard error, dealing with python 2.x encoding stupidity.
133	"""
134	sStreamEncoding = sys.stderr.encoding;
135	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
136	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
137	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
138	print(sString, file = sys.stderr);
139	else:
140	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
141
142
143	#
144	# Host OS and CPU.
145	#
146
147	def getHostOs():
148	"""
149	Gets the host OS name (short).
150
151	See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
152	"""
153	sPlatform = platform.system();
154	if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
155	sPlatform = sPlatform.lower();
156	elif sPlatform == 'Windows':
157	sPlatform = 'win';
158	elif sPlatform == 'SunOS':
159	sPlatform = 'solaris';
160	else:
161	raise Exception('Unsupported platform "%s"' % (sPlatform,));
162	return sPlatform;
163
164	g_sHostArch = None;
165
166	def getHostArch():
167	"""
168	Gets the host CPU architecture.
169
170	See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
171	"""
172	global g_sHostArch;
173	if g_sHostArch is None:
174	sArch = platform.machine();
175	if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
176	sArch = 'x86';
177	elif sArch in ('AMD64', 'amd64', 'x86_64'):
178	sArch = 'amd64';
179	elif sArch == 'i86pc': # SunOS
180	if platform.architecture()[0] == '64bit':
181	sArch = 'amd64';
182	else:
183	try:
184	sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
185	except:
186	pass;
187	sArch = sArch.strip();
188	if sArch != 'amd64':
189	sArch = 'x86';
190	elif sArch in ('arm64', 'ARM64', 'aarch64'):
191	sArch = 'arm64';
192	else:
193	raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
194	g_sHostArch = sArch;
195	return g_sHostArch;
196
197
198	def getHostOsDotArch():
199	"""
200	Gets the 'os.arch' for the host.
201	"""
202	return '%s.%s' % (getHostOs(), getHostArch());
203
204
205	def isValidOs(sOs):
206	"""
207	Validates the OS name.
208	"""
209	if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
210	'os2', 'solaris', 'win', 'os-agnostic'):
211	return True;
212	return False;
213
214
215	def isValidArch(sArch):
216	"""
217	Validates the CPU architecture name.
218	"""
219	if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
220	'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'arm64', 'alpha'):
221	return True;
222	return False;
223
224	def isValidOsDotArch(sOsDotArch):
225	"""
226	Validates the 'os.arch' string.
227	"""
228
229	asParts = sOsDotArch.split('.');
230	if asParts.length() != 2:
231	return False;
232	return isValidOs(asParts[0]) \
233	and isValidArch(asParts[1]);
234
235	def getHostOsVersion():
236	"""
237	Returns the host OS version. This is platform.release with additional
238	distro indicator on linux.
239	"""
240	sVersion = platform.release();
241	sOs = getHostOs();
242	if sOs == 'linux':
243	sDist = '';
244	try:
245	# try /etc/lsb-release first to distinguish between Debian and Ubuntu
246	with open('/etc/lsb-release') as oFile: # pylint: disable=unspecified-encoding
247	for sLine in oFile:
248	oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s=)\s"(.)"', sLine);
249	if oMatch is not None:
250	sDist = oMatch.group(1).strip();
251	except:
252	pass;
253	if sDist:
254	sVersion += ' / ' + sDist;
255	else:
256	asFiles = \
257	[
258	[ '/etc/debian_version', 'Debian v'],
259	[ '/etc/gentoo-release', '' ],
260	[ '/etc/oracle-release', '' ],
261	[ '/etc/redhat-release', '' ],
262	[ '/etc/SuSE-release', '' ],
263	];
264	for sFile, sPrefix in asFiles:
265	if os.path.isfile(sFile):
266	try:
267	with open(sFile) as oFile: # pylint: disable=unspecified-encoding
268	sLine = oFile.readline();
269	except:
270	continue;
271	sLine = sLine.strip()
272	if sLine:
273	sVersion += ' / ' + sPrefix + sLine;
274	break;
275
276	elif sOs == 'solaris':
277	sVersion = platform.version();
278	if os.path.isfile('/etc/release'):
279	try:
280	with open('/etc/release') as oFile: # pylint: disable=unspecified-encoding
281	sLast = oFile.readlines()[-1];
282	sLast = sLast.strip();
283	if sLast:
284	sVersion += ' (' + sLast + ')';
285	except:
286	pass;
287
288	elif sOs == 'darwin':
289	def getMacVersionName(sVersion):
290	"""
291	Figures out the Mac OS X/macOS code name from the numeric version.
292	"""
293	aOsVersion = sVersion.split('.') # example: ('10','15','7')
294	codenames = {"4": "Tiger",
295	"5": "Leopard",
296	"6": "Snow Leopard",
297	"7": "Lion",
298	"8": "Mountain Lion",
299	"9": "Mavericks",
300	"10": "Yosemite",
301	"11": "El Capitan",
302	"12": "Sierra",
303	"13": "High Sierra",
304	"14": "Mojave",
305	"15": "Catalina",
306	"16": "Wrong version",
307	}
308	codenames_afterCatalina = {"11": "Big Sur",
309	"12": "Monterey",
310	"13": "Ventura",
311	"14": "Sonoma",
312	"15": "Unknown 15",
313	"16": "Unknown 16"}
314
315	if aOsVersion[0] == '10':
316	sResult = codenames[aOsVersion[1]]
317	else:
318	sResult = codenames_afterCatalina[aOsVersion[0]]
319	return sResult
320
321	sOsxVersion = platform.mac_ver()[0]
322	sVersion += ' / OS X ' + sOsxVersion + ' (' + getMacVersionName(sOsxVersion) + ')'
323
324	elif sOs == 'win':
325	class OSVersionInfoEx(ctypes.Structure): # pylint: disable=used-before-assignment
326	""" OSVERSIONEX """
327	kaFields = [
328	('dwOSVersionInfoSize', ctypes.c_ulong),
329	('dwMajorVersion', ctypes.c_ulong),
330	('dwMinorVersion', ctypes.c_ulong),
331	('dwBuildNumber', ctypes.c_ulong),
332	('dwPlatformId', ctypes.c_ulong),
333	('szCSDVersion', ctypes.c_wchar*128),
334	('wServicePackMajor', ctypes.c_ushort),
335	('wServicePackMinor', ctypes.c_ushort),
336	('wSuiteMask', ctypes.c_ushort),
337	('wProductType', ctypes.c_byte),
338	('wReserved', ctypes.c_byte)]
339	_fields_ = kaFields # pylint: disable=invalid-name
340
341	def __init__(self):
342	super(OSVersionInfoEx, self).__init__()
343	self.dwOSVersionInfoSize = ctypes.sizeof(self)
344
345	oOsVersion = OSVersionInfoEx()
346	rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
347	if rc == 0:
348	# Python platform.release() is not reliable for newer server releases
349	if oOsVersion.wProductType != 1:
350	if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0 \
351	and oOsVersion.dwBuildNumber == 17763:
352	sVersion = '2019Server'
353	elif oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
354	sVersion = '2016Server'; #todo: should probably add dwBuildNumber for it as well..
355	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
356	sVersion = '2012ServerR2';
357	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
358	sVersion = '2012Server';
359	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
360	sVersion = '2008ServerR2';
361	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
362	sVersion = '2008Server';
363	elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
364	sVersion = '2003Server';
365	sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
366	if oOsVersion.wServicePackMajor:
367	sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
368	if oOsVersion.wServicePackMinor:
369	sVersion += '.' + str(oOsVersion.wServicePackMinor)
370
371	return sVersion;
372
373	def getPresentCpuCount():
374	"""
375	Gets the number of CPUs present in the system.
376
377	This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
378	that we return the active count rather than the maximum count. If we don't,
379	we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
380	never will have.
381
382	@todo This is probably not exactly what we get on non-windows...
383	"""
384
385	if getHostOs() == 'win':
386	fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
387	if fnGetActiveProcessorCount:
388	cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
389	if cCpus > 0:
390	return cCpus;
391
392	import multiprocessing
393	return multiprocessing.cpu_count();
394
395
396	#
397	# File system.
398	#
399
400	def openNoInherit(sFile, sMode = 'r'):
401	"""
402	Wrapper around open() that tries it's best to make sure the file isn't
403	inherited by child processes.
404
405	This is a best effort thing at the moment as it doesn't synchronizes with
406	child process spawning in any way. Thus it can be subject to races in
407	multithreaded programs.
408	"""
409
410	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
411	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
412	if uPythonVer >= ((3 << 16) \| 4):
413	oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
414	else:
415	try:
416	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
417	except:
418	# On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
419	if getHostOs() == 'win':
420	if uPythonVer < (3 << 16):
421	offComma = sMode.find(',');
422	if offComma < 0:
423	return open(sFile, sMode + 'N'); # pylint: disable=consider-using-with,unspecified-encoding
424	return open(sFile, # pylint: disable=consider-using-with,unspecified-encoding,bad-open-mode
425	sMode[:offComma] + 'N' + sMode[offComma:]);
426
427	# Just in case.
428	return open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
429
430	oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
431	#try:
432	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
433	#except:
434	# pass;
435	return oFile;
436
437	def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
438	"""
439	Wrapper around open() that tries it's best to make sure the file isn't
440	inherited by child processes.
441
442	This is a best effort thing at the moment as it doesn't synchronizes with
443	child process spawning in any way. Thus it can be subject to races in
444	multithreaded programs.
445	"""
446
447	if getHostOs() == 'win':
448	# Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
449	# pylint: disable=no-member,c-extension-no-member
450	fAccess = 0;
451	fDisposition = win32file.OPEN_EXISTING; # pylint: disable=used-before-assignment
452	if 'r' in sMode or '+' in sMode:
453	fAccess \|= win32file.GENERIC_READ;
454	if 'a' in sMode:
455	fAccess \|= win32file.GENERIC_WRITE;
456	fDisposition = win32file.OPEN_ALWAYS;
457	elif 'w' in sMode:
458	fAccess = win32file.GENERIC_WRITE;
459	if '+' in sMode:
460	fDisposition = win32file.OPEN_ALWAYS;
461	fAccess \|= win32file.GENERIC_READ;
462	else:
463	fDisposition = win32file.CREATE_ALWAYS;
464	if not fAccess:
465	fAccess \|= win32file.GENERIC_READ;
466	fSharing = (win32file.FILE_SHARE_READ \| win32file.FILE_SHARE_WRITE
467	\| win32file.FILE_SHARE_DELETE);
468	hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
469	if 'a' in sMode:
470	win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
471
472	# Turn the NT handle into a CRT file descriptor.
473	hDetachedFile = hFile.Detach();
474	if fAccess == win32file.GENERIC_READ:
475	fOpen = os.O_RDONLY;
476	elif fAccess == win32file.GENERIC_WRITE:
477	fOpen = os.O_WRONLY;
478	else:
479	fOpen = os.O_RDWR;
480	# pulint: enable=no-member,c-extension-no-member
481	if 'a' in sMode:
482	fOpen \|= os.O_APPEND;
483	if 'b' in sMode or 't' in sMode:
484	fOpen \|= os.O_TEXT; # pylint: disable=no-member
485	fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen); # pylint: disable=used-before-assignment
486
487	# Tell python to use this handle.
488	oFile = os.fdopen(fdFile, sMode);
489	else:
490	oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
491
492	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
493	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
494	if uPythonVer < ((3 << 16) \| 4):
495	try:
496	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
497	except:
498	pass;
499	else:
500	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
501	return oFile;
502
503	def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
504	"""
505	No exceptions os.readlink wrapper.
506	"""
507	try:
508	sRet = os.readlink(sPath); # pylint: disable=no-member
509	except:
510	return sXcptRet;
511	if hasattr(sRet, 'decode'):
512	sRet = sRet.decode(sEncoding, 'ignore');
513	return sRet;
514
515	def readFile(sFile, sMode = 'rb'):
516	"""
517	Reads the entire file.
518	"""
519	with open(sFile, sMode) as oFile: # pylint: disable=unspecified-encoding
520	sRet = oFile.read();
521	return sRet;
522
523	def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
524	"""
525	No exceptions common.readFile wrapper.
526	"""
527	try:
528	sRet = readFile(sFile, sMode);
529	except:
530	sRet = sXcptRet;
531	if sEncoding is not None and hasattr(sRet, 'decode'):
532	sRet = sRet.decode(sEncoding, 'ignore');
533	return sRet;
534
535	def noxcptRmDir(sDir, oXcptRet = False):
536	"""
537	No exceptions os.rmdir wrapper.
538	"""
539	oRet = True;
540	try:
541	os.rmdir(sDir);
542	except:
543	oRet = oXcptRet;
544	return oRet;
545
546	def noxcptDeleteFile(sFile, oXcptRet = False):
547	"""
548	No exceptions os.remove wrapper.
549	"""
550	oRet = True;
551	try:
552	os.remove(sFile);
553	except:
554	oRet = oXcptRet;
555	return oRet;
556
557
558	def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
559	# type: (string, (string, stat) -> bool) -> bool
560	"""
561	Recursively walks a directory tree, calling fnCallback for each.
562
563	fnCallback takes a full path and stat object (can be None). It
564	returns a boolean value, False stops walking and returns immediately.
565
566	Returns True or False depending on fnCallback.
567	Returns None fIgnoreExceptions is True and an exception was raised by listdir.
568	"""
569	def __worker(sCurDir):
570	""" Worker for """
571	try:
572	asNames = os.listdir(sCurDir);
573	except:
574	if not fIgnoreExceptions:
575	raise;
576	return None;
577	rc = True;
578	for sName in asNames:
579	if sName not in [ '.', '..' ]:
580	sFullName = os.path.join(sCurDir, sName);
581	try: oStat = os.lstat(sFullName);
582	except: oStat = None;
583	if fnCallback(sFullName, oStat) is False:
584	return False;
585	if oStat is not None and stat.S_ISDIR(oStat.st_mode):
586	rc = __worker(sFullName);
587	if rc is False:
588	break;
589	return rc;
590
591	# Ensure unicode path here so listdir also returns unicode on windows.
592	## @todo figure out unicode stuff on non-windows.
593	if sys.platform == 'win32':
594	sDir = unicode(sDir);
595	return __worker(sDir);
596
597
598
599	def formatFileMode(uMode):
600	# type: (int) -> string
601	"""
602	Format a st_mode value 'ls -la' fasion.
603	Returns string.
604	"""
605	if stat.S_ISDIR(uMode): sMode = 'd';
606	elif stat.S_ISREG(uMode): sMode = '-';
607	elif stat.S_ISLNK(uMode): sMode = 'l';
608	elif stat.S_ISFIFO(uMode): sMode = 'p';
609	elif stat.S_ISCHR(uMode): sMode = 'c';
610	elif stat.S_ISBLK(uMode): sMode = 'b';
611	elif stat.S_ISSOCK(uMode): sMode = 's';
612	else: sMode = '?';
613	## @todo sticky bits.
614	sMode += 'r' if uMode & stat.S_IRUSR else '-';
615	sMode += 'w' if uMode & stat.S_IWUSR else '-';
616	sMode += 'x' if uMode & stat.S_IXUSR else '-';
617	sMode += 'r' if uMode & stat.S_IRGRP else '-';
618	sMode += 'w' if uMode & stat.S_IWGRP else '-';
619	sMode += 'x' if uMode & stat.S_IXGRP else '-';
620	sMode += 'r' if uMode & stat.S_IROTH else '-';
621	sMode += 'w' if uMode & stat.S_IWOTH else '-';
622	sMode += 'x' if uMode & stat.S_IXOTH else '-';
623	sMode += ' ';
624	return sMode;
625
626
627	def formatFileStat(oStat):
628	# type: (stat) -> string
629	"""
630	Format a stat result 'ls -la' fasion (numeric IDs).
631	Returns string.
632	"""
633	return '%s %3s %4s %4s %10s %s' \
634	% (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
635	time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
636
637	## Good buffer for file operations.
638	g_cbGoodBufferSize = 256*1024;
639
640	## The original shutil.copyfileobj.
641	g_fnOriginalShCopyFileObj = None;
642
643	def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
644	""" shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
645	return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
646
647	def __installShUtilHacks(shutil):
648	""" Installs the shutil buffer size hacks. """
649	global g_fnOriginalShCopyFileObj;
650	if g_fnOriginalShCopyFileObj is None:
651	g_fnOriginalShCopyFileObj = shutil.copyfileobj;
652	shutil.copyfileobj = __myshutilcopyfileobj;
653	return True;
654
655
656	def copyFileSimple(sFileSrc, sFileDst):
657	"""
658	Wrapper around shutil.copyfile that simply copies the data of a regular file.
659	Raises exception on failure.
660	Return True for show.
661	"""
662	import shutil;
663	__installShUtilHacks(shutil);
664	return shutil.copyfile(sFileSrc, sFileDst);
665
666
667	def getDiskUsage(sPath):
668	"""
669	Get free space of a partition that corresponds to specified sPath in MB.
670
671	Returns partition free space value in MB.
672	"""
673	if platform.system() == 'Windows':
674	oCTypeFreeSpace = ctypes.c_ulonglong(0);
675	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
676	ctypes.pointer(oCTypeFreeSpace));
677	cbFreeSpace = oCTypeFreeSpace.value;
678	else:
679	oStats = os.statvfs(sPath); # pylint: disable=no-member
680	cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
681
682	# Convert to MB
683	cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
684
685	return cMbFreeSpace;
686
687
688
689	#
690	# SubProcess.
691	#
692
693	def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
694	"""
695	If the "executable" is a python script, insert the python interpreter at
696	the head of the argument list so that it will work on systems which doesn't
697	support hash-bang scripts.
698	"""
699
700	asArgs = dKeywordArgs.get('args');
701	if asArgs is None:
702	asArgs = aPositionalArgs[0];
703
704	if asArgs[0].endswith('.py'):
705	if sys.executable:
706	asArgs.insert(0, sys.executable);
707	else:
708	asArgs.insert(0, 'python');
709
710	# paranoia...
711	if dKeywordArgs.get('args') is not None:
712	dKeywordArgs['args'] = asArgs;
713	else:
714	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
715	return None;
716
717	def processPopenSafe(aPositionalArgs, *dKeywordArgs):
718	"""
719	Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
720	"""
721	if getHostOs() == 'win':
722	if dKeywordArgs.get('creationflags', 0) == 0:
723	dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
724	return subprocess.Popen(aPositionalArgs, *dKeywordArgs); # pylint: disable=consider-using-with
725
726	def processStart(aPositionalArgs, *dKeywordArgs):
727	"""
728	Wrapper around subprocess.Popen to deal with its absence in older
729	python versions.
730	Returns process object on success which can be worked on.
731	"""
732	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
733	return processPopenSafe(aPositionalArgs, *dKeywordArgs);
734
735	def processCall(aPositionalArgs, *dKeywordArgs):
736	"""
737	Wrapper around subprocess.call to deal with its absence in older
738	python versions.
739	Returns process exit code (see subprocess.poll).
740	"""
741	assert dKeywordArgs.get('stdout') is None;
742	assert dKeywordArgs.get('stderr') is None;
743	oProcess = processStart(aPositionalArgs, *dKeywordArgs);
744	return oProcess.wait();
745
746	def processOutputChecked(aPositionalArgs, *dKeywordArgs):
747	"""
748	Wrapper around subprocess.check_output to deal with its absense in older
749	python versions.
750
751	Extra keywords for specifying now output is to be decoded:
752	sEncoding='utf-8
753	fIgnoreEncoding=True/False
754	"""
755	sEncoding = dKeywordArgs.get('sEncoding');
756	if sEncoding is not None: del dKeywordArgs['sEncoding'];
757	else: sEncoding = 'utf-8';
758
759	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
760	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
761	else: fIgnoreEncoding = True;
762
763	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
764	oProcess = processPopenSafe(stdout=subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
765
766	sOutput, _ = oProcess.communicate();
767	iExitCode = oProcess.poll();
768
769	if iExitCode != 0:
770	asArgs = dKeywordArgs.get('args');
771	if asArgs is None:
772	asArgs = aPositionalArgs[0];
773	print(sOutput);
774	raise subprocess.CalledProcessError(iExitCode, asArgs);
775
776	if hasattr(sOutput, 'decode'):
777	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
778	return sOutput;
779
780	def processOutputUnchecked(aPositionalArgs, *dKeywordArgs):
781	"""
782	Similar to processOutputChecked, but returns status code and both stdout
783	and stderr results.
784
785	Extra keywords for specifying now output is to be decoded:
786	sEncoding='utf-8
787	fIgnoreEncoding=True/False
788	"""
789	sEncoding = dKeywordArgs.get('sEncoding');
790	if sEncoding is not None: del dKeywordArgs['sEncoding'];
791	else: sEncoding = 'utf-8';
792
793	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
794	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
795	else: fIgnoreEncoding = True;
796
797	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
798	oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
799
800	sOutput, sError = oProcess.communicate();
801	iExitCode = oProcess.poll();
802
803	if hasattr(sOutput, 'decode'):
804	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
805	if hasattr(sError, 'decode'):
806	sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
807	return (iExitCode, sOutput, sError);
808
809	g_fOldSudo = None;
810	def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
811	"""
812	Adds 'sudo' (or similar) to the args parameter, whereever it is.
813	"""
814
815	# Are we root?
816	fIsRoot = True;
817	try:
818	fIsRoot = os.getuid() == 0; # pylint: disable=no-member
819	except:
820	pass;
821
822	# If not, prepend sudo (non-interactive, simulate initial login).
823	if fIsRoot is not True:
824	asArgs = dKeywordArgs.get('args');
825	if asArgs is None:
826	asArgs = aPositionalArgs[0];
827
828	# Detect old sudo.
829	global g_fOldSudo;
830	if g_fOldSudo is None:
831	try:
832	sVersion = str(processOutputChecked(['sudo', '-V']));
833	except:
834	sVersion = '1.7.0';
835	sVersion = sVersion.strip().split('\n', 1)[0];
836	sVersion = sVersion.replace('Sudo version', '').strip();
837	g_fOldSudo = len(sVersion) >= 4 \
838	and sVersion[0] == '1' \
839	and sVersion[1] == '.' \
840	and sVersion[2] <= '6' \
841	and sVersion[3] == '.';
842
843	asArgs.insert(0, 'sudo');
844	if not g_fOldSudo:
845	asArgs.insert(1, '-n');
846	if fInitialEnv and not g_fOldSudo:
847	asArgs.insert(1, '-i');
848
849	# paranoia...
850	if dKeywordArgs.get('args') is not None:
851	dKeywordArgs['args'] = asArgs;
852	else:
853	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
854	return None;
855
856
857	def sudoProcessStart(aPositionalArgs, *dKeywordArgs):
858	"""
859	sudo (or similar) + subprocess.Popen,
860	returning the process object on success.
861	"""
862	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
863	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
864	return processStart(aPositionalArgs, *dKeywordArgs);
865
866	def sudoProcessCall(aPositionalArgs, *dKeywordArgs):
867	"""
868	sudo (or similar) + subprocess.call
869	"""
870	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
871	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
872	return processCall(aPositionalArgs, *dKeywordArgs);
873
874	def sudoProcessOutputChecked(aPositionalArgs, *dKeywordArgs):
875	"""
876	sudo (or similar) + subprocess.check_output.
877	"""
878	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
879	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
880	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
881
882	def sudoProcessOutputCheckedNoI(aPositionalArgs, *dKeywordArgs):
883	"""
884	sudo (or similar) + subprocess.check_output, except '-i' isn't used.
885	"""
886	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
887	_sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
888	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
889
890	def sudoProcessPopen(aPositionalArgs, *dKeywordArgs):
891	"""
892	sudo (or similar) + processPopenSafe.
893	"""
894	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
895	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
896	return processPopenSafe(aPositionalArgs, *dKeywordArgs);
897
898
899	def whichProgram(sName, sPath = None):
900	"""
901	Works similar to the 'which' utility on unix.
902
903	Returns path to the given program if found.
904	Returns None if not found.
905	"""
906	sHost = getHostOs();
907	sSep = ';' if sHost in [ 'win', 'os2' ] else ':';
908
909	if sPath is None:
910	if sHost == 'win':
911	sPath = os.environ.get('Path', None);
912	else:
913	sPath = os.environ.get('PATH', None);
914	if sPath is None:
915	return None;
916
917	for sDir in sPath.split(sSep):
918	if sDir.strip() != '':
919	sTest = os.path.abspath(os.path.join(sDir, sName));
920	else:
921	sTest = os.path.abspath(sName);
922	if os.path.exists(sTest):
923	return sTest;
924
925	return None;
926
927	#
928	# Generic process stuff.
929	#
930
931	def processInterrupt(uPid):
932	"""
933	Sends a SIGINT or equivalent to interrupt the specified process.
934	Returns True on success, False on failure.
935
936	On Windows hosts this may not work unless the process happens to be a
937	process group leader.
938	"""
939	if sys.platform == 'win32':
940	try:
941	win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
942	uPid);
943	fRc = True;
944	except:
945	fRc = False;
946	else:
947	try:
948	os.kill(uPid, signal.SIGINT);
949	fRc = True;
950	except:
951	fRc = False;
952	return fRc;
953
954	def sendUserSignal1(uPid):
955	"""
956	Sends a SIGUSR1 or equivalent to nudge the process into shutting down
957	(VBoxSVC) or something.
958	Returns True on success, False on failure or if not supported (win).
959
960	On Windows hosts this may not work unless the process happens to be a
961	process group leader.
962	"""
963	if sys.platform == 'win32':
964	fRc = False;
965	else:
966	try:
967	os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
968	fRc = True;
969	except:
970	fRc = False;
971	return fRc;
972
973	def processTerminate(uPid):
974	"""
975	Terminates the process in a nice manner (SIGTERM or equivalent).
976	Returns True on success, False on failure.
977	"""
978	fRc = False;
979	if sys.platform == 'win32':
980	try:
981	hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
982	False, uPid);
983	except:
984	pass;
985	else:
986	try:
987	win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
988	0x40010004); # DBG_TERMINATE_PROCESS
989	fRc = True;
990	except:
991	pass;
992	hProcess.Close(); #win32api.CloseHandle(hProcess)
993	else:
994	try:
995	os.kill(uPid, signal.SIGTERM);
996	fRc = True;
997	except:
998	pass;
999	return fRc;
1000
1001	def processKill(uPid):
1002	"""
1003	Terminates the process with extreme prejudice (SIGKILL).
1004	Returns True on success, False on failure.
1005	"""
1006	if sys.platform == 'win32':
1007	fRc = processTerminate(uPid);
1008	else:
1009	try:
1010	os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
1011	fRc = True;
1012	except:
1013	fRc = False;
1014	return fRc;
1015
1016	def processKillWithNameCheck(uPid, sName):
1017	"""
1018	Like processKill(), but checks if the process name matches before killing
1019	it. This is intended for killing using potentially stale pid values.
1020
1021	Returns True on success, False on failure.
1022	"""
1023
1024	if processCheckPidAndName(uPid, sName) is not True:
1025	return False;
1026	return processKill(uPid);
1027
1028
1029	def processExists(uPid):
1030	"""
1031	Checks if the specified process exits.
1032	This will only work if we can signal/open the process.
1033
1034	Returns True if it positively exists, False otherwise.
1035	"""
1036	sHostOs = getHostOs();
1037	if sHostOs == 'win':
1038	fRc = False;
1039	# We try open the process for waiting since this is generally only forbidden in a very few cases.
1040	try:
1041	hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
1042	False, uPid);
1043	except pywintypes.error as oXcpt: # pylint: disable=no-member,used-before-assignment
1044	if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED: # pylint: disable=used-before-assignment
1045	fRc = True;
1046	except:
1047	pass;
1048	else:
1049	hProcess.Close();
1050	fRc = True;
1051	else:
1052	fRc = False;
1053	try:
1054	os.kill(uPid, 0);
1055	fRc = True;
1056	except OSError as oXcpt:
1057	if oXcpt.errno == errno.EPERM:
1058	fRc = True;
1059	except:
1060	pass;
1061	return fRc;
1062
1063	def processCheckPidAndName(uPid, sName):
1064	"""
1065	Checks if a process PID and NAME matches.
1066	"""
1067	fRc = processExists(uPid);
1068	if fRc is not True:
1069	return False;
1070
1071	if sys.platform == 'win32':
1072	try:
1073	from win32com.client import GetObject; # pylint: disable=import-error
1074	oWmi = GetObject('winmgmts:');
1075	aoProcesses = oWmi.InstancesOf('Win32_Process');
1076	for oProcess in aoProcesses:
1077	if long(oProcess.Properties_("ProcessId").Value) == uPid:
1078	sCurName = oProcess.Properties_("Name").Value;
1079	#reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
1080	sName = sName.lower();
1081	sCurName = sCurName.lower();
1082	if os.path.basename(sName) == sName:
1083	sCurName = os.path.basename(sCurName);
1084
1085	if sCurName == sName \
1086	or sCurName + '.exe' == sName \
1087	or sCurName == sName + '.exe':
1088	fRc = True;
1089	break;
1090	except:
1091	#reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
1092	pass;
1093	else:
1094	if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
1095	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1096	elif sys.platform in ('sunos5',):
1097	asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1098	elif sys.platform in ('darwin',):
1099	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
1100	else:
1101	asPsCmd = None;
1102
1103	if asPsCmd is not None:
1104	try:
1105	oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE); # pylint: disable=consider-using-with
1106	sCurName = oPs.communicate()[0];
1107	iExitCode = oPs.wait();
1108	except:
1109	#reporter.logXcpt();
1110	return False;
1111
1112	# ps fails with non-zero exit code if the pid wasn't found.
1113	if iExitCode != 0:
1114	return False;
1115	if sCurName is None:
1116	return False;
1117	sCurName = sCurName.strip();
1118	if not sCurName:
1119	return False;
1120
1121	if os.path.basename(sName) == sName:
1122	sCurName = os.path.basename(sCurName);
1123	elif os.path.basename(sCurName) == sCurName:
1124	sName = os.path.basename(sName);
1125
1126	if sCurName != sName:
1127	return False;
1128
1129	fRc = True;
1130	return fRc;
1131
1132	def processGetInfo(uPid, fSudo = False):
1133	"""
1134	Tries to acquire state information of the given process.
1135
1136	Returns a string with the information on success or None on failure or
1137	if the host is not supported.
1138
1139	Note that the format of the information is host system dependent and will
1140	likely differ much between different hosts.
1141	"""
1142	fRc = processExists(uPid);
1143	if fRc is not True:
1144	return None;
1145
1146	sHostOs = getHostOs();
1147	if sHostOs in [ 'linux',]:
1148	sGdb = '/usr/bin/gdb';
1149	if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1150	if not os.path.isfile(sGdb): sGdb = 'gdb';
1151	aasCmd = [
1152	[ sGdb, '-batch',
1153	'-ex', 'set pagination off',
1154	'-ex', 'thread apply all bt',
1155	'-ex', 'info proc mapping',
1156	'-ex', 'info sharedlibrary',
1157	'-p', '%u' % (uPid,), ],
1158	];
1159	elif sHostOs == 'darwin':
1160	# LLDB doesn't work in batch mode when attaching to a process, at least
1161	# with macOS Sierra (10.12). GDB might not be installed. Use the sample
1162	# tool instead with a 1 second duration and 1000ms sampling interval to
1163	# get one stack trace. For the process mappings use vmmap.
1164	aasCmd = [
1165	[ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1166	[ '/usr/bin/vmmap', '%u' % (uPid,), ],
1167	];
1168	elif sHostOs == 'solaris':
1169	aasCmd = [
1170	[ '/usr/bin/pstack', '%u' % (uPid,), ],
1171	[ '/usr/bin/pmap', '%u' % (uPid,), ],
1172	];
1173	else:
1174	aasCmd = [];
1175
1176	sInfo = '';
1177	for asCmd in aasCmd:
1178	try:
1179	if fSudo:
1180	sThisInfo = sudoProcessOutputChecked(asCmd);
1181	else:
1182	sThisInfo = processOutputChecked(asCmd);
1183	if sThisInfo is not None:
1184	sInfo += sThisInfo;
1185	except:
1186	pass;
1187	if not sInfo:
1188	sInfo = None;
1189
1190	return sInfo;
1191
1192
1193	class ProcessInfo(object):
1194	"""Process info."""
1195	def __init__(self, iPid):
1196	self.iPid = iPid;
1197	self.iParentPid = None;
1198	self.sImage = None;
1199	self.sName = None;
1200	self.asArgs = None;
1201	self.sCwd = None;
1202	self.iGid = None;
1203	self.iUid = None;
1204	self.iProcGroup = None;
1205	self.iSessionId = None;
1206
1207	def loadAll(self):
1208	"""Load all the info."""
1209	sOs = getHostOs();
1210	if sOs == 'linux':
1211	sProc = '/proc/%s/' % (self.iPid,);
1212	if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1213	if self.sImage is None:
1214	self.sImage = noxcptReadFile(sProc + 'comm', None);
1215	if self.sImage: self.sImage = self.sImage.strip();
1216	if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1217	if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1218	#elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1219	# sProc = '/proc/%s/' % (self.iPid,);
1220	# if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1221	# if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1222	else:
1223	pass;
1224	if self.sName is None and self.sImage is not None:
1225	self.sName = self.sImage;
1226
1227	def windowsGrabProcessInfo(self, oProcess):
1228	"""Windows specific loadAll."""
1229	try: self.sName = oProcess.Properties_("Name").Value;
1230	except: pass;
1231	try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1232	except: pass;
1233	try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1234	except: pass;
1235	try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1236	except: pass;
1237	try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1238	except: pass;
1239	if self.sName is None and self.sImage is not None:
1240	self.sName = self.sImage;
1241
1242	def getBaseImageName(self):
1243	"""
1244	Gets the base image name if available, use the process name if not available.
1245	Returns image/process base name or None.
1246	"""
1247	sRet = self.sImage if self.sName is None else self.sName;
1248	if sRet is None:
1249	self.loadAll();
1250	sRet = self.sImage if self.sName is None else self.sName;
1251	if sRet is None:
1252	if not self.asArgs:
1253	return None;
1254	sRet = self.asArgs[0];
1255	if not sRet:
1256	return None;
1257	return os.path.basename(sRet);
1258
1259	def getBaseImageNameNoExeSuff(self):
1260	"""
1261	Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1262	"""
1263	sRet = self.getBaseImageName();
1264	if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1265	if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1266	sRet = sRet[:-4];
1267	return sRet;
1268
1269
1270	def processListAll():
1271	"""
1272	Return a list of ProcessInfo objects for all the processes in the system
1273	that the current user can see.
1274	"""
1275	asProcesses = [];
1276
1277	sOs = getHostOs();
1278	if sOs == 'win':
1279	from win32com.client import GetObject; # pylint: disable=import-error
1280	oWmi = GetObject('winmgmts:');
1281	aoProcesses = oWmi.InstancesOf('Win32_Process');
1282	for oProcess in aoProcesses:
1283	try:
1284	iPid = int(oProcess.Properties_("ProcessId").Value);
1285	except:
1286	continue;
1287	oMyInfo = ProcessInfo(iPid);
1288	oMyInfo.windowsGrabProcessInfo(oProcess);
1289	asProcesses.append(oMyInfo);
1290	return asProcesses;
1291
1292	if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1293	try:
1294	asDirs = os.listdir('/proc');
1295	except:
1296	asDirs = [];
1297	for sDir in asDirs:
1298	if sDir.isdigit():
1299	asProcesses.append(ProcessInfo(int(sDir),));
1300	return asProcesses;
1301
1302	#
1303	# The other OSes parses the output from the 'ps' utility.
1304	#
1305	asPsCmd = [
1306	'/bin/ps', # 0
1307	'-A', # 1
1308	'-o', 'pid=', # 2,3
1309	'-o', 'ppid=', # 4,5
1310	'-o', 'pgid=', # 6,7
1311	'-o', 'sid=', # 8,9
1312	'-o', 'uid=', # 10,11
1313	'-o', 'gid=', # 12,13
1314	'-o', 'comm=' # 14,15
1315	];
1316
1317	if sOs == 'darwin':
1318	assert asPsCmd[9] == 'sid=';
1319	asPsCmd[9] = 'sess=';
1320	elif sOs == 'solaris':
1321	asPsCmd[0] = '/usr/bin/ps';
1322
1323	try:
1324	sRaw = processOutputChecked(asPsCmd);
1325	except:
1326	return asProcesses;
1327
1328	for sLine in sRaw.split('\n'):
1329	sLine = sLine.lstrip();
1330	if len(sLine) < 7 or not sLine[0].isdigit():
1331	continue;
1332
1333	iField = 0;
1334	off = 0;
1335	aoFields = [None, None, None, None, None, None, None];
1336	while iField < 7:
1337	# Eat whitespace.
1338	while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1339	off += 1;
1340
1341	# Final field / EOL.
1342	if iField == 6:
1343	aoFields[6] = sLine[off:];
1344	break;
1345	if off >= len(sLine):
1346	break;
1347
1348	# Generic field parsing.
1349	offStart = off;
1350	off += 1;
1351	while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1352	off += 1;
1353	try:
1354	if iField != 3:
1355	aoFields[iField] = int(sLine[offStart:off]);
1356	else:
1357	aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1358	except:
1359	pass;
1360	iField += 1;
1361
1362	if aoFields[0] is not None:
1363	oMyInfo = ProcessInfo(aoFields[0]);
1364	oMyInfo.iParentPid = aoFields[1];
1365	oMyInfo.iProcGroup = aoFields[2];
1366	oMyInfo.iSessionId = aoFields[3];
1367	oMyInfo.iUid = aoFields[4];
1368	oMyInfo.iGid = aoFields[5];
1369	oMyInfo.sName = aoFields[6];
1370	asProcesses.append(oMyInfo);
1371
1372	return asProcesses;
1373
1374
1375	def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1376	"""
1377	Looks for information regarding the demise of the given process.
1378	"""
1379	sOs = getHostOs();
1380	if sOs == 'darwin':
1381	#
1382	# On darwin we look for crash and diagnostic reports.
1383	#
1384	asLogDirs = [
1385	u'/Library/Logs/DiagnosticReports/',
1386	u'/Library/Logs/CrashReporter/',
1387	u'~/Library/Logs/DiagnosticReports/',
1388	u'~/Library/Logs/CrashReporter/',
1389	];
1390	for sDir in asLogDirs:
1391	sDir = os.path.expanduser(sDir);
1392	if not os.path.isdir(sDir):
1393	continue;
1394	try:
1395	asDirEntries = os.listdir(sDir);
1396	except:
1397	continue;
1398	for sEntry in asDirEntries:
1399	# Only interested in .crash files.
1400	_, sSuff = os.path.splitext(sEntry);
1401	if sSuff != '.crash':
1402	continue;
1403
1404	# The pid can be found at the end of the first line.
1405	sFull = os.path.join(sDir, sEntry);
1406	try:
1407	with open(sFull, 'r') as oFile: # pylint: disable=unspecified-encoding
1408	sFirstLine = oFile.readline();
1409	except:
1410	continue;
1411	if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1412	continue;
1413	offPid = len(sFirstLine) - 3;
1414	while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1415	offPid -= 1;
1416	try: uReportPid = int(sFirstLine[offPid:-2]);
1417	except: continue;
1418
1419	# Does the pid we found match?
1420	if uReportPid == uPid:
1421	fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1422	fnCrashFile(sFull, False);
1423	elif sOs == 'win':
1424	#
1425	# Getting WER reports would be great, however we have trouble match the
1426	# PID to those as they seems not to mention it in the brief reports.
1427	# Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1428	# location - see the windows readme for the testbox script) and what
1429	# the MSDN article lists for now.
1430	#
1431	# It's been observed on Windows server 2012 that the dump files takes
1432	# the form: <processimage>.<decimal-pid>.dmp
1433	#
1434	asDmpDirs = [
1435	u'%SystemDrive%/CrashDumps/', # Testboxes.
1436	u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1437	u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1438	u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1439	u'%WINDIR%/ServiceProfiles/',
1440	u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1441	];
1442	sMatchSuffix = '.%u.dmp' % (uPid,);
1443
1444	for sDir in asDmpDirs:
1445	sDir = os.path.expandvars(sDir);
1446	if not os.path.isdir(sDir):
1447	continue;
1448	try:
1449	asDirEntries = os.listdir(sDir);
1450	except:
1451	continue;
1452	for sEntry in asDirEntries:
1453	if sEntry.endswith(sMatchSuffix):
1454	sFull = os.path.join(sDir, sEntry);
1455	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1456	fnCrashFile(sFull, True);
1457	elif sOs == 'solaris':
1458	asDmpDirs = [];
1459	try:
1460	sScratchPath = os.environ.get('TESTBOX_PATH_SCRATCH', None);
1461	asDmpDirs.extend([ sScratchPath ]);
1462	except:
1463	pass;
1464	# Some other useful locations as fallback.
1465	asDmpDirs.extend([
1466	u'/var/cores/'
1467	]);
1468	#
1469	# Solaris by default creates a core file in the directory of the crashing process with the name 'core'.
1470	#
1471	# As we need to distinguish the core files correlating to their PIDs and have a persistent storage location,
1472	# the host needs to be tweaked via:
1473	#
1474	# ```coreadm -g /var/cores/core.%f.%p```
1475	#
1476	sMatchRegEx = r'core\..*\.%u' % (uPid);
1477	for sDir in asDmpDirs:
1478	sDir = os.path.expandvars(sDir);
1479	if not os.path.isdir(sDir):
1480	continue;
1481	try:
1482	asDirEntries = os.listdir(sDir);
1483	except:
1484	continue;
1485	for sEntry in asDirEntries:
1486	fnLog('Entry: %s' % (os.path.join(sDir, sEntry)));
1487	if re.search(sMatchRegEx, sEntry):
1488	sFull = os.path.join(sDir, sEntry);
1489	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1490	fnCrashFile(sFull, True);
1491	else:
1492	pass; ## TODO
1493	return None;
1494
1495
1496	#
1497	# Time.
1498	#
1499
1500	#
1501	# The following test case shows how time.time() only have ~ms resolution
1502	# on Windows (tested W10) and why it therefore makes sense to try use
1503	# performance counters.
1504	#
1505	# Note! We cannot use time.clock() as the timestamp must be portable across
1506	# processes. See timeout testcase problem on win hosts (no logs).
1507	# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1508	#
1509	#import sys;
1510	#import time;
1511	#from common import utils;
1512	#
1513	#atSeries = [];
1514	#for i in xrange(1,160):
1515	# if i == 159: time.sleep(10);
1516	# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1517	#
1518	#tPrev = atSeries[0]
1519	#for tCur in atSeries:
1520	# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1521	# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1522	# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1523	# print '';
1524	# tPrev = tCur
1525	#
1526	#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1527	#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1528	#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1529
1530	g_fWinUseWinPerfCounter = sys.platform == 'win32';
1531	g_fpWinPerfCounterFreq = None;
1532	g_oFuncwinQueryPerformanceCounter = None;
1533
1534	def _winInitPerfCounter():
1535	""" Initializes the use of performance counters. """
1536	global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1537
1538	uFrequency = ctypes.c_ulonglong(0);
1539	if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1540	if uFrequency.value >= 1000:
1541	#print 'uFrequency = %s' % (uFrequency,);
1542	#print 'type(uFrequency) = %s' % (type(uFrequency),);
1543	g_fpWinPerfCounterFreq = float(uFrequency.value);
1544
1545	# Check that querying the counter works too.
1546	global g_oFuncwinQueryPerformanceCounter
1547	g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1548	uCurValue = ctypes.c_ulonglong(0);
1549	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1550	if uCurValue.value > 0:
1551	return True;
1552	g_fWinUseWinPerfCounter = False;
1553	return False;
1554
1555	def _winFloatTime():
1556	""" Gets floating point time on windows. """
1557	if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1558	uCurValue = ctypes.c_ulonglong(0);
1559	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1560	return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1561	return time.time();
1562
1563	def timestampNano():
1564	"""
1565	Gets a nanosecond timestamp.
1566	"""
1567	if g_fWinUseWinPerfCounter is True:
1568	return long(_winFloatTime() * 1000000000);
1569	return long(time.time() * 1000000000);
1570
1571	def timestampMilli():
1572	"""
1573	Gets a millisecond timestamp.
1574	"""
1575	if g_fWinUseWinPerfCounter is True:
1576	return long(_winFloatTime() * 1000);
1577	return long(time.time() * 1000);
1578
1579	def timestampSecond():
1580	"""
1581	Gets a second timestamp.
1582	"""
1583	if g_fWinUseWinPerfCounter is True:
1584	return long(_winFloatTime());
1585	return long(time.time());
1586
1587	def secondsSinceUnixEpoch():
1588	"""
1589	Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1590	"""
1591	## ASSUMES This returns unix epoch time on all systems we care about...
1592	return time.time();
1593
1594	def getTimePrefix():
1595	"""
1596	Returns a timestamp prefix, typically used for logging. UTC.
1597	"""
1598	try:
1599	oNow = datetime.datetime.utcnow();
1600	sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1601	except:
1602	sTs = 'getTimePrefix-exception';
1603	return sTs;
1604
1605	def getTimePrefixAndIsoTimestamp():
1606	"""
1607	Returns current UTC as log prefix and iso timestamp.
1608	"""
1609	try:
1610	oNow = datetime.datetime.utcnow();
1611	sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1612	sTsIso = formatIsoTimestamp(oNow);
1613	except:
1614	sTsPrf = sTsIso = 'getTimePrefix-exception';
1615	return (sTsPrf, sTsIso);
1616
1617	class UtcTzInfo(datetime.tzinfo):
1618	"""UTC TZ Info Class"""
1619	def utcoffset(self, _):
1620	return datetime.timedelta(0);
1621	def tzname(self, _):
1622	return "UTC";
1623	def dst(self, _):
1624	return datetime.timedelta(0);
1625
1626	class GenTzInfo(datetime.tzinfo):
1627	"""Generic TZ Info Class"""
1628	def __init__(self, offInMin):
1629	datetime.tzinfo.__init__(self);
1630	self.offInMin = offInMin;
1631	def utcoffset(self, _):
1632	return datetime.timedelta(minutes = self.offInMin);
1633	def tzname(self, _):
1634	if self.offInMin >= 0:
1635	return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1636	return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1637	def dst(self, _):
1638	return datetime.timedelta(0);
1639
1640	def formatIsoTimestamp(oNow):
1641	"""Formats the datetime object as an ISO timestamp."""
1642	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1643	sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1644	return sTs;
1645
1646	def getIsoTimestamp():
1647	"""Returns the current UTC timestamp as a string."""
1648	return formatIsoTimestamp(datetime.datetime.utcnow());
1649
1650	def formatShortIsoTimestamp(oNow):
1651	"""Formats the datetime object as an ISO timestamp, but w/o microseconds."""
1652	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1653	return oNow.strftime('%Y-%m-%dT%H:%M:%SZ');
1654
1655	def getShortIsoTimestamp():
1656	"""Returns the current UTC timestamp as a string, but w/o microseconds."""
1657	return formatShortIsoTimestamp(datetime.datetime.utcnow());
1658
1659	def convertDateTimeToZulu(oDateTime):
1660	""" Converts oDateTime to zulu time if it has timezone info. """
1661	if oDateTime.tzinfo is not None:
1662	oDateTime = oDateTime.astimezone(UtcTzInfo());
1663	else:
1664	oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1665	return oDateTime;
1666
1667	def parseIsoTimestamp(sTs):
1668	"""
1669	Parses a typical ISO timestamp, returing a datetime object, reasonably
1670	forgiving, but will throw weird indexing/conversion errors if the input
1671	is malformed.
1672	"""
1673	# YYYY-MM-DD
1674	iYear = int(sTs[0:4]);
1675	assert(sTs[4] == '-');
1676	iMonth = int(sTs[5:7]);
1677	assert(sTs[7] == '-');
1678	iDay = int(sTs[8:10]);
1679
1680	# Skip separator
1681	sTime = sTs[10:];
1682	while sTime[0] in 'Tt \t\n\r':
1683	sTime = sTime[1:];
1684
1685	# HH:MM[:SS]
1686	iHour = int(sTime[0:2]);
1687	assert(sTime[2] == ':');
1688	iMin = int(sTime[3:5]);
1689	if sTime[5] == ':':
1690	iSec = int(sTime[6:8]);
1691
1692	# Fraction?
1693	offTime = 8;
1694	iMicroseconds = 0;
1695	if offTime < len(sTime) and sTime[offTime] in '.,':
1696	offTime += 1;
1697	cchFraction = 0;
1698	while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1699	cchFraction += 1;
1700	if cchFraction > 0:
1701	iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1702	offTime += cchFraction;
1703	while cchFraction < 6:
1704	iMicroseconds *= 10;
1705	cchFraction += 1;
1706	while cchFraction > 6:
1707	iMicroseconds = iMicroseconds // 10;
1708	cchFraction -= 1;
1709
1710	else:
1711	iSec = 0;
1712	iMicroseconds = 0;
1713	offTime = 5;
1714
1715	# Naive?
1716	if offTime >= len(sTime):
1717	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1718
1719	# Zulu?
1720	if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1721	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1722
1723	# Some kind of offset afterwards, and strptime is useless. sigh.
1724	if sTime[offTime] in '+-':
1725	chSign = sTime[offTime];
1726	offTime += 1;
1727	cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1728	offTime += 2;
1729	if offTime < len(sTime) and sTime[offTime] in ':':
1730	offTime += 1;
1731	if offTime + 2 <= len(sTime):
1732	cMinTz += int(sTime[offTime : (offTime + 2)]);
1733	offTime += 2;
1734	assert offTime == len(sTime);
1735	if chSign == '-':
1736	cMinTz = -cMinTz;
1737	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1738	assert False, sTs;
1739	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1740
1741	def normalizeIsoTimestampToZulu(sTs):
1742	"""
1743	Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1744	+ convertDateTimeToZulu + formatIsoTimestamp).
1745	Returns ISO tiemstamp string.
1746	"""
1747	return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1748
1749	def getLocalHourOfWeek():
1750	""" Local hour of week (0 based). """
1751	oNow = datetime.datetime.now();
1752	return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1753
1754
1755	def formatIntervalSeconds(cSeconds):
1756	""" Format a seconds interval into a nice 01h 00m 22s string """
1757	# Two simple special cases.
1758	if cSeconds < 60:
1759	return '%ss' % (cSeconds,);
1760	if cSeconds < 3600:
1761	cMins = cSeconds // 60;
1762	cSecs = cSeconds % 60;
1763	if cSecs == 0:
1764	return '%sm' % (cMins,);
1765	return '%sm %ss' % (cMins, cSecs,);
1766
1767	# Generic and a bit slower.
1768	cDays = cSeconds // 86400;
1769	cSeconds %= 86400;
1770	cHours = cSeconds // 3600;
1771	cSeconds %= 3600;
1772	cMins = cSeconds // 60;
1773	cSecs = cSeconds % 60;
1774	sRet = '';
1775	if cDays > 0:
1776	sRet = '%sd ' % (cDays,);
1777	if cHours > 0:
1778	sRet += '%sh ' % (cHours,);
1779	if cMins > 0:
1780	sRet += '%sm ' % (cMins,);
1781	if cSecs > 0:
1782	sRet += '%ss ' % (cSecs,);
1783	assert sRet; assert sRet[-1] == ' ';
1784	return sRet[:-1];
1785
1786	def formatIntervalSeconds2(oSeconds):
1787	"""
1788	Flexible input version of formatIntervalSeconds for use in WUI forms where
1789	data is usually already string form.
1790	"""
1791	if isinstance(oSeconds, (int, long)):
1792	return formatIntervalSeconds(oSeconds);
1793	if not isString(oSeconds):
1794	try:
1795	lSeconds = long(oSeconds);
1796	except:
1797	pass;
1798	else:
1799	if lSeconds >= 0:
1800	return formatIntervalSeconds2(lSeconds);
1801	return oSeconds;
1802
1803	def parseIntervalSeconds(sString):
1804	"""
1805	Reverse of formatIntervalSeconds.
1806
1807	Returns (cSeconds, sError), where sError is None on success.
1808	"""
1809
1810	# We might given non-strings, just return them without any fuss.
1811	if not isString(sString):
1812	if isinstance(sString, (int, long)) or sString is None:
1813	return (sString, None);
1814	## @todo time/date objects?
1815	return (int(sString), None);
1816
1817	# Strip it and make sure it's not empty.
1818	sString = sString.strip();
1819	if not sString:
1820	return (0, 'Empty interval string.');
1821
1822	#
1823	# Split up the input into a list of 'valueN, unitN, ...'.
1824	#
1825	# Don't want to spend too much time trying to make re.split do exactly what
1826	# I need here, so please forgive the extra pass I'm making here.
1827	#
1828	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1829	asParts = [];
1830	for sPart in asRawParts:
1831	sPart = sPart.strip();
1832	if sPart:
1833	asParts.append(sPart);
1834	if not asParts:
1835	return (0, 'Empty interval string or something?');
1836
1837	#
1838	# Process them one or two at the time.
1839	#
1840	cSeconds = 0;
1841	asErrors = [];
1842	i = 0;
1843	while i < len(asParts):
1844	sNumber = asParts[i];
1845	i += 1;
1846	if sNumber.isdigit():
1847	iNumber = int(sNumber);
1848
1849	sUnit = 's';
1850	if i < len(asParts) and not asParts[i].isdigit():
1851	sUnit = asParts[i];
1852	i += 1;
1853
1854	sUnitLower = sUnit.lower();
1855	if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1856	pass;
1857	elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1858	iNumber *= 60;
1859	elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1860	iNumber *= 3600;
1861	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1862	iNumber *= 86400;
1863	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1864	iNumber = 7 86400;
1865	else:
1866	asErrors.append('Unknown unit "%s".' % (sUnit,));
1867	cSeconds += iNumber;
1868	else:
1869	asErrors.append('Bad number "%s".' % (sNumber,));
1870	return (cSeconds, None if not asErrors else ' '.join(asErrors));
1871
1872	def formatIntervalHours(cHours):
1873	""" Format a hours interval into a nice 1w 2d 1h string. """
1874	# Simple special cases.
1875	if cHours < 24:
1876	return '%sh' % (cHours,);
1877
1878	# Generic and a bit slower.
1879	cWeeks = cHours / (7 * 24);
1880	cHours %= 7 * 24;
1881	cDays = cHours / 24;
1882	cHours %= 24;
1883	sRet = '';
1884	if cWeeks > 0:
1885	sRet = '%sw ' % (cWeeks,);
1886	if cDays > 0:
1887	sRet = '%sd ' % (cDays,);
1888	if cHours > 0:
1889	sRet += '%sh ' % (cHours,);
1890	assert sRet; assert sRet[-1] == ' ';
1891	return sRet[:-1];
1892
1893	def parseIntervalHours(sString):
1894	"""
1895	Reverse of formatIntervalHours.
1896
1897	Returns (cHours, sError), where sError is None on success.
1898	"""
1899
1900	# We might given non-strings, just return them without any fuss.
1901	if not isString(sString):
1902	if isinstance(sString, (int, long)) or sString is None:
1903	return (sString, None);
1904	## @todo time/date objects?
1905	return (int(sString), None);
1906
1907	# Strip it and make sure it's not empty.
1908	sString = sString.strip();
1909	if not sString:
1910	return (0, 'Empty interval string.');
1911
1912	#
1913	# Split up the input into a list of 'valueN, unitN, ...'.
1914	#
1915	# Don't want to spend too much time trying to make re.split do exactly what
1916	# I need here, so please forgive the extra pass I'm making here.
1917	#
1918	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1919	asParts = [];
1920	for sPart in asRawParts:
1921	sPart = sPart.strip();
1922	if sPart:
1923	asParts.append(sPart);
1924	if not asParts:
1925	return (0, 'Empty interval string or something?');
1926
1927	#
1928	# Process them one or two at the time.
1929	#
1930	cHours = 0;
1931	asErrors = [];
1932	i = 0;
1933	while i < len(asParts):
1934	sNumber = asParts[i];
1935	i += 1;
1936	if sNumber.isdigit():
1937	iNumber = int(sNumber);
1938
1939	sUnit = 'h';
1940	if i < len(asParts) and not asParts[i].isdigit():
1941	sUnit = asParts[i];
1942	i += 1;
1943
1944	sUnitLower = sUnit.lower();
1945	if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1946	pass;
1947	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1948	iNumber *= 24;
1949	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1950	iNumber = 7 24;
1951	else:
1952	asErrors.append('Unknown unit "%s".' % (sUnit,));
1953	cHours += iNumber;
1954	else:
1955	asErrors.append('Bad number "%s".' % (sNumber,));
1956	return (cHours, None if not asErrors else ' '.join(asErrors));
1957
1958
1959	#
1960	# Introspection.
1961	#
1962
1963	def getCallerName(oFrame=None, iFrame=2):
1964	"""
1965	Returns the name of the caller's caller.
1966	"""
1967	if oFrame is None:
1968	try:
1969	raise Exception();
1970	except:
1971	oFrame = sys.exc_info()[2].tb_frame.f_back;
1972	while iFrame > 1:
1973	if oFrame is not None:
1974	oFrame = oFrame.f_back;
1975	iFrame = iFrame - 1;
1976	if oFrame is not None:
1977	sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1978	return sName;
1979	return "unknown";
1980
1981
1982	def getXcptInfo(cFrames = 1):
1983	"""
1984	Gets text detailing the exception. (Good for logging.)
1985	Returns list of info strings.
1986	"""
1987
1988	#
1989	# Try get exception info.
1990	#
1991	try:
1992	oType, oValue, oTraceback = sys.exc_info();
1993	except:
1994	oType = oValue = oTraceback = None;
1995	if oType is not None:
1996
1997	#
1998	# Try format the info
1999	#
2000	asRet = [];
2001	try:
2002	try:
2003	asRet = asRet + traceback.format_exception_only(oType, oValue);
2004	asTraceBack = traceback.format_tb(oTraceback);
2005	if cFrames is not None and cFrames <= 1:
2006	asRet.append(asTraceBack[-1]);
2007	else:
2008	asRet.append('Traceback:')
2009	for iFrame in range(min(cFrames, len(asTraceBack))):
2010	asRet.append(asTraceBack[-iFrame - 1]);
2011	asRet.append('Stack:')
2012	asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
2013	except:
2014	asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
2015
2016	if not asRet:
2017	asRet.append('No exception info...');
2018	except:
2019	asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
2020	else:
2021	asRet = ['Couldn\'t find exception traceback.'];
2022	return asRet;
2023
2024
2025	def getObjectTypeName(oObject):
2026	"""
2027	Get the type name of the given object.
2028	"""
2029	if oObject is None:
2030	return 'None';
2031
2032	# Get the type object.
2033	try:
2034	oType = type(oObject);
2035	except:
2036	return 'type-throws-exception';
2037
2038	# Python 2.x only: Handle old-style object wrappers.
2039	if sys.version_info[0] < 3:
2040	try:
2041	from types import InstanceType; # pylint: disable=no-name-in-module
2042	if oType == InstanceType:
2043	oType = oObject.__class__;
2044	except:
2045	pass;
2046
2047	# Get the name.
2048	try:
2049	return oType.__name__;
2050	except:
2051	return '__type__-throws-exception';
2052
2053
2054	def chmodPlusX(sFile):
2055	"""
2056	Makes the specified file or directory executable.
2057	Returns success indicator, no exceptions.
2058
2059	Note! Symbolic links are followed and the target will be changed.
2060	"""
2061	try:
2062	oStat = os.stat(sFile);
2063	except:
2064	return False;
2065	try:
2066	os.chmod(sFile, oStat.st_mode \| stat.S_IXUSR \| stat.S_IXGRP \| stat.S_IXOTH);
2067	except:
2068	return False;
2069	return True;
2070
2071
2072	#
2073	# TestSuite stuff.
2074	#
2075
2076	def isRunningFromCheckout(cScriptDepth = 1):
2077	"""
2078	Checks if we're running from the SVN checkout or not.
2079	"""
2080
2081	try:
2082	sFile = __file__;
2083	cScriptDepth = 1;
2084	except:
2085	sFile = sys.argv[0];
2086
2087	sDir = os.path.abspath(sFile);
2088	while cScriptDepth >= 0:
2089	sDir = os.path.dirname(sDir);
2090	if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
2091	or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
2092	return True;
2093	cScriptDepth -= 1;
2094
2095	return False;
2096
2097
2098	#
2099	# Bourne shell argument fun.
2100	#
2101
2102
2103	def argsSplit(sCmdLine):
2104	"""
2105	Given a bourne shell command line invocation, split it up into arguments
2106	assuming IFS is space.
2107	Returns None on syntax error.
2108	"""
2109	## @todo bourne shell argument parsing!
2110	return sCmdLine.split(' ');
2111
2112	def argsGetFirst(sCmdLine):
2113	"""
2114	Given a bourne shell command line invocation, get return the first argument
2115	assuming IFS is space.
2116	Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
2117	"""
2118	asArgs = argsSplit(sCmdLine);
2119	if not asArgs:
2120	return None;
2121
2122	return asArgs[0];
2123
2124	#
2125	# String helpers.
2126	#
2127
2128	def stricmp(sFirst, sSecond):
2129	"""
2130	Compares to strings in an case insensitive fashion.
2131
2132	Python doesn't seem to have any way of doing the correctly, so this is just
2133	an approximation using lower.
2134	"""
2135	if sFirst == sSecond:
2136	return 0;
2137	sLower1 = sFirst.lower();
2138	sLower2 = sSecond.lower();
2139	if sLower1 == sLower2:
2140	return 0;
2141	if sLower1 < sLower2:
2142	return -1;
2143	return 1;
2144
2145
2146	def versionCompare(sVer1, sVer2):
2147	"""
2148	Compares to version strings in a fashion similar to RTStrVersionCompare.
2149	"""
2150
2151	## @todo implement me!!
2152
2153	if sVer1 == sVer2:
2154	return 0;
2155	if sVer1 < sVer2:
2156	return -1;
2157	return 1;
2158
2159
2160	def formatNumber(lNum, sThousandSep = ' '):
2161	"""
2162	Formats a decimal number with pretty separators.
2163	"""
2164	sNum = str(lNum);
2165	sRet = sNum[-3:];
2166	off = len(sNum) - 3;
2167	while off > 0:
2168	off -= 3;
2169	sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
2170	return sRet;
2171
2172
2173	def formatNumberNbsp(lNum):
2174	"""
2175	Formats a decimal number with pretty separators.
2176	"""
2177	sRet = formatNumber(lNum);
2178	return unicode(sRet).replace(' ', u'\u00a0');
2179
2180
2181	def isString(oString):
2182	"""
2183	Checks if the object is a string object, hiding difference between python 2 and 3.
2184
2185	Returns True if it's a string of some kind.
2186	Returns False if not.
2187	"""
2188	if sys.version_info[0] >= 3:
2189	return isinstance(oString, str);
2190	return isinstance(oString, basestring); # pylint: disable=undefined-variable
2191
2192
2193	def hasNonAsciiCharacters(sText):
2194	"""
2195	Returns True is specified string has non-ASCII characters, False if ASCII only.
2196	"""
2197	if isString(sText):
2198	for ch in sText:
2199	if ord(ch) >= 128:
2200	return True;
2201	else:
2202	# Probably byte array or some such thing.
2203	for ch in sText:
2204	if ch >= 128 or ch < 0:
2205	return True;
2206	return False;
2207
2208
2209	#
2210	# Unpacking.
2211	#
2212
2213	def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2214	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2215	"""
2216	Worker for unpackFile that deals with ZIP files, same function signature.
2217	"""
2218	import zipfile
2219	if fnError is None:
2220	fnError = fnLog;
2221
2222	fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2223
2224	# Open it.
2225	try: oZipFile = zipfile.ZipFile(sArchive, 'r'); # pylint: disable=consider-using-with
2226	except Exception as oXcpt:
2227	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2228	return None;
2229
2230	# Extract all members.
2231	asMembers = [];
2232	try:
2233	for sMember in oZipFile.namelist():
2234	if fnFilter is None or fnFilter(sMember) is not False:
2235	if sMember.endswith('/'):
2236	os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2237	else:
2238	oZipFile.extract(sMember, sDstDir);
2239	asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2240	except Exception as oXcpt:
2241	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2242	asMembers = None;
2243
2244	# close it.
2245	try: oZipFile.close();
2246	except Exception as oXcpt:
2247	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2248	asMembers = None;
2249
2250	return asMembers;
2251
2252
2253	## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2254	g_fTarCopyFileObjOverriddend = False;
2255
2256	def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2257	""" tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2258	_ = bufsize;
2259	if length is None:
2260	__myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2261	elif length > 0:
2262	cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2263	for _ in xrange(cFull):
2264	abBuffer = src.read(g_cbGoodBufferSize);
2265	dst.write(abBuffer);
2266	if len(abBuffer) != g_cbGoodBufferSize:
2267	raise exception('unexpected end of source file');
2268	if cbRemainder > 0:
2269	abBuffer = src.read(cbRemainder);
2270	dst.write(abBuffer);
2271	if len(abBuffer) != cbRemainder:
2272	raise exception('unexpected end of source file');
2273
2274
2275	def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2276	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2277	"""
2278	Worker for unpackFile that deals with tarballs, same function signature.
2279	"""
2280	import shutil;
2281	import tarfile;
2282	if fnError is None:
2283	fnError = fnLog;
2284
2285	fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2286
2287	#
2288	# Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2289	# 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2290	# 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2291	#
2292	if True is True: # pylint: disable=comparison-with-itself,comparison-of-constants
2293	__installShUtilHacks(shutil);
2294	global g_fTarCopyFileObjOverriddend;
2295	if g_fTarCopyFileObjOverriddend is False:
2296	g_fTarCopyFileObjOverriddend = True;
2297	#if sys.hexversion < 0x03060000:
2298	tarfile.copyfileobj = __mytarfilecopyfileobj;
2299
2300	#
2301	# Open it.
2302	#
2303	# Note! We not using 'r:*' because we cannot allow seeking compressed files!
2304	# That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2305	#
2306	try:
2307	if sys.hexversion >= 0x03060000:
2308	oTarFile = tarfile.open(sArchive, 'r\|*', # pylint: disable=consider-using-with
2309	bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2310	else:
2311	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize); # pylint: disable=consider-using-with
2312	except Exception as oXcpt:
2313	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2314	return None;
2315
2316	# Extract all members.
2317	asMembers = [];
2318	try:
2319	for oTarInfo in oTarFile:
2320	try:
2321	if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2322	if oTarInfo.islnk():
2323	# Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2324	# in the compressed tar stream. So, fall back on shutil.copy2 instead.
2325	sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2326	sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2327	sParentDir = os.path.dirname(sLinkFile);
2328	try: os.unlink(sLinkFile);
2329	except: pass;
2330	if sParentDir and not os.path.exists(sParentDir):
2331	os.makedirs(sParentDir);
2332	try: os.link(sLinkTarget, sLinkFile);
2333	except: shutil.copy2(sLinkTarget, sLinkFile);
2334	else:
2335	if oTarInfo.isdir():
2336	# Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2337	oTarInfo.mode \|= 0x1c0; # (octal: 0700)
2338	if hasattr(tarfile, 'tar_filter'):
2339	oTarFile.extract(oTarInfo, sDstDir, filter = 'tar');
2340	else:
2341	oTarFile.extract(oTarInfo, sDstDir);
2342	asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2343	except Exception as oXcpt:
2344	fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2345	for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2346	fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2347	for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2348	fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2349	asMembers = None;
2350	break;
2351	except Exception as oXcpt:
2352	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2353	asMembers = None;
2354
2355	#
2356	# Finally, close it.
2357	#
2358	try: oTarFile.close();
2359	except Exception as oXcpt:
2360	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2361	asMembers = None;
2362
2363	return asMembers;
2364
2365
2366	def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2367	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2368	"""
2369	Unpacks the given file if it has a know archive extension, otherwise do
2370	nothing.
2371
2372	fnLog & fnError both take a string parameter.
2373
2374	fnFilter takes a member name (string) and returns True if it's included
2375	and False if excluded.
2376
2377	Returns list of the extracted files (full path) on success.
2378	Returns empty list if not a supported archive format.
2379	Returns None on failure. Raises no exceptions.
2380	"""
2381	sBaseNameLower = os.path.basename(sArchive).lower();
2382
2383	#
2384	# Zip file?
2385	#
2386	if sBaseNameLower.endswith('.zip'):
2387	return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2388
2389	#
2390	# Tarball?
2391	#
2392	if sBaseNameLower.endswith('.tar') \
2393	or sBaseNameLower.endswith('.tar.gz') \
2394	or sBaseNameLower.endswith('.tgz') \
2395	or sBaseNameLower.endswith('.tar.bz2'):
2396	return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2397
2398	#
2399	# Cannot classify it from the name, so just return that to the caller.
2400	#
2401	fnLog('Not unpacking "%s".' % (sArchive,));
2402	return [];
2403
2404
2405	#
2406	# Misc.
2407	#
2408	def areBytesEqual(oLeft, oRight):
2409	"""
2410	Compares two byte arrays, strings or whatnot.
2411
2412	returns true / false accordingly.
2413	"""
2414
2415	# If both are None, consider them equal (bogus?):
2416	if oLeft is None and oRight is None:
2417	return True;
2418
2419	# If just one is None, they can't match:
2420	if oLeft is None or oRight is None:
2421	return False;
2422
2423	# If both have the same type, use the compare operator of the class:
2424	if type(oLeft) is type(oRight):
2425	#print('same type: %s' % (oLeft == oRight,));
2426	return oLeft == oRight;
2427
2428	# On the offchance that they're both strings, but of different types.
2429	if isString(oLeft) and isString(oRight):
2430	#print('string compare: %s' % (oLeft == oRight,));
2431	return oLeft == oRight;
2432
2433	#
2434	# See if byte/buffer stuff that can be compared directory. If not convert
2435	# strings to bytes.
2436	#
2437	# Note! For 2.x, we must convert both sides to the buffer type or the
2438	# comparison may fail despite it working okay in test cases.
2439	#
2440	if sys.version_info[0] >= 3:
2441	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2442	return oLeft == oRight;
2443
2444	if isString(oLeft):
2445	try: oLeft = bytes(oLeft, 'utf-8');
2446	except: pass;
2447	if isString(oRight):
2448	try: oRight = bytes(oRight, 'utf-8');
2449	except: pass;
2450	else:
2451	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2452	if isinstance(oLeft, bytearray):
2453	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2454	else:
2455	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2456	#print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2457	return oLeft == oRight;
2458
2459	if isString(oLeft):
2460	try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2461	except: pass;
2462	if isString(oRight):
2463	try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2464	except: pass;
2465
2466	# Check if we now have the same type for both:
2467	if type(oLeft) is type(oRight):
2468	#print('same type now: %s' % (oLeft == oRight,));
2469	return oLeft == oRight;
2470
2471	# Check if we now have buffer/memoryview vs bytes/bytesarray again.
2472	if sys.version_info[0] >= 3:
2473	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2474	return oLeft == oRight;
2475	else:
2476	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2477	if isinstance(oLeft, bytearray):
2478	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2479	else:
2480	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2481	#print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2482	return oLeft == oRight;
2483
2484	# Do item by item comparison:
2485	if len(oLeft) != len(oRight):
2486	#print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2487	return False;
2488	i = len(oLeft);
2489	while i > 0:
2490	i = i - 1;
2491
2492	iElmLeft = oLeft[i];
2493	if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2494	iElmLeft = ord(iElmLeft);
2495
2496	iElmRight = oRight[i];
2497	if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2498	iElmRight = ord(iElmRight);
2499
2500	if iElmLeft != iElmRight:
2501	#print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2502	return False;
2503	return True;
2504
2505
2506	def calcCrc32OfFile(sFile):
2507	"""
2508	Simple helper for calculating the CRC32 of a file.
2509
2510	Throws stuff if the file cannot be opened or read successfully.
2511	"""
2512	import zlib;
2513
2514	uCrc32 = 0;
2515	with open(sFile, 'rb') as oFile: # pylint: disable=unspecified-encoding
2516	while True:
2517	oBuf = oFile.read(1024 * 1024);
2518	if not oBuf:
2519	break
2520	uCrc32 = zlib.crc32(oBuf, uCrc32);
2521
2522	return uCrc32 % 2**32;
2523
2524
2525	#
2526	# Unit testing.
2527	#
2528
2529	# pylint: disable=missing-docstring
2530	# pylint: disable=undefined-variable
2531	class BuildCategoryDataTestCase(unittest.TestCase):
2532	def testIntervalSeconds(self):
2533	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2534	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2535	self.assertEqual(parseIntervalSeconds('123'), (123, None));
2536	self.assertEqual(parseIntervalSeconds(123), (123, None));
2537	self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2538	self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2539	self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2540	self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2541	self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2542	self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2543	self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2544
2545	def testZuluNormalization(self):
2546	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2547	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2548	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2549	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2550	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2551
2552	def testHasNonAsciiChars(self):
2553	self.assertEqual(hasNonAsciiCharacters(''), False);
2554	self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB&$%&)@#(!)@(#!(#$&#$&%*Y@#$IQWN---00;'), False);
2555	self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2556	self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2557	self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2558	self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2559	self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2560	self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2561	self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2562	self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2563
2564	def testAreBytesEqual(self):
2565	self.assertEqual(areBytesEqual(None, None), True);
2566	self.assertEqual(areBytesEqual(None, ''), False);
2567	self.assertEqual(areBytesEqual('', ''), True);
2568	self.assertEqual(areBytesEqual('1', '1'), True);
2569	self.assertEqual(areBytesEqual('12345', '1234'), False);
2570	self.assertEqual(areBytesEqual('1234', '1234'), True);
2571	self.assertEqual(areBytesEqual('1234', b'1234'), True);
2572	self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2573	self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2574	self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2575	self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2576	self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2577	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2578	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2579	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2580	if sys.version_info[0] >= 3:
2581	pass;
2582	else:
2583	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2584	bytearray([0x31,0x32,0x33,0x34])), True);
2585	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2586	bytearray([0x99,0x32,0x32,0x34])), False);
2587	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2588	buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2589	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2590	buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2591	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2592	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2593	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2594
2595	if __name__ == '__main__':
2596	unittest.main();
2597	# not reached.

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 106199

Download in other formats: