utils.py@ 107926

Last change on this file since 107926 was 107926, checked in by vboxsync, 2 months ago
ValidationKit/common/utils.py: Print the actual codename of macOS 15
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 90.0 KB

Line
1	# -- coding: utf-8 --
2	# $Id: utils.py 107926 2025-01-23 19:04:53Z vboxsync $
3	# pylint: disable=too-many-lines
4
5	"""
6	Common Utility Functions.
7	"""
8
9	from __future__ import print_function;
10
11	__copyright__ = \
12	"""
13	Copyright (C) 2012-2024 Oracle and/or its affiliates.
14
15	This file is part of VirtualBox base platform packages, as
16	available from https://www.virtualbox.org.
17
18	This program is free software; you can redistribute it and/or
19	modify it under the terms of the GNU General Public License
20	as published by the Free Software Foundation, in version 3 of the
21	License.
22
23	This program is distributed in the hope that it will be useful, but
24	WITHOUT ANY WARRANTY; without even the implied warranty of
25	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
26	General Public License for more details.
27
28	You should have received a copy of the GNU General Public License
29	along with this program; if not, see <https://www.gnu.org/licenses>.
30
31	The contents of this file may alternatively be used under the terms
32	of the Common Development and Distribution License Version 1.0
33	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
34	in the VirtualBox distribution, in which case the provisions of the
35	CDDL are applicable instead of those of the GPL.
36
37	You may elect to license modified versions of this file under the
38	terms and conditions of either the GPL or the CDDL or both.
39
40	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
41	"""
42	__version__ = "$Revision: 107926 $"
43
44
45	# Standard Python imports.
46	import datetime;
47	import errno;
48	import os;
49	import platform;
50	import re;
51	import stat;
52	import subprocess;
53	import sys;
54	import time;
55	import traceback;
56	import unittest;
57
58	if sys.platform == 'win32':
59	import ctypes;
60	import msvcrt; # pylint: disable=import-error
61	import win32api; # pylint: disable=import-error
62	import win32con; # pylint: disable=import-error
63	import win32console; # pylint: disable=import-error
64	import win32file; # pylint: disable=import-error
65	import win32process; # pylint: disable=import-error
66	import winerror; # pylint: disable=import-error
67	import pywintypes; # pylint: disable=import-error
68	else:
69	import signal;
70
71	# Python 3 hacks:
72	if sys.version_info[0] >= 3:
73	unicode = str; # pylint: disable=redefined-builtin,invalid-name
74	xrange = range; # pylint: disable=redefined-builtin,invalid-name
75	long = int; # pylint: disable=redefined-builtin,invalid-name
76
77
78	#
79	# Python 2/3 glue.
80	#
81
82	if sys.version_info[0] >= 3:
83	def iteritems(dDict):
84	"""
85	Wrapper around dict.items() / dict.iteritems().
86	"""
87	return iter(dDict.items());
88	else:
89	def iteritems(dDict):
90	"""
91	Wrapper around dict.items() / dict.iteritems().
92	"""
93	return dDict.iteritems();
94
95
96	#
97	# Strings.
98	#
99
100	def toUnicode(sString, encoding = None, errors = 'strict'):
101	"""
102	A little like the python 2 unicode() function.
103	"""
104	if sys.version_info[0] >= 3:
105	if isinstance(sString, bytes):
106	return str(sString, encoding if encoding else 'utf-8', errors);
107	else:
108	if not isinstance(sString, unicode):
109	return unicode(sString, encoding if encoding else 'utf-8', errors);
110	return sString;
111
112
113
114	#
115	# Output.
116	#
117
118	def printOut(sString):
119	"""
120	Outputs a string to standard output, dealing with python 2.x encoding stupidity.
121	"""
122	sStreamEncoding = sys.stdout.encoding;
123	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
124	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
125	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
126	print(sString);
127	else:
128	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
129
130	def printErr(sString):
131	"""
132	Outputs a string to standard error, dealing with python 2.x encoding stupidity.
133	"""
134	sStreamEncoding = sys.stderr.encoding;
135	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
136	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
137	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
138	print(sString, file = sys.stderr);
139	else:
140	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
141
142
143	#
144	# Host OS and CPU.
145	#
146
147	def getHostOs():
148	"""
149	Gets the host OS name (short).
150
151	See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
152	"""
153	sPlatform = platform.system();
154	if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
155	sPlatform = sPlatform.lower();
156	elif sPlatform == 'Windows':
157	sPlatform = 'win';
158	elif sPlatform == 'SunOS':
159	sPlatform = 'solaris';
160	else:
161	raise Exception('Unsupported platform "%s"' % (sPlatform,));
162	return sPlatform;
163
164	g_sHostArch = None;
165
166	def getHostArch():
167	"""
168	Gets the host CPU architecture.
169
170	See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
171	"""
172	global g_sHostArch;
173	if g_sHostArch is None:
174	sArch = platform.machine();
175	if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
176	sArch = 'x86';
177	elif sArch in ('AMD64', 'amd64', 'x86_64'):
178	sArch = 'amd64';
179	elif sArch == 'i86pc': # SunOS
180	if platform.architecture()[0] == '64bit':
181	sArch = 'amd64';
182	else:
183	try:
184	sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
185	except:
186	pass;
187	sArch = sArch.strip();
188	if sArch != 'amd64':
189	sArch = 'x86';
190	elif sArch in ('arm64', 'ARM64', 'aarch64'):
191	sArch = 'arm64';
192	else:
193	raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
194	g_sHostArch = sArch;
195	return g_sHostArch;
196
197
198	def getHostOsDotArch():
199	"""
200	Gets the 'os.arch' for the host.
201	"""
202	return '%s.%s' % (getHostOs(), getHostArch());
203
204
205	def isValidOs(sOs):
206	"""
207	Validates the OS name.
208	"""
209	if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
210	'os2', 'solaris', 'win', 'os-agnostic'):
211	return True;
212	return False;
213
214
215	def isValidArch(sArch):
216	"""
217	Validates the CPU architecture name.
218	"""
219	if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
220	'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'arm64', 'alpha'):
221	return True;
222	return False;
223
224	def isValidOsDotArch(sOsDotArch):
225	"""
226	Validates the 'os.arch' string.
227	"""
228
229	asParts = sOsDotArch.split('.');
230	if asParts.length() != 2:
231	return False;
232	return isValidOs(asParts[0]) \
233	and isValidArch(asParts[1]);
234
235	def getHostOsVersion():
236	"""
237	Returns the host OS version. This is platform.release with additional
238	distro indicator on linux.
239	"""
240	sVersion = platform.release();
241	sOs = getHostOs();
242	if sOs == 'linux':
243	sDist = '';
244	try:
245	# try /etc/lsb-release first to distinguish between Debian and Ubuntu
246	with open('/etc/lsb-release') as oFile: # pylint: disable=unspecified-encoding
247	for sLine in oFile:
248	oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s=)\s"(.)"', sLine);
249	if oMatch is not None:
250	sDist = oMatch.group(1).strip();
251	except:
252	pass;
253	if sDist:
254	sVersion += ' / ' + sDist;
255	else:
256	asFiles = \
257	[
258	[ '/etc/debian_version', 'Debian v'],
259	[ '/etc/gentoo-release', '' ],
260	[ '/etc/oracle-release', '' ],
261	[ '/etc/redhat-release', '' ],
262	[ '/etc/SuSE-release', '' ],
263	];
264	for sFile, sPrefix in asFiles:
265	if os.path.isfile(sFile):
266	try:
267	with open(sFile) as oFile: # pylint: disable=unspecified-encoding
268	sLine = oFile.readline();
269	except:
270	continue;
271	sLine = sLine.strip()
272	if sLine:
273	sVersion += ' / ' + sPrefix + sLine;
274	break;
275
276	elif sOs == 'solaris':
277	sVersion = platform.version();
278	if os.path.isfile('/etc/release'):
279	try:
280	with open('/etc/release') as oFile: # pylint: disable=unspecified-encoding
281	sLast = oFile.readlines()[-1];
282	sLast = sLast.strip();
283	if sLast:
284	sVersion += ' (' + sLast + ')';
285	except:
286	pass;
287
288	elif sOs == 'darwin':
289	def getMacVersionName(sVersion):
290	"""
291	Figures out the Mac OS X/macOS code name from the numeric version.
292	"""
293	aOsVersion = sVersion.split('.') # example: ('10','15','7')
294	codenames = {"4": "Tiger",
295	"5": "Leopard",
296	"6": "Snow Leopard",
297	"7": "Lion",
298	"8": "Mountain Lion",
299	"9": "Mavericks",
300	"10": "Yosemite",
301	"11": "El Capitan",
302	"12": "Sierra",
303	"13": "High Sierra",
304	"14": "Mojave",
305	"15": "Catalina",
306	"16": "Wrong version",
307	}
308	codenames_afterCatalina = {"11": "Big Sur",
309	"12": "Monterey",
310	"13": "Ventura",
311	"14": "Sonoma",
312	"15": "Sequoia",
313	"16": "Unknown 16",
314	"17": "Unknown 17"}
315
316	if aOsVersion[0] == '10':
317	sResult = codenames[aOsVersion[1]]
318	else:
319	sResult = codenames_afterCatalina[aOsVersion[0]]
320	return sResult
321
322	sOsxVersion = platform.mac_ver()[0]
323	sVersion += ' / OS X ' + sOsxVersion + ' (' + getMacVersionName(sOsxVersion) + ')'
324
325	elif sOs == 'win':
326	class OSVersionInfoEx(ctypes.Structure): # pylint: disable=used-before-assignment
327	""" OSVERSIONEX """
328	kaFields = [
329	('dwOSVersionInfoSize', ctypes.c_ulong),
330	('dwMajorVersion', ctypes.c_ulong),
331	('dwMinorVersion', ctypes.c_ulong),
332	('dwBuildNumber', ctypes.c_ulong),
333	('dwPlatformId', ctypes.c_ulong),
334	('szCSDVersion', ctypes.c_wchar*128),
335	('wServicePackMajor', ctypes.c_ushort),
336	('wServicePackMinor', ctypes.c_ushort),
337	('wSuiteMask', ctypes.c_ushort),
338	('wProductType', ctypes.c_byte),
339	('wReserved', ctypes.c_byte)]
340	_fields_ = kaFields # pylint: disable=invalid-name
341
342	def __init__(self):
343	super(OSVersionInfoEx, self).__init__()
344	self.dwOSVersionInfoSize = ctypes.sizeof(self)
345
346	oOsVersion = OSVersionInfoEx()
347	rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
348	if rc == 0:
349	# Python platform.release() is not reliable for newer server releases
350	if oOsVersion.wProductType != 1:
351	if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0 \
352	and oOsVersion.dwBuildNumber == 17763:
353	sVersion = '2019Server'
354	elif oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
355	sVersion = '2016Server'; #todo: should probably add dwBuildNumber for it as well..
356	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
357	sVersion = '2012ServerR2';
358	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
359	sVersion = '2012Server';
360	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
361	sVersion = '2008ServerR2';
362	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
363	sVersion = '2008Server';
364	elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
365	sVersion = '2003Server';
366	sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
367	if oOsVersion.wServicePackMajor:
368	sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
369	if oOsVersion.wServicePackMinor:
370	sVersion += '.' + str(oOsVersion.wServicePackMinor)
371
372	return sVersion;
373
374	def getPresentCpuCount():
375	"""
376	Gets the number of CPUs present in the system.
377
378	This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
379	that we return the active count rather than the maximum count. If we don't,
380	we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
381	never will have.
382
383	@todo This is probably not exactly what we get on non-windows...
384	"""
385
386	if getHostOs() == 'win':
387	fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
388	if fnGetActiveProcessorCount:
389	cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
390	if cCpus > 0:
391	return cCpus;
392
393	import multiprocessing
394	return multiprocessing.cpu_count();
395
396
397	#
398	# File system.
399	#
400
401	def openNoInherit(sFile, sMode = 'r'):
402	"""
403	Wrapper around open() that tries it's best to make sure the file isn't
404	inherited by child processes.
405
406	This is a best effort thing at the moment as it doesn't synchronizes with
407	child process spawning in any way. Thus it can be subject to races in
408	multithreaded programs.
409	"""
410
411	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
412	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
413	if uPythonVer >= ((3 << 16) \| 4):
414	oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
415	else:
416	try:
417	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
418	except:
419	# On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
420	if getHostOs() == 'win':
421	if uPythonVer < (3 << 16):
422	offComma = sMode.find(',');
423	if offComma < 0:
424	return open(sFile, sMode + 'N'); # pylint: disable=consider-using-with,unspecified-encoding
425	return open(sFile, # pylint: disable=consider-using-with,unspecified-encoding,bad-open-mode
426	sMode[:offComma] + 'N' + sMode[offComma:]);
427
428	# Just in case.
429	return open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
430
431	oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
432	#try:
433	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
434	#except:
435	# pass;
436	return oFile;
437
438	def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
439	"""
440	Wrapper around open() that tries it's best to make sure the file isn't
441	inherited by child processes.
442
443	This is a best effort thing at the moment as it doesn't synchronizes with
444	child process spawning in any way. Thus it can be subject to races in
445	multithreaded programs.
446	"""
447
448	if getHostOs() == 'win':
449	# Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
450	# pylint: disable=no-member,c-extension-no-member
451	fAccess = 0;
452	fDisposition = win32file.OPEN_EXISTING; # pylint: disable=used-before-assignment
453	if 'r' in sMode or '+' in sMode:
454	fAccess \|= win32file.GENERIC_READ;
455	if 'a' in sMode:
456	fAccess \|= win32file.GENERIC_WRITE;
457	fDisposition = win32file.OPEN_ALWAYS;
458	elif 'w' in sMode:
459	fAccess = win32file.GENERIC_WRITE;
460	if '+' in sMode:
461	fDisposition = win32file.OPEN_ALWAYS;
462	fAccess \|= win32file.GENERIC_READ;
463	else:
464	fDisposition = win32file.CREATE_ALWAYS;
465	if not fAccess:
466	fAccess \|= win32file.GENERIC_READ;
467	fSharing = (win32file.FILE_SHARE_READ \| win32file.FILE_SHARE_WRITE
468	\| win32file.FILE_SHARE_DELETE);
469	hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
470	if 'a' in sMode:
471	win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
472
473	# Turn the NT handle into a CRT file descriptor.
474	hDetachedFile = hFile.Detach();
475	if fAccess == win32file.GENERIC_READ:
476	fOpen = os.O_RDONLY;
477	elif fAccess == win32file.GENERIC_WRITE:
478	fOpen = os.O_WRONLY;
479	else:
480	fOpen = os.O_RDWR;
481	# pulint: enable=no-member,c-extension-no-member
482	if 'a' in sMode:
483	fOpen \|= os.O_APPEND;
484	if 'b' in sMode or 't' in sMode:
485	fOpen \|= os.O_TEXT; # pylint: disable=no-member
486	fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen); # pylint: disable=used-before-assignment
487
488	# Tell python to use this handle.
489	oFile = os.fdopen(fdFile, sMode);
490	else:
491	oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding
492
493	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
494	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
495	if uPythonVer < ((3 << 16) \| 4):
496	try:
497	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
498	except:
499	pass;
500	else:
501	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
502	return oFile;
503
504	def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
505	"""
506	No exceptions os.readlink wrapper.
507	"""
508	try:
509	sRet = os.readlink(sPath); # pylint: disable=no-member
510	except:
511	return sXcptRet;
512	if hasattr(sRet, 'decode'):
513	sRet = sRet.decode(sEncoding, 'ignore');
514	return sRet;
515
516	def readFile(sFile, sMode = 'rb'):
517	"""
518	Reads the entire file.
519	"""
520	with open(sFile, sMode) as oFile: # pylint: disable=unspecified-encoding
521	sRet = oFile.read();
522	return sRet;
523
524	def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
525	"""
526	No exceptions common.readFile wrapper.
527	"""
528	try:
529	sRet = readFile(sFile, sMode);
530	except:
531	sRet = sXcptRet;
532	if sEncoding is not None and hasattr(sRet, 'decode'):
533	sRet = sRet.decode(sEncoding, 'ignore');
534	return sRet;
535
536	def noxcptRmDir(sDir, oXcptRet = False):
537	"""
538	No exceptions os.rmdir wrapper.
539	"""
540	oRet = True;
541	try:
542	os.rmdir(sDir);
543	except:
544	oRet = oXcptRet;
545	return oRet;
546
547	def noxcptDeleteFile(sFile, oXcptRet = False):
548	"""
549	No exceptions os.remove wrapper.
550	"""
551	oRet = True;
552	try:
553	os.remove(sFile);
554	except:
555	oRet = oXcptRet;
556	return oRet;
557
558
559	def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
560	# type: (string, (string, stat) -> bool) -> bool
561	"""
562	Recursively walks a directory tree, calling fnCallback for each.
563
564	fnCallback takes a full path and stat object (can be None). It
565	returns a boolean value, False stops walking and returns immediately.
566
567	Returns True or False depending on fnCallback.
568	Returns None fIgnoreExceptions is True and an exception was raised by listdir.
569	"""
570	def __worker(sCurDir):
571	""" Worker for """
572	try:
573	asNames = os.listdir(sCurDir);
574	except:
575	if not fIgnoreExceptions:
576	raise;
577	return None;
578	rc = True;
579	for sName in asNames:
580	if sName not in [ '.', '..' ]:
581	sFullName = os.path.join(sCurDir, sName);
582	try: oStat = os.lstat(sFullName);
583	except: oStat = None;
584	if fnCallback(sFullName, oStat) is False:
585	return False;
586	if oStat is not None and stat.S_ISDIR(oStat.st_mode):
587	rc = __worker(sFullName);
588	if rc is False:
589	break;
590	return rc;
591
592	# Ensure unicode path here so listdir also returns unicode on windows.
593	## @todo figure out unicode stuff on non-windows.
594	if sys.platform == 'win32':
595	sDir = unicode(sDir);
596	return __worker(sDir);
597
598
599
600	def formatFileMode(uMode):
601	# type: (int) -> string
602	"""
603	Format a st_mode value 'ls -la' fasion.
604	Returns string.
605	"""
606	if stat.S_ISDIR(uMode): sMode = 'd';
607	elif stat.S_ISREG(uMode): sMode = '-';
608	elif stat.S_ISLNK(uMode): sMode = 'l';
609	elif stat.S_ISFIFO(uMode): sMode = 'p';
610	elif stat.S_ISCHR(uMode): sMode = 'c';
611	elif stat.S_ISBLK(uMode): sMode = 'b';
612	elif stat.S_ISSOCK(uMode): sMode = 's';
613	else: sMode = '?';
614	## @todo sticky bits.
615	sMode += 'r' if uMode & stat.S_IRUSR else '-';
616	sMode += 'w' if uMode & stat.S_IWUSR else '-';
617	sMode += 'x' if uMode & stat.S_IXUSR else '-';
618	sMode += 'r' if uMode & stat.S_IRGRP else '-';
619	sMode += 'w' if uMode & stat.S_IWGRP else '-';
620	sMode += 'x' if uMode & stat.S_IXGRP else '-';
621	sMode += 'r' if uMode & stat.S_IROTH else '-';
622	sMode += 'w' if uMode & stat.S_IWOTH else '-';
623	sMode += 'x' if uMode & stat.S_IXOTH else '-';
624	sMode += ' ';
625	return sMode;
626
627
628	def formatFileStat(oStat):
629	# type: (stat) -> string
630	"""
631	Format a stat result 'ls -la' fasion (numeric IDs).
632	Returns string.
633	"""
634	return '%s %3s %4s %4s %10s %s' \
635	% (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
636	time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
637
638	## Good buffer for file operations.
639	g_cbGoodBufferSize = 256*1024;
640
641	## The original shutil.copyfileobj.
642	g_fnOriginalShCopyFileObj = None;
643
644	def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
645	""" shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
646	return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
647
648	def __installShUtilHacks(shutil):
649	""" Installs the shutil buffer size hacks. """
650	global g_fnOriginalShCopyFileObj;
651	if g_fnOriginalShCopyFileObj is None:
652	g_fnOriginalShCopyFileObj = shutil.copyfileobj;
653	shutil.copyfileobj = __myshutilcopyfileobj;
654	return True;
655
656
657	def copyFileSimple(sFileSrc, sFileDst):
658	"""
659	Wrapper around shutil.copyfile that simply copies the data of a regular file.
660	Raises exception on failure.
661	Return True for show.
662	"""
663	import shutil;
664	__installShUtilHacks(shutil);
665	return shutil.copyfile(sFileSrc, sFileDst);
666
667
668	def getDiskUsage(sPath):
669	"""
670	Get free space of a partition that corresponds to specified sPath in MB.
671
672	Returns partition free space value in MB.
673	"""
674	if platform.system() == 'Windows':
675	oCTypeFreeSpace = ctypes.c_ulonglong(0);
676	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
677	ctypes.pointer(oCTypeFreeSpace));
678	cbFreeSpace = oCTypeFreeSpace.value;
679	else:
680	oStats = os.statvfs(sPath); # pylint: disable=no-member
681	cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
682
683	# Convert to MB
684	cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
685
686	return cMbFreeSpace;
687
688
689
690	#
691	# SubProcess.
692	#
693
694	def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
695	"""
696	If the "executable" is a python script, insert the python interpreter at
697	the head of the argument list so that it will work on systems which doesn't
698	support hash-bang scripts.
699	"""
700
701	asArgs = dKeywordArgs.get('args');
702	if asArgs is None:
703	asArgs = aPositionalArgs[0];
704
705	if asArgs[0].endswith('.py'):
706	if sys.executable:
707	asArgs.insert(0, sys.executable);
708	else:
709	asArgs.insert(0, 'python');
710
711	# paranoia...
712	if dKeywordArgs.get('args') is not None:
713	dKeywordArgs['args'] = asArgs;
714	else:
715	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
716	return None;
717
718	def processPopenSafe(aPositionalArgs, *dKeywordArgs):
719	"""
720	Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
721	"""
722	if getHostOs() == 'win':
723	if dKeywordArgs.get('creationflags', 0) == 0:
724	dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
725	return subprocess.Popen(aPositionalArgs, *dKeywordArgs); # pylint: disable=consider-using-with
726
727	def processStart(aPositionalArgs, *dKeywordArgs):
728	"""
729	Wrapper around subprocess.Popen to deal with its absence in older
730	python versions.
731	Returns process object on success which can be worked on.
732	"""
733	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
734	return processPopenSafe(aPositionalArgs, *dKeywordArgs);
735
736	def processCall(aPositionalArgs, *dKeywordArgs):
737	"""
738	Wrapper around subprocess.call to deal with its absence in older
739	python versions.
740	Returns process exit code (see subprocess.poll).
741	"""
742	assert dKeywordArgs.get('stdout') is None;
743	assert dKeywordArgs.get('stderr') is None;
744	oProcess = processStart(aPositionalArgs, *dKeywordArgs);
745	return oProcess.wait();
746
747	def processOutputChecked(aPositionalArgs, *dKeywordArgs):
748	"""
749	Wrapper around subprocess.check_output to deal with its absense in older
750	python versions.
751
752	Extra keywords for specifying now output is to be decoded:
753	sEncoding='utf-8
754	fIgnoreEncoding=True/False
755	"""
756	sEncoding = dKeywordArgs.get('sEncoding');
757	if sEncoding is not None: del dKeywordArgs['sEncoding'];
758	else: sEncoding = 'utf-8';
759
760	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
761	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
762	else: fIgnoreEncoding = True;
763
764	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
765	oProcess = processPopenSafe(stdout=subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
766
767	sOutput, _ = oProcess.communicate();
768	iExitCode = oProcess.poll();
769
770	if iExitCode != 0:
771	asArgs = dKeywordArgs.get('args');
772	if asArgs is None:
773	asArgs = aPositionalArgs[0];
774	print(sOutput);
775	raise subprocess.CalledProcessError(iExitCode, asArgs);
776
777	if hasattr(sOutput, 'decode'):
778	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
779	return sOutput;
780
781	def processOutputUnchecked(aPositionalArgs, *dKeywordArgs):
782	"""
783	Similar to processOutputChecked, but returns status code and both stdout
784	and stderr results.
785
786	Extra keywords for specifying now output is to be decoded:
787	sEncoding='utf-8
788	fIgnoreEncoding=True/False
789	"""
790	sEncoding = dKeywordArgs.get('sEncoding');
791	if sEncoding is not None: del dKeywordArgs['sEncoding'];
792	else: sEncoding = 'utf-8';
793
794	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
795	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
796	else: fIgnoreEncoding = True;
797
798	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
799	oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
800
801	sOutput, sError = oProcess.communicate();
802	iExitCode = oProcess.poll();
803
804	if hasattr(sOutput, 'decode'):
805	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
806	if hasattr(sError, 'decode'):
807	sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
808	return (iExitCode, sOutput, sError);
809
810	g_fOldSudo = None;
811	def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
812	"""
813	Adds 'sudo' (or similar) to the args parameter, whereever it is.
814	"""
815
816	# Are we root?
817	fIsRoot = True;
818	try:
819	fIsRoot = os.getuid() == 0; # pylint: disable=no-member
820	except:
821	pass;
822
823	# If not, prepend sudo (non-interactive, simulate initial login).
824	if fIsRoot is not True:
825	asArgs = dKeywordArgs.get('args');
826	if asArgs is None:
827	asArgs = aPositionalArgs[0];
828
829	# Detect old sudo.
830	global g_fOldSudo;
831	if g_fOldSudo is None:
832	try:
833	sVersion = str(processOutputChecked(['sudo', '-V']));
834	except:
835	sVersion = '1.7.0';
836	sVersion = sVersion.strip().split('\n', 1)[0];
837	sVersion = sVersion.replace('Sudo version', '').strip();
838	g_fOldSudo = len(sVersion) >= 4 \
839	and sVersion[0] == '1' \
840	and sVersion[1] == '.' \
841	and sVersion[2] <= '6' \
842	and sVersion[3] == '.';
843
844	asArgs.insert(0, 'sudo');
845	if not g_fOldSudo:
846	asArgs.insert(1, '-n');
847	if fInitialEnv and not g_fOldSudo:
848	asArgs.insert(1, '-i');
849
850	# paranoia...
851	if dKeywordArgs.get('args') is not None:
852	dKeywordArgs['args'] = asArgs;
853	else:
854	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
855	return None;
856
857
858	def sudoProcessStart(aPositionalArgs, *dKeywordArgs):
859	"""
860	sudo (or similar) + subprocess.Popen,
861	returning the process object on success.
862	"""
863	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
864	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
865	return processStart(aPositionalArgs, *dKeywordArgs);
866
867	def sudoProcessCall(aPositionalArgs, *dKeywordArgs):
868	"""
869	sudo (or similar) + subprocess.call
870	"""
871	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
872	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
873	return processCall(aPositionalArgs, *dKeywordArgs);
874
875	def sudoProcessOutputChecked(aPositionalArgs, *dKeywordArgs):
876	"""
877	sudo (or similar) + subprocess.check_output.
878	"""
879	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
880	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
881	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
882
883	def sudoProcessOutputCheckedNoI(aPositionalArgs, *dKeywordArgs):
884	"""
885	sudo (or similar) + subprocess.check_output, except '-i' isn't used.
886	"""
887	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
888	_sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
889	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
890
891	def sudoProcessPopen(aPositionalArgs, *dKeywordArgs):
892	"""
893	sudo (or similar) + processPopenSafe.
894	"""
895	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
896	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
897	return processPopenSafe(aPositionalArgs, *dKeywordArgs);
898
899
900	def whichProgram(sName, sPath = None):
901	"""
902	Works similar to the 'which' utility on unix.
903
904	Returns path to the given program if found.
905	Returns None if not found.
906	"""
907	sHost = getHostOs();
908	sSep = ';' if sHost in [ 'win', 'os2' ] else ':';
909
910	if sPath is None:
911	if sHost == 'win':
912	sPath = os.environ.get('Path', None);
913	else:
914	sPath = os.environ.get('PATH', None);
915	if sPath is None:
916	return None;
917
918	for sDir in sPath.split(sSep):
919	if sDir.strip() != '':
920	sTest = os.path.abspath(os.path.join(sDir, sName));
921	else:
922	sTest = os.path.abspath(sName);
923	if os.path.exists(sTest):
924	return sTest;
925
926	return None;
927
928	#
929	# Generic process stuff.
930	#
931
932	def processInterrupt(uPid):
933	"""
934	Sends a SIGINT or equivalent to interrupt the specified process.
935	Returns True on success, False on failure.
936
937	On Windows hosts this may not work unless the process happens to be a
938	process group leader.
939	"""
940	if sys.platform == 'win32':
941	try:
942	win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
943	uPid);
944	fRc = True;
945	except:
946	fRc = False;
947	else:
948	try:
949	os.kill(uPid, signal.SIGINT);
950	fRc = True;
951	except:
952	fRc = False;
953	return fRc;
954
955	def sendUserSignal1(uPid):
956	"""
957	Sends a SIGUSR1 or equivalent to nudge the process into shutting down
958	(VBoxSVC) or something.
959	Returns True on success, False on failure or if not supported (win).
960
961	On Windows hosts this may not work unless the process happens to be a
962	process group leader.
963	"""
964	if sys.platform == 'win32':
965	fRc = False;
966	else:
967	try:
968	os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
969	fRc = True;
970	except:
971	fRc = False;
972	return fRc;
973
974	def processTerminate(uPid):
975	"""
976	Terminates the process in a nice manner (SIGTERM or equivalent).
977	Returns True on success, False on failure.
978	"""
979	fRc = False;
980	if sys.platform == 'win32':
981	try:
982	hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
983	False, uPid);
984	except:
985	pass;
986	else:
987	try:
988	win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
989	0x40010004); # DBG_TERMINATE_PROCESS
990	fRc = True;
991	except:
992	pass;
993	hProcess.Close(); #win32api.CloseHandle(hProcess)
994	else:
995	try:
996	os.kill(uPid, signal.SIGTERM);
997	fRc = True;
998	except:
999	pass;
1000	return fRc;
1001
1002	def processKill(uPid):
1003	"""
1004	Terminates the process with extreme prejudice (SIGKILL).
1005	Returns True on success, False on failure.
1006	"""
1007	if sys.platform == 'win32':
1008	fRc = processTerminate(uPid);
1009	else:
1010	try:
1011	os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
1012	fRc = True;
1013	except:
1014	fRc = False;
1015	return fRc;
1016
1017	def processKillWithNameCheck(uPid, sName):
1018	"""
1019	Like processKill(), but checks if the process name matches before killing
1020	it. This is intended for killing using potentially stale pid values.
1021
1022	Returns True on success, False on failure.
1023	"""
1024
1025	if processCheckPidAndName(uPid, sName) is not True:
1026	return False;
1027	return processKill(uPid);
1028
1029
1030	def processExists(uPid):
1031	"""
1032	Checks if the specified process exits.
1033	This will only work if we can signal/open the process.
1034
1035	Returns True if it positively exists, False otherwise.
1036	"""
1037	sHostOs = getHostOs();
1038	if sHostOs == 'win':
1039	fRc = False;
1040	# We try open the process for waiting since this is generally only forbidden in a very few cases.
1041	try:
1042	hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
1043	False, uPid);
1044	except pywintypes.error as oXcpt: # pylint: disable=no-member,used-before-assignment
1045	if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED: # pylint: disable=used-before-assignment
1046	fRc = True;
1047	except:
1048	pass;
1049	else:
1050	hProcess.Close();
1051	fRc = True;
1052	else:
1053	fRc = False;
1054	try:
1055	os.kill(uPid, 0);
1056	fRc = True;
1057	except OSError as oXcpt:
1058	if oXcpt.errno == errno.EPERM:
1059	fRc = True;
1060	except:
1061	pass;
1062	return fRc;
1063
1064	def processCheckPidAndName(uPid, sName):
1065	"""
1066	Checks if a process PID and NAME matches.
1067	"""
1068	fRc = processExists(uPid);
1069	if fRc is not True:
1070	return False;
1071
1072	if sys.platform == 'win32':
1073	try:
1074	from win32com.client import GetObject; # pylint: disable=import-error
1075	oWmi = GetObject('winmgmts:');
1076	aoProcesses = oWmi.InstancesOf('Win32_Process');
1077	for oProcess in aoProcesses:
1078	if long(oProcess.Properties_("ProcessId").Value) == uPid:
1079	sCurName = oProcess.Properties_("Name").Value;
1080	#reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
1081	sName = sName.lower();
1082	sCurName = sCurName.lower();
1083	if os.path.basename(sName) == sName:
1084	sCurName = os.path.basename(sCurName);
1085
1086	if sCurName == sName \
1087	or sCurName + '.exe' == sName \
1088	or sCurName == sName + '.exe':
1089	fRc = True;
1090	break;
1091	except:
1092	#reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
1093	pass;
1094	else:
1095	if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
1096	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1097	elif sys.platform in ('sunos5',):
1098	asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1099	elif sys.platform in ('darwin',):
1100	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
1101	else:
1102	asPsCmd = None;
1103
1104	if asPsCmd is not None:
1105	try:
1106	oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE); # pylint: disable=consider-using-with
1107	sCurName = oPs.communicate()[0];
1108	iExitCode = oPs.wait();
1109	except:
1110	#reporter.logXcpt();
1111	return False;
1112
1113	# ps fails with non-zero exit code if the pid wasn't found.
1114	if iExitCode != 0:
1115	return False;
1116	if sCurName is None:
1117	return False;
1118	sCurName = sCurName.strip();
1119	if not sCurName:
1120	return False;
1121
1122	if os.path.basename(sName) == sName:
1123	sCurName = os.path.basename(sCurName);
1124	elif os.path.basename(sCurName) == sCurName:
1125	sName = os.path.basename(sName);
1126
1127	if sCurName != sName:
1128	return False;
1129
1130	fRc = True;
1131	return fRc;
1132
1133	def processGetInfo(uPid, fSudo = False):
1134	"""
1135	Tries to acquire state information of the given process.
1136
1137	Returns a string with the information on success or None on failure or
1138	if the host is not supported.
1139
1140	Note that the format of the information is host system dependent and will
1141	likely differ much between different hosts.
1142	"""
1143	fRc = processExists(uPid);
1144	if fRc is not True:
1145	return None;
1146
1147	sHostOs = getHostOs();
1148	if sHostOs in [ 'linux',]:
1149	sGdb = '/usr/bin/gdb';
1150	if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1151	if not os.path.isfile(sGdb): sGdb = 'gdb';
1152	aasCmd = [
1153	[ sGdb, '-batch',
1154	'-ex', 'set pagination off',
1155	'-ex', 'thread apply all bt',
1156	'-ex', 'info proc mapping',
1157	'-ex', 'info sharedlibrary',
1158	'-p', '%u' % (uPid,), ],
1159	];
1160	elif sHostOs == 'darwin':
1161	# LLDB doesn't work in batch mode when attaching to a process, at least
1162	# with macOS Sierra (10.12). GDB might not be installed. Use the sample
1163	# tool instead with a 1 second duration and 1000ms sampling interval to
1164	# get one stack trace. For the process mappings use vmmap.
1165	aasCmd = [
1166	[ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1167	[ '/usr/bin/vmmap', '%u' % (uPid,), ],
1168	];
1169	elif sHostOs == 'solaris':
1170	aasCmd = [
1171	[ '/usr/bin/pstack', '%u' % (uPid,), ],
1172	[ '/usr/bin/pmap', '%u' % (uPid,), ],
1173	];
1174	else:
1175	aasCmd = [];
1176
1177	sInfo = '';
1178	for asCmd in aasCmd:
1179	try:
1180	if fSudo:
1181	sThisInfo = sudoProcessOutputChecked(asCmd);
1182	else:
1183	sThisInfo = processOutputChecked(asCmd);
1184	if sThisInfo is not None:
1185	sInfo += sThisInfo;
1186	except:
1187	pass;
1188	if not sInfo:
1189	sInfo = None;
1190
1191	return sInfo;
1192
1193
1194	class ProcessInfo(object):
1195	"""Process info."""
1196	def __init__(self, iPid):
1197	self.iPid = iPid;
1198	self.iParentPid = None;
1199	self.sImage = None;
1200	self.sName = None;
1201	self.asArgs = None;
1202	self.sCwd = None;
1203	self.iGid = None;
1204	self.iUid = None;
1205	self.iProcGroup = None;
1206	self.iSessionId = None;
1207
1208	def loadAll(self):
1209	"""Load all the info."""
1210	sOs = getHostOs();
1211	if sOs == 'linux':
1212	sProc = '/proc/%s/' % (self.iPid,);
1213	if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1214	if self.sImage is None:
1215	self.sImage = noxcptReadFile(sProc + 'comm', None);
1216	if self.sImage: self.sImage = self.sImage.strip();
1217	if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1218	if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1219	#elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1220	# sProc = '/proc/%s/' % (self.iPid,);
1221	# if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1222	# if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1223	else:
1224	pass;
1225	if self.sName is None and self.sImage is not None:
1226	self.sName = self.sImage;
1227
1228	def windowsGrabProcessInfo(self, oProcess):
1229	"""Windows specific loadAll."""
1230	try: self.sName = oProcess.Properties_("Name").Value;
1231	except: pass;
1232	try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1233	except: pass;
1234	try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1235	except: pass;
1236	try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1237	except: pass;
1238	try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1239	except: pass;
1240	if self.sName is None and self.sImage is not None:
1241	self.sName = self.sImage;
1242
1243	def getBaseImageName(self):
1244	"""
1245	Gets the base image name if available, use the process name if not available.
1246	Returns image/process base name or None.
1247	"""
1248	sRet = self.sImage if self.sName is None else self.sName;
1249	if sRet is None:
1250	self.loadAll();
1251	sRet = self.sImage if self.sName is None else self.sName;
1252	if sRet is None:
1253	if not self.asArgs:
1254	return None;
1255	sRet = self.asArgs[0];
1256	if not sRet:
1257	return None;
1258	return os.path.basename(sRet);
1259
1260	def getBaseImageNameNoExeSuff(self):
1261	"""
1262	Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1263	"""
1264	sRet = self.getBaseImageName();
1265	if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1266	if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1267	sRet = sRet[:-4];
1268	return sRet;
1269
1270
1271	def processListAll():
1272	"""
1273	Return a list of ProcessInfo objects for all the processes in the system
1274	that the current user can see.
1275	"""
1276	asProcesses = [];
1277
1278	sOs = getHostOs();
1279	if sOs == 'win':
1280	from win32com.client import GetObject; # pylint: disable=import-error
1281	oWmi = GetObject('winmgmts:');
1282	aoProcesses = oWmi.InstancesOf('Win32_Process');
1283	for oProcess in aoProcesses:
1284	try:
1285	iPid = int(oProcess.Properties_("ProcessId").Value);
1286	except:
1287	continue;
1288	oMyInfo = ProcessInfo(iPid);
1289	oMyInfo.windowsGrabProcessInfo(oProcess);
1290	asProcesses.append(oMyInfo);
1291	return asProcesses;
1292
1293	if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1294	try:
1295	asDirs = os.listdir('/proc');
1296	except:
1297	asDirs = [];
1298	for sDir in asDirs:
1299	if sDir.isdigit():
1300	asProcesses.append(ProcessInfo(int(sDir),));
1301	return asProcesses;
1302
1303	#
1304	# The other OSes parses the output from the 'ps' utility.
1305	#
1306	asPsCmd = [
1307	'/bin/ps', # 0
1308	'-A', # 1
1309	'-o', 'pid=', # 2,3
1310	'-o', 'ppid=', # 4,5
1311	'-o', 'pgid=', # 6,7
1312	'-o', 'sid=', # 8,9
1313	'-o', 'uid=', # 10,11
1314	'-o', 'gid=', # 12,13
1315	'-o', 'comm=' # 14,15
1316	];
1317
1318	if sOs == 'darwin':
1319	assert asPsCmd[9] == 'sid=';
1320	asPsCmd[9] = 'sess=';
1321	elif sOs == 'solaris':
1322	asPsCmd[0] = '/usr/bin/ps';
1323
1324	try:
1325	sRaw = processOutputChecked(asPsCmd);
1326	except:
1327	return asProcesses;
1328
1329	for sLine in sRaw.split('\n'):
1330	sLine = sLine.lstrip();
1331	if len(sLine) < 7 or not sLine[0].isdigit():
1332	continue;
1333
1334	iField = 0;
1335	off = 0;
1336	aoFields = [None, None, None, None, None, None, None];
1337	while iField < 7:
1338	# Eat whitespace.
1339	while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1340	off += 1;
1341
1342	# Final field / EOL.
1343	if iField == 6:
1344	aoFields[6] = sLine[off:];
1345	break;
1346	if off >= len(sLine):
1347	break;
1348
1349	# Generic field parsing.
1350	offStart = off;
1351	off += 1;
1352	while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1353	off += 1;
1354	try:
1355	if iField != 3:
1356	aoFields[iField] = int(sLine[offStart:off]);
1357	else:
1358	aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1359	except:
1360	pass;
1361	iField += 1;
1362
1363	if aoFields[0] is not None:
1364	oMyInfo = ProcessInfo(aoFields[0]);
1365	oMyInfo.iParentPid = aoFields[1];
1366	oMyInfo.iProcGroup = aoFields[2];
1367	oMyInfo.iSessionId = aoFields[3];
1368	oMyInfo.iUid = aoFields[4];
1369	oMyInfo.iGid = aoFields[5];
1370	oMyInfo.sName = aoFields[6];
1371	asProcesses.append(oMyInfo);
1372
1373	return asProcesses;
1374
1375
1376	def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1377	"""
1378	Looks for information regarding the demise of the given process.
1379	"""
1380	sOs = getHostOs();
1381	if sOs == 'darwin':
1382	#
1383	# On darwin we look for crash and diagnostic reports.
1384	#
1385	asLogDirs = [
1386	u'/Library/Logs/DiagnosticReports/',
1387	u'/Library/Logs/CrashReporter/',
1388	u'~/Library/Logs/DiagnosticReports/',
1389	u'~/Library/Logs/CrashReporter/',
1390	];
1391	for sDir in asLogDirs:
1392	sDir = os.path.expanduser(sDir);
1393	if not os.path.isdir(sDir):
1394	continue;
1395	try:
1396	asDirEntries = os.listdir(sDir);
1397	except:
1398	continue;
1399	for sEntry in asDirEntries:
1400	# Only interested in .crash files.
1401	_, sSuff = os.path.splitext(sEntry);
1402	if sSuff != '.crash':
1403	continue;
1404
1405	# The pid can be found at the end of the first line.
1406	sFull = os.path.join(sDir, sEntry);
1407	try:
1408	with open(sFull, 'r') as oFile: # pylint: disable=unspecified-encoding
1409	sFirstLine = oFile.readline();
1410	except:
1411	continue;
1412	if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1413	continue;
1414	offPid = len(sFirstLine) - 3;
1415	while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1416	offPid -= 1;
1417	try: uReportPid = int(sFirstLine[offPid:-2]);
1418	except: continue;
1419
1420	# Does the pid we found match?
1421	if uReportPid == uPid:
1422	fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1423	fnCrashFile(sFull, False);
1424	elif sOs == 'win':
1425	#
1426	# Getting WER reports would be great, however we have trouble match the
1427	# PID to those as they seems not to mention it in the brief reports.
1428	# Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1429	# location - see the windows readme for the testbox script) and what
1430	# the MSDN article lists for now.
1431	#
1432	# It's been observed on Windows server 2012 that the dump files takes
1433	# the form: <processimage>.<decimal-pid>.dmp
1434	#
1435	asDmpDirs = [
1436	u'%SystemDrive%/CrashDumps/', # Testboxes.
1437	u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1438	u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1439	u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1440	u'%WINDIR%/ServiceProfiles/',
1441	u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1442	];
1443	sMatchSuffix = '.%u.dmp' % (uPid,);
1444
1445	for sDir in asDmpDirs:
1446	sDir = os.path.expandvars(sDir);
1447	if not os.path.isdir(sDir):
1448	continue;
1449	try:
1450	asDirEntries = os.listdir(sDir);
1451	except:
1452	continue;
1453	for sEntry in asDirEntries:
1454	if sEntry.endswith(sMatchSuffix):
1455	sFull = os.path.join(sDir, sEntry);
1456	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1457	fnCrashFile(sFull, True);
1458	elif sOs == 'solaris':
1459	asDmpDirs = [];
1460	try:
1461	sScratchPath = os.environ.get('TESTBOX_PATH_SCRATCH', None);
1462	asDmpDirs.extend([ sScratchPath ]);
1463	except:
1464	pass;
1465	# Some other useful locations as fallback.
1466	asDmpDirs.extend([
1467	u'/var/cores/'
1468	]);
1469	#
1470	# Solaris by default creates a core file in the directory of the crashing process with the name 'core'.
1471	#
1472	# As we need to distinguish the core files correlating to their PIDs and have a persistent storage location,
1473	# the host needs to be tweaked via:
1474	#
1475	# ```coreadm -g /var/cores/core.%f.%p```
1476	#
1477	sMatchRegEx = r'core\..*\.%u' % (uPid);
1478	for sDir in asDmpDirs:
1479	sDir = os.path.expandvars(sDir);
1480	if not os.path.isdir(sDir):
1481	continue;
1482	try:
1483	asDirEntries = os.listdir(sDir);
1484	except:
1485	continue;
1486	for sEntry in asDirEntries:
1487	fnLog('Entry: %s' % (os.path.join(sDir, sEntry)));
1488	if re.search(sMatchRegEx, sEntry):
1489	sFull = os.path.join(sDir, sEntry);
1490	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1491	fnCrashFile(sFull, True);
1492	else:
1493	pass; ## TODO
1494	return None;
1495
1496
1497	#
1498	# Time.
1499	#
1500
1501	#
1502	# The following test case shows how time.time() only have ~ms resolution
1503	# on Windows (tested W10) and why it therefore makes sense to try use
1504	# performance counters.
1505	#
1506	# Note! We cannot use time.clock() as the timestamp must be portable across
1507	# processes. See timeout testcase problem on win hosts (no logs).
1508	# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1509	#
1510	#import sys;
1511	#import time;
1512	#from common import utils;
1513	#
1514	#atSeries = [];
1515	#for i in xrange(1,160):
1516	# if i == 159: time.sleep(10);
1517	# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1518	#
1519	#tPrev = atSeries[0]
1520	#for tCur in atSeries:
1521	# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1522	# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1523	# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1524	# print '';
1525	# tPrev = tCur
1526	#
1527	#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1528	#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1529	#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1530
1531	g_fWinUseWinPerfCounter = sys.platform == 'win32';
1532	g_fpWinPerfCounterFreq = None;
1533	g_oFuncwinQueryPerformanceCounter = None;
1534
1535	def _winInitPerfCounter():
1536	""" Initializes the use of performance counters. """
1537	global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1538
1539	uFrequency = ctypes.c_ulonglong(0);
1540	if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1541	if uFrequency.value >= 1000:
1542	#print 'uFrequency = %s' % (uFrequency,);
1543	#print 'type(uFrequency) = %s' % (type(uFrequency),);
1544	g_fpWinPerfCounterFreq = float(uFrequency.value);
1545
1546	# Check that querying the counter works too.
1547	global g_oFuncwinQueryPerformanceCounter
1548	g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1549	uCurValue = ctypes.c_ulonglong(0);
1550	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1551	if uCurValue.value > 0:
1552	return True;
1553	g_fWinUseWinPerfCounter = False;
1554	return False;
1555
1556	def _winFloatTime():
1557	""" Gets floating point time on windows. """
1558	if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1559	uCurValue = ctypes.c_ulonglong(0);
1560	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1561	return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1562	return time.time();
1563
1564	def timestampNano():
1565	"""
1566	Gets a nanosecond timestamp.
1567	"""
1568	if g_fWinUseWinPerfCounter is True:
1569	return long(_winFloatTime() * 1000000000);
1570	return long(time.time() * 1000000000);
1571
1572	def timestampMilli():
1573	"""
1574	Gets a millisecond timestamp.
1575	"""
1576	if g_fWinUseWinPerfCounter is True:
1577	return long(_winFloatTime() * 1000);
1578	return long(time.time() * 1000);
1579
1580	def timestampSecond():
1581	"""
1582	Gets a second timestamp.
1583	"""
1584	if g_fWinUseWinPerfCounter is True:
1585	return long(_winFloatTime());
1586	return long(time.time());
1587
1588	def secondsSinceUnixEpoch():
1589	"""
1590	Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1591	"""
1592	## ASSUMES This returns unix epoch time on all systems we care about...
1593	return time.time();
1594
1595	def getTimePrefix():
1596	"""
1597	Returns a timestamp prefix, typically used for logging. UTC.
1598	"""
1599	try:
1600	oNow = datetime.datetime.utcnow();
1601	sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1602	except:
1603	sTs = 'getTimePrefix-exception';
1604	return sTs;
1605
1606	def getTimePrefixAndIsoTimestamp():
1607	"""
1608	Returns current UTC as log prefix and iso timestamp.
1609	"""
1610	try:
1611	oNow = datetime.datetime.utcnow();
1612	sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1613	sTsIso = formatIsoTimestamp(oNow);
1614	except:
1615	sTsPrf = sTsIso = 'getTimePrefix-exception';
1616	return (sTsPrf, sTsIso);
1617
1618	class UtcTzInfo(datetime.tzinfo):
1619	"""UTC TZ Info Class"""
1620	def utcoffset(self, _):
1621	return datetime.timedelta(0);
1622	def tzname(self, _):
1623	return "UTC";
1624	def dst(self, _):
1625	return datetime.timedelta(0);
1626
1627	class GenTzInfo(datetime.tzinfo):
1628	"""Generic TZ Info Class"""
1629	def __init__(self, offInMin):
1630	datetime.tzinfo.__init__(self);
1631	self.offInMin = offInMin;
1632	def utcoffset(self, _):
1633	return datetime.timedelta(minutes = self.offInMin);
1634	def tzname(self, _):
1635	if self.offInMin >= 0:
1636	return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1637	return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1638	def dst(self, _):
1639	return datetime.timedelta(0);
1640
1641	def formatIsoTimestamp(oNow):
1642	"""Formats the datetime object as an ISO timestamp."""
1643	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1644	sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1645	return sTs;
1646
1647	def getIsoTimestamp():
1648	"""Returns the current UTC timestamp as a string."""
1649	return formatIsoTimestamp(datetime.datetime.utcnow());
1650
1651	def formatShortIsoTimestamp(oNow):
1652	"""Formats the datetime object as an ISO timestamp, but w/o microseconds."""
1653	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1654	return oNow.strftime('%Y-%m-%dT%H:%M:%SZ');
1655
1656	def getShortIsoTimestamp():
1657	"""Returns the current UTC timestamp as a string, but w/o microseconds."""
1658	return formatShortIsoTimestamp(datetime.datetime.utcnow());
1659
1660	def convertDateTimeToZulu(oDateTime):
1661	""" Converts oDateTime to zulu time if it has timezone info. """
1662	if oDateTime.tzinfo is not None:
1663	oDateTime = oDateTime.astimezone(UtcTzInfo());
1664	else:
1665	oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1666	return oDateTime;
1667
1668	def parseIsoTimestamp(sTs):
1669	"""
1670	Parses a typical ISO timestamp, returing a datetime object, reasonably
1671	forgiving, but will throw weird indexing/conversion errors if the input
1672	is malformed.
1673	"""
1674	# YYYY-MM-DD
1675	iYear = int(sTs[0:4]);
1676	assert(sTs[4] == '-');
1677	iMonth = int(sTs[5:7]);
1678	assert(sTs[7] == '-');
1679	iDay = int(sTs[8:10]);
1680
1681	# Skip separator
1682	sTime = sTs[10:];
1683	while sTime[0] in 'Tt \t\n\r':
1684	sTime = sTime[1:];
1685
1686	# HH:MM[:SS]
1687	iHour = int(sTime[0:2]);
1688	assert(sTime[2] == ':');
1689	iMin = int(sTime[3:5]);
1690	if sTime[5] == ':':
1691	iSec = int(sTime[6:8]);
1692
1693	# Fraction?
1694	offTime = 8;
1695	iMicroseconds = 0;
1696	if offTime < len(sTime) and sTime[offTime] in '.,':
1697	offTime += 1;
1698	cchFraction = 0;
1699	while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1700	cchFraction += 1;
1701	if cchFraction > 0:
1702	iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1703	offTime += cchFraction;
1704	while cchFraction < 6:
1705	iMicroseconds *= 10;
1706	cchFraction += 1;
1707	while cchFraction > 6:
1708	iMicroseconds = iMicroseconds // 10;
1709	cchFraction -= 1;
1710
1711	else:
1712	iSec = 0;
1713	iMicroseconds = 0;
1714	offTime = 5;
1715
1716	# Naive?
1717	if offTime >= len(sTime):
1718	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1719
1720	# Zulu?
1721	if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1722	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1723
1724	# Some kind of offset afterwards, and strptime is useless. sigh.
1725	if sTime[offTime] in '+-':
1726	chSign = sTime[offTime];
1727	offTime += 1;
1728	cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1729	offTime += 2;
1730	if offTime < len(sTime) and sTime[offTime] in ':':
1731	offTime += 1;
1732	if offTime + 2 <= len(sTime):
1733	cMinTz += int(sTime[offTime : (offTime + 2)]);
1734	offTime += 2;
1735	assert offTime == len(sTime);
1736	if chSign == '-':
1737	cMinTz = -cMinTz;
1738	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1739	assert False, sTs;
1740	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1741
1742	def normalizeIsoTimestampToZulu(sTs):
1743	"""
1744	Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1745	+ convertDateTimeToZulu + formatIsoTimestamp).
1746	Returns ISO tiemstamp string.
1747	"""
1748	return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1749
1750	def getLocalHourOfWeek():
1751	""" Local hour of week (0 based). """
1752	oNow = datetime.datetime.now();
1753	return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1754
1755
1756	def formatIntervalSeconds(cSeconds):
1757	""" Format a seconds interval into a nice 01h 00m 22s string """
1758	# Two simple special cases.
1759	if cSeconds < 60:
1760	return '%ss' % (cSeconds,);
1761	if cSeconds < 3600:
1762	cMins = cSeconds // 60;
1763	cSecs = cSeconds % 60;
1764	if cSecs == 0:
1765	return '%sm' % (cMins,);
1766	return '%sm %ss' % (cMins, cSecs,);
1767
1768	# Generic and a bit slower.
1769	cDays = cSeconds // 86400;
1770	cSeconds %= 86400;
1771	cHours = cSeconds // 3600;
1772	cSeconds %= 3600;
1773	cMins = cSeconds // 60;
1774	cSecs = cSeconds % 60;
1775	sRet = '';
1776	if cDays > 0:
1777	sRet = '%sd ' % (cDays,);
1778	if cHours > 0:
1779	sRet += '%sh ' % (cHours,);
1780	if cMins > 0:
1781	sRet += '%sm ' % (cMins,);
1782	if cSecs > 0:
1783	sRet += '%ss ' % (cSecs,);
1784	assert sRet; assert sRet[-1] == ' ';
1785	return sRet[:-1];
1786
1787	def formatIntervalSeconds2(oSeconds):
1788	"""
1789	Flexible input version of formatIntervalSeconds for use in WUI forms where
1790	data is usually already string form.
1791	"""
1792	if isinstance(oSeconds, (int, long)):
1793	return formatIntervalSeconds(oSeconds);
1794	if not isString(oSeconds):
1795	try:
1796	lSeconds = long(oSeconds);
1797	except:
1798	pass;
1799	else:
1800	if lSeconds >= 0:
1801	return formatIntervalSeconds2(lSeconds);
1802	return oSeconds;
1803
1804	def parseIntervalSeconds(sString):
1805	"""
1806	Reverse of formatIntervalSeconds.
1807
1808	Returns (cSeconds, sError), where sError is None on success.
1809	"""
1810
1811	# We might given non-strings, just return them without any fuss.
1812	if not isString(sString):
1813	if isinstance(sString, (int, long)) or sString is None:
1814	return (sString, None);
1815	## @todo time/date objects?
1816	return (int(sString), None);
1817
1818	# Strip it and make sure it's not empty.
1819	sString = sString.strip();
1820	if not sString:
1821	return (0, 'Empty interval string.');
1822
1823	#
1824	# Split up the input into a list of 'valueN, unitN, ...'.
1825	#
1826	# Don't want to spend too much time trying to make re.split do exactly what
1827	# I need here, so please forgive the extra pass I'm making here.
1828	#
1829	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1830	asParts = [];
1831	for sPart in asRawParts:
1832	sPart = sPart.strip();
1833	if sPart:
1834	asParts.append(sPart);
1835	if not asParts:
1836	return (0, 'Empty interval string or something?');
1837
1838	#
1839	# Process them one or two at the time.
1840	#
1841	cSeconds = 0;
1842	asErrors = [];
1843	i = 0;
1844	while i < len(asParts):
1845	sNumber = asParts[i];
1846	i += 1;
1847	if sNumber.isdigit():
1848	iNumber = int(sNumber);
1849
1850	sUnit = 's';
1851	if i < len(asParts) and not asParts[i].isdigit():
1852	sUnit = asParts[i];
1853	i += 1;
1854
1855	sUnitLower = sUnit.lower();
1856	if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1857	pass;
1858	elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1859	iNumber *= 60;
1860	elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1861	iNumber *= 3600;
1862	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1863	iNumber *= 86400;
1864	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1865	iNumber = 7 86400;
1866	else:
1867	asErrors.append('Unknown unit "%s".' % (sUnit,));
1868	cSeconds += iNumber;
1869	else:
1870	asErrors.append('Bad number "%s".' % (sNumber,));
1871	return (cSeconds, None if not asErrors else ' '.join(asErrors));
1872
1873	def formatIntervalHours(cHours):
1874	""" Format a hours interval into a nice 1w 2d 1h string. """
1875	# Simple special cases.
1876	if cHours < 24:
1877	return '%sh' % (cHours,);
1878
1879	# Generic and a bit slower.
1880	cWeeks = cHours / (7 * 24);
1881	cHours %= 7 * 24;
1882	cDays = cHours / 24;
1883	cHours %= 24;
1884	sRet = '';
1885	if cWeeks > 0:
1886	sRet = '%sw ' % (cWeeks,);
1887	if cDays > 0:
1888	sRet = '%sd ' % (cDays,);
1889	if cHours > 0:
1890	sRet += '%sh ' % (cHours,);
1891	assert sRet; assert sRet[-1] == ' ';
1892	return sRet[:-1];
1893
1894	def parseIntervalHours(sString):
1895	"""
1896	Reverse of formatIntervalHours.
1897
1898	Returns (cHours, sError), where sError is None on success.
1899	"""
1900
1901	# We might given non-strings, just return them without any fuss.
1902	if not isString(sString):
1903	if isinstance(sString, (int, long)) or sString is None:
1904	return (sString, None);
1905	## @todo time/date objects?
1906	return (int(sString), None);
1907
1908	# Strip it and make sure it's not empty.
1909	sString = sString.strip();
1910	if not sString:
1911	return (0, 'Empty interval string.');
1912
1913	#
1914	# Split up the input into a list of 'valueN, unitN, ...'.
1915	#
1916	# Don't want to spend too much time trying to make re.split do exactly what
1917	# I need here, so please forgive the extra pass I'm making here.
1918	#
1919	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1920	asParts = [];
1921	for sPart in asRawParts:
1922	sPart = sPart.strip();
1923	if sPart:
1924	asParts.append(sPart);
1925	if not asParts:
1926	return (0, 'Empty interval string or something?');
1927
1928	#
1929	# Process them one or two at the time.
1930	#
1931	cHours = 0;
1932	asErrors = [];
1933	i = 0;
1934	while i < len(asParts):
1935	sNumber = asParts[i];
1936	i += 1;
1937	if sNumber.isdigit():
1938	iNumber = int(sNumber);
1939
1940	sUnit = 'h';
1941	if i < len(asParts) and not asParts[i].isdigit():
1942	sUnit = asParts[i];
1943	i += 1;
1944
1945	sUnitLower = sUnit.lower();
1946	if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1947	pass;
1948	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1949	iNumber *= 24;
1950	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1951	iNumber = 7 24;
1952	else:
1953	asErrors.append('Unknown unit "%s".' % (sUnit,));
1954	cHours += iNumber;
1955	else:
1956	asErrors.append('Bad number "%s".' % (sNumber,));
1957	return (cHours, None if not asErrors else ' '.join(asErrors));
1958
1959
1960	#
1961	# Introspection.
1962	#
1963
1964	def getCallerName(oFrame=None, iFrame=2):
1965	"""
1966	Returns the name of the caller's caller.
1967	"""
1968	if oFrame is None:
1969	try:
1970	raise Exception();
1971	except:
1972	oFrame = sys.exc_info()[2].tb_frame.f_back;
1973	while iFrame > 1:
1974	if oFrame is not None:
1975	oFrame = oFrame.f_back;
1976	iFrame = iFrame - 1;
1977	if oFrame is not None:
1978	sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1979	return sName;
1980	return "unknown";
1981
1982
1983	def getXcptInfo(cFrames = 1):
1984	"""
1985	Gets text detailing the exception. (Good for logging.)
1986	Returns list of info strings.
1987	"""
1988
1989	#
1990	# Try get exception info.
1991	#
1992	try:
1993	oType, oValue, oTraceback = sys.exc_info();
1994	except:
1995	oType = oValue = oTraceback = None;
1996	if oType is not None:
1997
1998	#
1999	# Try format the info
2000	#
2001	asRet = [];
2002	try:
2003	try:
2004	asRet = asRet + traceback.format_exception_only(oType, oValue);
2005	asTraceBack = traceback.format_tb(oTraceback);
2006	if cFrames is not None and cFrames <= 1:
2007	asRet.append(asTraceBack[-1]);
2008	else:
2009	asRet.append('Traceback:')
2010	for iFrame in range(min(cFrames, len(asTraceBack))):
2011	asRet.append(asTraceBack[-iFrame - 1]);
2012	asRet.append('Stack:')
2013	asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
2014	except:
2015	asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
2016
2017	if not asRet:
2018	asRet.append('No exception info...');
2019	except:
2020	asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
2021	else:
2022	asRet = ['Couldn\'t find exception traceback.'];
2023	return asRet;
2024
2025
2026	def getObjectTypeName(oObject):
2027	"""
2028	Get the type name of the given object.
2029	"""
2030	if oObject is None:
2031	return 'None';
2032
2033	# Get the type object.
2034	try:
2035	oType = type(oObject);
2036	except:
2037	return 'type-throws-exception';
2038
2039	# Python 2.x only: Handle old-style object wrappers.
2040	if sys.version_info[0] < 3:
2041	try:
2042	from types import InstanceType; # pylint: disable=no-name-in-module
2043	if oType == InstanceType:
2044	oType = oObject.__class__;
2045	except:
2046	pass;
2047
2048	# Get the name.
2049	try:
2050	return oType.__name__;
2051	except:
2052	return '__type__-throws-exception';
2053
2054
2055	def chmodPlusX(sFile):
2056	"""
2057	Makes the specified file or directory executable.
2058	Returns success indicator, no exceptions.
2059
2060	Note! Symbolic links are followed and the target will be changed.
2061	"""
2062	try:
2063	oStat = os.stat(sFile);
2064	except:
2065	return False;
2066	try:
2067	os.chmod(sFile, oStat.st_mode \| stat.S_IXUSR \| stat.S_IXGRP \| stat.S_IXOTH);
2068	except:
2069	return False;
2070	return True;
2071
2072
2073	#
2074	# TestSuite stuff.
2075	#
2076
2077	def isRunningFromCheckout(cScriptDepth = 1):
2078	"""
2079	Checks if we're running from the SVN checkout or not.
2080	"""
2081
2082	try:
2083	sFile = __file__;
2084	cScriptDepth = 1;
2085	except:
2086	sFile = sys.argv[0];
2087
2088	sDir = os.path.abspath(sFile);
2089	while cScriptDepth >= 0:
2090	sDir = os.path.dirname(sDir);
2091	if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
2092	or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
2093	return True;
2094	cScriptDepth -= 1;
2095
2096	return False;
2097
2098
2099	#
2100	# Bourne shell argument fun.
2101	#
2102
2103
2104	def argsSplit(sCmdLine):
2105	"""
2106	Given a bourne shell command line invocation, split it up into arguments
2107	assuming IFS is space.
2108	Returns None on syntax error.
2109	"""
2110	## @todo bourne shell argument parsing!
2111	return sCmdLine.split(' ');
2112
2113	def argsGetFirst(sCmdLine):
2114	"""
2115	Given a bourne shell command line invocation, get return the first argument
2116	assuming IFS is space.
2117	Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
2118	"""
2119	asArgs = argsSplit(sCmdLine);
2120	if not asArgs:
2121	return None;
2122
2123	return asArgs[0];
2124
2125	#
2126	# String helpers.
2127	#
2128
2129	def stricmp(sFirst, sSecond):
2130	"""
2131	Compares to strings in an case insensitive fashion.
2132
2133	Python doesn't seem to have any way of doing the correctly, so this is just
2134	an approximation using lower.
2135	"""
2136	if sFirst == sSecond:
2137	return 0;
2138	sLower1 = sFirst.lower();
2139	sLower2 = sSecond.lower();
2140	if sLower1 == sLower2:
2141	return 0;
2142	if sLower1 < sLower2:
2143	return -1;
2144	return 1;
2145
2146
2147	def versionCompare(sVer1, sVer2):
2148	"""
2149	Compares to version strings in a fashion similar to RTStrVersionCompare.
2150	"""
2151
2152	## @todo implement me!!
2153
2154	if sVer1 == sVer2:
2155	return 0;
2156	if sVer1 < sVer2:
2157	return -1;
2158	return 1;
2159
2160
2161	def formatNumber(lNum, sThousandSep = ' '):
2162	"""
2163	Formats a decimal number with pretty separators.
2164	"""
2165	sNum = str(lNum);
2166	sRet = sNum[-3:];
2167	off = len(sNum) - 3;
2168	while off > 0:
2169	off -= 3;
2170	sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
2171	return sRet;
2172
2173
2174	def formatNumberNbsp(lNum):
2175	"""
2176	Formats a decimal number with pretty separators.
2177	"""
2178	sRet = formatNumber(lNum);
2179	return unicode(sRet).replace(' ', u'\u00a0');
2180
2181
2182	def isString(oString):
2183	"""
2184	Checks if the object is a string object, hiding difference between python 2 and 3.
2185
2186	Returns True if it's a string of some kind.
2187	Returns False if not.
2188	"""
2189	if sys.version_info[0] >= 3:
2190	return isinstance(oString, str);
2191	return isinstance(oString, basestring); # pylint: disable=undefined-variable
2192
2193
2194	def hasNonAsciiCharacters(sText):
2195	"""
2196	Returns True is specified string has non-ASCII characters, False if ASCII only.
2197	"""
2198	if isString(sText):
2199	for ch in sText:
2200	if ord(ch) >= 128:
2201	return True;
2202	else:
2203	# Probably byte array or some such thing.
2204	for ch in sText:
2205	if ch >= 128 or ch < 0:
2206	return True;
2207	return False;
2208
2209
2210	#
2211	# Unpacking.
2212	#
2213
2214	def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2215	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2216	"""
2217	Worker for unpackFile that deals with ZIP files, same function signature.
2218	"""
2219	import zipfile
2220	if fnError is None:
2221	fnError = fnLog;
2222
2223	fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2224
2225	# Open it.
2226	try: oZipFile = zipfile.ZipFile(sArchive, 'r'); # pylint: disable=consider-using-with
2227	except Exception as oXcpt:
2228	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2229	return None;
2230
2231	# Extract all members.
2232	asMembers = [];
2233	try:
2234	for sMember in oZipFile.namelist():
2235	if fnFilter is None or fnFilter(sMember) is not False:
2236	if sMember.endswith('/'):
2237	os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2238	else:
2239	oZipFile.extract(sMember, sDstDir);
2240	asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2241	except Exception as oXcpt:
2242	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2243	asMembers = None;
2244
2245	# close it.
2246	try: oZipFile.close();
2247	except Exception as oXcpt:
2248	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2249	asMembers = None;
2250
2251	return asMembers;
2252
2253
2254	## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2255	g_fTarCopyFileObjOverriddend = False;
2256
2257	def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2258	""" tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2259	_ = bufsize;
2260	if length is None:
2261	__myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2262	elif length > 0:
2263	cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2264	for _ in xrange(cFull):
2265	abBuffer = src.read(g_cbGoodBufferSize);
2266	dst.write(abBuffer);
2267	if len(abBuffer) != g_cbGoodBufferSize:
2268	raise exception('unexpected end of source file');
2269	if cbRemainder > 0:
2270	abBuffer = src.read(cbRemainder);
2271	dst.write(abBuffer);
2272	if len(abBuffer) != cbRemainder:
2273	raise exception('unexpected end of source file');
2274
2275
2276	def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2277	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2278	"""
2279	Worker for unpackFile that deals with tarballs, same function signature.
2280	"""
2281	import shutil;
2282	import tarfile;
2283	if fnError is None:
2284	fnError = fnLog;
2285
2286	fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2287
2288	#
2289	# Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2290	# 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2291	# 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2292	#
2293	if True is True: # pylint: disable=comparison-with-itself,comparison-of-constants
2294	__installShUtilHacks(shutil);
2295	global g_fTarCopyFileObjOverriddend;
2296	if g_fTarCopyFileObjOverriddend is False:
2297	g_fTarCopyFileObjOverriddend = True;
2298	#if sys.hexversion < 0x03060000:
2299	tarfile.copyfileobj = __mytarfilecopyfileobj;
2300
2301	#
2302	# Open it.
2303	#
2304	# Note! We not using 'r:*' because we cannot allow seeking compressed files!
2305	# That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2306	#
2307	try:
2308	if sys.hexversion >= 0x03060000:
2309	oTarFile = tarfile.open(sArchive, 'r\|*', # pylint: disable=consider-using-with
2310	bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2311	else:
2312	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize); # pylint: disable=consider-using-with
2313	except Exception as oXcpt:
2314	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2315	return None;
2316
2317	# Extract all members.
2318	asMembers = [];
2319	try:
2320	for oTarInfo in oTarFile:
2321	try:
2322	if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2323	if oTarInfo.islnk():
2324	# Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2325	# in the compressed tar stream. So, fall back on shutil.copy2 instead.
2326	sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2327	sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2328	sParentDir = os.path.dirname(sLinkFile);
2329	try: os.unlink(sLinkFile);
2330	except: pass;
2331	if sParentDir and not os.path.exists(sParentDir):
2332	os.makedirs(sParentDir);
2333	try: os.link(sLinkTarget, sLinkFile);
2334	except: shutil.copy2(sLinkTarget, sLinkFile);
2335	else:
2336	if oTarInfo.isdir():
2337	# Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2338	oTarInfo.mode \|= 0x1c0; # (octal: 0700)
2339	if hasattr(tarfile, 'tar_filter'):
2340	oTarFile.extract(oTarInfo, sDstDir, filter = 'tar');
2341	else:
2342	oTarFile.extract(oTarInfo, sDstDir);
2343	asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2344	except Exception as oXcpt:
2345	fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2346	for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2347	fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2348	for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2349	fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2350	asMembers = None;
2351	break;
2352	except Exception as oXcpt:
2353	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2354	asMembers = None;
2355
2356	#
2357	# Finally, close it.
2358	#
2359	try: oTarFile.close();
2360	except Exception as oXcpt:
2361	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2362	asMembers = None;
2363
2364	return asMembers;
2365
2366
2367	def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2368	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2369	"""
2370	Unpacks the given file if it has a know archive extension, otherwise do
2371	nothing.
2372
2373	fnLog & fnError both take a string parameter.
2374
2375	fnFilter takes a member name (string) and returns True if it's included
2376	and False if excluded.
2377
2378	Returns list of the extracted files (full path) on success.
2379	Returns empty list if not a supported archive format.
2380	Returns None on failure. Raises no exceptions.
2381	"""
2382	sBaseNameLower = os.path.basename(sArchive).lower();
2383
2384	#
2385	# Zip file?
2386	#
2387	if sBaseNameLower.endswith('.zip'):
2388	return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2389
2390	#
2391	# Tarball?
2392	#
2393	if sBaseNameLower.endswith('.tar') \
2394	or sBaseNameLower.endswith('.tar.gz') \
2395	or sBaseNameLower.endswith('.tgz') \
2396	or sBaseNameLower.endswith('.tar.bz2'):
2397	return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2398
2399	#
2400	# Cannot classify it from the name, so just return that to the caller.
2401	#
2402	fnLog('Not unpacking "%s".' % (sArchive,));
2403	return [];
2404
2405
2406	#
2407	# Misc.
2408	#
2409	def areBytesEqual(oLeft, oRight):
2410	"""
2411	Compares two byte arrays, strings or whatnot.
2412
2413	returns true / false accordingly.
2414	"""
2415
2416	# If both are None, consider them equal (bogus?):
2417	if oLeft is None and oRight is None:
2418	return True;
2419
2420	# If just one is None, they can't match:
2421	if oLeft is None or oRight is None:
2422	return False;
2423
2424	# If both have the same type, use the compare operator of the class:
2425	if type(oLeft) is type(oRight):
2426	#print('same type: %s' % (oLeft == oRight,));
2427	return oLeft == oRight;
2428
2429	# On the offchance that they're both strings, but of different types.
2430	if isString(oLeft) and isString(oRight):
2431	#print('string compare: %s' % (oLeft == oRight,));
2432	return oLeft == oRight;
2433
2434	#
2435	# See if byte/buffer stuff that can be compared directory. If not convert
2436	# strings to bytes.
2437	#
2438	# Note! For 2.x, we must convert both sides to the buffer type or the
2439	# comparison may fail despite it working okay in test cases.
2440	#
2441	if sys.version_info[0] >= 3:
2442	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2443	return oLeft == oRight;
2444
2445	if isString(oLeft):
2446	try: oLeft = bytes(oLeft, 'utf-8');
2447	except: pass;
2448	if isString(oRight):
2449	try: oRight = bytes(oRight, 'utf-8');
2450	except: pass;
2451	else:
2452	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2453	if isinstance(oLeft, bytearray):
2454	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2455	else:
2456	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2457	#print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2458	return oLeft == oRight;
2459
2460	if isString(oLeft):
2461	try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2462	except: pass;
2463	if isString(oRight):
2464	try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2465	except: pass;
2466
2467	# Check if we now have the same type for both:
2468	if type(oLeft) is type(oRight):
2469	#print('same type now: %s' % (oLeft == oRight,));
2470	return oLeft == oRight;
2471
2472	# Check if we now have buffer/memoryview vs bytes/bytesarray again.
2473	if sys.version_info[0] >= 3:
2474	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2475	return oLeft == oRight;
2476	else:
2477	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2478	if isinstance(oLeft, bytearray):
2479	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2480	else:
2481	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2482	#print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2483	return oLeft == oRight;
2484
2485	# Do item by item comparison:
2486	if len(oLeft) != len(oRight):
2487	#print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2488	return False;
2489	i = len(oLeft);
2490	while i > 0:
2491	i = i - 1;
2492
2493	iElmLeft = oLeft[i];
2494	if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2495	iElmLeft = ord(iElmLeft);
2496
2497	iElmRight = oRight[i];
2498	if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2499	iElmRight = ord(iElmRight);
2500
2501	if iElmLeft != iElmRight:
2502	#print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2503	return False;
2504	return True;
2505
2506
2507	def calcCrc32OfFile(sFile):
2508	"""
2509	Simple helper for calculating the CRC32 of a file.
2510
2511	Throws stuff if the file cannot be opened or read successfully.
2512	"""
2513	import zlib;
2514
2515	uCrc32 = 0;
2516	with open(sFile, 'rb') as oFile: # pylint: disable=unspecified-encoding
2517	while True:
2518	oBuf = oFile.read(1024 * 1024);
2519	if not oBuf:
2520	break
2521	uCrc32 = zlib.crc32(oBuf, uCrc32);
2522
2523	return uCrc32 % 2**32;
2524
2525
2526	#
2527	# Unit testing.
2528	#
2529
2530	# pylint: disable=missing-docstring
2531	# pylint: disable=undefined-variable
2532	class BuildCategoryDataTestCase(unittest.TestCase):
2533	def testIntervalSeconds(self):
2534	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2535	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2536	self.assertEqual(parseIntervalSeconds('123'), (123, None));
2537	self.assertEqual(parseIntervalSeconds(123), (123, None));
2538	self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2539	self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2540	self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2541	self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2542	self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2543	self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2544	self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2545
2546	def testZuluNormalization(self):
2547	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2548	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2549	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2550	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2551	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2552
2553	def testHasNonAsciiChars(self):
2554	self.assertEqual(hasNonAsciiCharacters(''), False);
2555	self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB&$%&)@#(!)@(#!(#$&#$&%*Y@#$IQWN---00;'), False);
2556	self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2557	self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2558	self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2559	self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2560	self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2561	self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2562	self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2563	self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2564
2565	def testAreBytesEqual(self):
2566	self.assertEqual(areBytesEqual(None, None), True);
2567	self.assertEqual(areBytesEqual(None, ''), False);
2568	self.assertEqual(areBytesEqual('', ''), True);
2569	self.assertEqual(areBytesEqual('1', '1'), True);
2570	self.assertEqual(areBytesEqual('12345', '1234'), False);
2571	self.assertEqual(areBytesEqual('1234', '1234'), True);
2572	self.assertEqual(areBytesEqual('1234', b'1234'), True);
2573	self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2574	self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2575	self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2576	self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2577	self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2578	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2579	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2580	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2581	if sys.version_info[0] >= 3:
2582	pass;
2583	else:
2584	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2585	bytearray([0x31,0x32,0x33,0x34])), True);
2586	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2587	bytearray([0x99,0x32,0x32,0x34])), False);
2588	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2589	buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2590	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2591	buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2592	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2593	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2594	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2595
2596	if __name__ == '__main__':
2597	unittest.main();
2598	# not reached.

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 107926

Download in other formats: