schedulerbase.py@ 65040

Last change on this file since 65040 was 65034, checked in by vboxsync, 8 years ago
testmanager/schedulerbase.py: More fixing on the problem of getting stuck doing nothing when a scheduling group is out of new builds to test.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 66.3 KB

Line
1	# -- coding: utf-8 --
2	# $Id: schedulerbase.py 65034 2016-12-29 23:14:18Z vboxsync $
3	# pylint: disable=C0302
4
5
6	"""
7	Test Manager - Base class and utilities for the schedulers.
8	"""
9
10	__copyright__ = \
11	"""
12	Copyright (C) 2012-2016 Oracle Corporation
13
14	This file is part of VirtualBox Open Source Edition (OSE), as
15	available from http://www.virtualbox.org. This file is free software;
16	you can redistribute it and/or modify it under the terms of the GNU
17	General Public License (GPL) as published by the Free Software
18	Foundation, in version 2 as it comes in the "COPYING" file of the
19	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
20	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
21
22	The contents of this file may alternatively be used under the terms
23	of the Common Development and Distribution License Version 1.0
24	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
25	VirtualBox OSE distribution, in which case the provisions of the
26	CDDL are applicable instead of those of the GPL.
27
28	You may elect to license modified versions of this file under the
29	terms and conditions of either the GPL or the CDDL or both.
30	"""
31	__version__ = "$Revision: 65034 $"
32
33
34	# Standard python imports.
35	import unittest;
36
37	# Validation Kit imports.
38	from common import utils, constants;
39	from testmanager import config;
40	from testmanager.core.build import BuildDataEx, BuildLogic;
41	from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, TMExceptionBase;
42	from testmanager.core.buildsource import BuildSourceData, BuildSourceLogic;
43	from testmanager.core.globalresource import GlobalResourceLogic;
44	from testmanager.core.schedgroup import SchedGroupData, SchedGroupLogic;
45	from testmanager.core.systemlog import SystemLogData, SystemLogLogic;
46	from testmanager.core.testbox import TestBoxData, TestBoxDataEx;
47	from testmanager.core.testboxstatus import TestBoxStatusData, TestBoxStatusLogic;
48	from testmanager.core.testcase import TestCaseLogic;
49	from testmanager.core.testcaseargs import TestCaseArgsDataEx, TestCaseArgsLogic;
50	from testmanager.core.testset import TestSetData, TestSetLogic;
51
52
53	class ReCreateQueueData(object):
54	"""
55	Data object for recreating a scheduling queue.
56
57	It's mostly a storage object, but has a few data checking operation
58	associated with it.
59	"""
60
61	def __init__(self, oDb, idSchedGroup):
62	#
63	# Load data from the database.
64	#
65
66	# Will extend the entries with aoTestCases and dTestCases members
67	# further down. checkForGroupDepCycles will add aidTestGroupPreReqs.
68	self.aoTestGroups = SchedGroupLogic(oDb).getMembers(idSchedGroup);
69
70	# aoTestCases entries are TestCaseData instance with iSchedPriority
71	# and idTestGroup added for our purposes.
72	# We will add oTestGroup and aoArgsVariations members to each further down.
73	self.aoTestCases = SchedGroupLogic(oDb).getTestCasesForGroup(idSchedGroup, cMax = 4096);
74
75	# Load dependencies.
76	oTestCaseLogic = TestCaseLogic(oDb)
77	for oTestCase in self.aoTestCases:
78	oTestCase.aidPreReqs = oTestCaseLogic.getTestCasePreReqIds(oTestCase.idTestCase, cMax = 4096);
79
80	# aoTestCases entries are TestCaseArgsData instance with iSchedPriority
81	# and idTestGroup added for our purposes.
82	# We will add oTestGroup and oTestCase members to each further down.
83	self.aoArgsVariations = SchedGroupLogic(oDb).getTestCaseArgsForGroup(idSchedGroup, cMax = 65536);
84
85	#
86	# Generate global lookups.
87	#
88
89	# Generate a testcase lookup dictionary for use when working on
90	# argument variations.
91	self.dTestCases = dict();
92	for oTestCase in self.aoTestCases:
93	self.dTestCases[oTestCase.idTestCase] = oTestCase;
94	assert len(self.dTestCases) <= len(self.aoTestCases); # Note! Can be shorter!
95
96	# Generate a testgroup lookup dictionary.
97	self.dTestGroups = dict();
98	for oTestGroup in self.aoTestGroups:
99	self.dTestGroups[oTestGroup.idTestGroup] = oTestGroup;
100	assert len(self.dTestGroups) == len(self.aoTestGroups);
101
102	#
103	# Associate extra members with the base data.
104	#
105	if len(self.aoTestGroups) > 0:
106	# Prep the test groups.
107	for oTestGroup in self.aoTestGroups:
108	oTestGroup.aoTestCases = list();
109	oTestGroup.dTestCases = dict();
110
111	# Link testcases to their group, both directions. Prep testcases for
112	# argument varation association.
113	oTestGroup = self.aoTestGroups[0];
114	for oTestCase in self.aoTestCases:
115	if oTestGroup.idTestGroup != oTestCase.idTestGroup:
116	oTestGroup = self.dTestGroups[oTestCase.idTestGroup];
117
118	assert oTestCase.idTestCase not in oTestGroup.dTestCases;
119	oTestGroup.dTestCases[oTestCase.idTestCase] = oTestCase;
120	oTestGroup.aoTestCases.append(oTestCase);
121	oTestCase.oTestGroup = oTestGroup;
122	oTestCase.aoArgsVariations = list();
123
124	# Associate testcase argument variations with their testcases (group)
125	# in both directions.
126	oTestGroup = self.aoTestGroups[0];
127	oTestCase = self.aoTestCases[0] if len(self.aoTestCases) > 0 else None;
128	for oArgVariation in self.aoArgsVariations:
129	if oTestGroup.idTestGroup != oArgVariation.idTestGroup:
130	oTestGroup = self.dTestGroups[oArgVariation.idTestGroup];
131	if oTestCase.idTestCase != oArgVariation.idTestCase or oTestCase.idTestGroup != oArgVariation.idTestGroup:
132	oTestCase = oTestGroup.dTestCases[oArgVariation.idTestCase];
133
134	oTestCase.aoArgsVariations.append(oArgVariation);
135	oArgVariation.oTestCase = oTestCase;
136	oArgVariation.oTestGroup = oTestGroup;
137
138	else:
139	assert len(self.aoTestCases) == 0;
140	assert len(self.aoArgsVariations) == 0;
141	# done.
142
143	@staticmethod
144	def _addPreReqError(aoErrors, aidChain, oObj, sMsg):
145	""" Returns a chain of IDs error entry. """
146
147	sMsg += ' Dependency chain: %s' % (aidChain[0],);
148	for i in range(1, len(aidChain)):
149	sMsg += ' -> %s' % (aidChain[i],);
150
151	aoErrors.append([sMsg, oObj]);
152	return aoErrors;
153
154	def checkForGroupDepCycles(self):
155	"""
156	Checks for testgroup depencency cycles and any missing testgroup
157	dependencies.
158	Returns array of errors (see SchedulderBase.recreateQueue()).
159	"""
160	aoErrors = list();
161	for oTestGroup in self.aoTestGroups:
162	idPreReq = oTestGroup.idTestGroupPreReq;
163	if idPreReq is None:
164	oTestGroup.aidTestGroupPreReqs = list();
165	continue;
166
167	aidChain = [oTestGroup.idTestGroup,];
168	while idPreReq is not None:
169	aidChain.append(idPreReq);
170	if len(aidChain) >= 10:
171	self._addPreReqError(aoErrors, aidChain, oTestGroup,
172	'TestGroup #%s prerequisite chain is too long!'
173	% (oTestGroup.idTestGroup,));
174	break;
175
176	oDep = self.dTestGroups.get(idPreReq, None);
177	if oDep is None:
178	self._addPreReqError(aoErrors, aidChain, oTestGroup,
179	'TestGroup #%s prerequisite #%s is not in the scheduling group!'
180	% (oTestGroup.idTestGroup, idPreReq,));
181	break;
182
183	idPreReq = oDep.idTestGroupPreReq;
184	oTestGroup.aidTestGroupPreReqs = aidChain[1:];
185
186	return aoErrors;
187
188
189	def checkForMissingTestCaseDeps(self):
190	"""
191	Checks that testcase dependencies stays within bounds. We do not allow
192	dependencies outside a testgroup, no dependency cycles or even remotely
193	long dependency chains.
194
195	Returns array of errors (see SchedulderBase.recreateQueue()).
196	"""
197	aoErrors = list();
198	for oTestGroup in self.aoTestGroups:
199	for oTestCase in oTestGroup.aoTestCases:
200	if len(oTestCase.aidPreReqs) == 0:
201	continue;
202
203	# Stupid recursion code using special stack(s).
204	aiIndexes = [(oTestCase, 0), ];
205	aidChain = [oTestCase.idTestGroup,];
206	while len(aiIndexes) > 0:
207	(oCur, i) = aiIndexes[-1];
208	if i >= len(oCur.aidPreReqs):
209	aiIndexes.pop();
210	aidChain.pop();
211	else:
212	aiIndexes[-1][1] = i + 1; # whatever happens, we'll advance on the current level.
213
214	idPreReq = oTestCase.aidPreReqs[i];
215	oDep = oTestGroup.dTestCases.get(idPreReq, None);
216	if oDep is None:
217	self._addPreReqError(aoErrors, aidChain, oTestCase,
218	'TestCase #%s prerequisite #%s is not in the scheduling group!'
219	% (oTestCase.idTestCase, idPreReq));
220	elif idPreReq in aidChain:
221	self._addPreReqError(aoErrors, aidChain, oTestCase,
222	'TestCase #%s prerequisite #%s creates a cycle!'
223	% (oTestCase.idTestCase, idPreReq));
224	elif len(oDep.aiPreReqs) == 0:
225	pass;
226	elif len(aidChain) >= 10:
227	self._addPreReqError(aoErrors, aidChain, oTestCase,
228	'TestCase #%s prerequisite chain is too long!' % (oTestCase.idTestCase,));
229	else:
230	aiIndexes.append((oDep, 0));
231	aidChain.append(idPreReq);
232
233	return aoErrors;
234
235	def deepTestGroupSort(self):
236	"""
237	Sorts the testgroups and their testcases by priority and dependencies.
238	Note! Don't call this before checking for dependency cycles!
239	"""
240	if len(self.aoTestGroups) == 0:
241	return;
242
243	#
244	# ASSUMES groups as well as testcases are sorted by priority by the
245	# database. So we only have to concern ourselves with the dependency
246	# sorting.
247	#
248	iGrpPrio = self.aoTestGroups[0].iSchedPriority;
249	for oTestGroup in self.aoTestGroups:
250	if oTestGroup.iSchedPriority > iGrpPrio:
251	raise TMExceptionBase('Incorrectly sorted testgroups returned by database.');
252	iGrpPrio = oTestGroup.iSchedPriority;
253
254	if len(oTestGroup.aoTestCases) > 0:
255	iTstPrio = oTestGroup.aoTestCases[0];
256	for oTestCase in oTestGroup.aoTestCases:
257	if oTestCase.iSchedPriority > iTstPrio:
258	raise TMExceptionBase('Incorrectly sorted testcases returned by database.');
259
260	#
261	# Sort the testgroups by dependencies.
262	#
263	i = 0;
264	while i < len(self.aoTestGroups):
265	oTestGroup = self.aoTestGroups[i];
266	if oTestGroup.idTestGroupPreReq is not None:
267	iPreReq = self.aoTestGroups.index(self.dTestGroups[oTestGroup.idTestGroupPreReq]);
268	if iPreReq > i:
269	# The prerequisite is after the current entry. Move the
270	# current entry so that it's following it's prereq entry.
271	self.aoTestGroups.insert(iPreReq + 1, oTestGroup);
272	self.aoTestGroups.pop(i);
273	continue;
274	assert iPreReq < i;
275	i += 1; # Advance.
276
277	#
278	# Sort the testcases by dependencies.
279	# Same algorithm as above, just more prerequisites.
280	#
281	for oTestGroup in self.aoTestGroups:
282	i = 0;
283	while i < len(oTestGroup.aoTestCases):
284	oTestCase = oTestGroup.aoTestCases[i];
285	if len(oTestCase.aidPreReqs) > 0:
286	for idPreReq in oTestCase.aidPreReqs:
287	iPreReq = oTestGroup.aoTestCases.index(oTestGroup.dTestCases[idPreReq]);
288	if iPreReq > i:
289	# The prerequisite is after the current entry. Move the
290	# current entry so that it's following it's prereq entry.
291	oTestGroup.aoTestGroups.insert(iPreReq + 1, oTestCase);
292	oTestGroup.aoTestGroups.pop(i);
293	i -= 1; # Don't advance.
294	break;
295	assert iPreReq < i;
296	i += 1; # Advance.
297
298
299	return True;
300
301
302
303	class SchedQueueData(ModelDataBase):
304	"""
305	Scheduling queue data item.
306	"""
307
308	ksIdAttr = 'idSchedGroup';
309
310	ksParam_idSchedGroup = 'SchedQueueData_idSchedGroup';
311	ksParam_idItem = 'SchedQueueData_idItem';
312	ksParam_offQueue = 'SchedQueueData_offQueue';
313	ksParam_idGenTestCaseArgs = 'SchedQueueData_idGenTestCaseArgs';
314	ksParam_idTestGroup = 'SchedQueueData_idTestGroup';
315	ksParam_aidTestGroupPreReqs = 'SchedQueueData_aidTestGroupPreReqs';
316	ksParam_bmHourlySchedule = 'SchedQueueData_bmHourlySchedule';
317	ksParam_tsConfig = 'SchedQueueData_tsConfig';
318	ksParam_tsLastScheduled = 'SchedQueueData_tsLastScheduled';
319	ksParam_idTestSetGangLeader = 'SchedQueueData_idTestSetGangLeader';
320	ksParam_cMissingGangMembers = 'SchedQueueData_cMissingGangMembers';
321
322	kasAllowNullAttributes = [ 'idItem', 'offQueue', 'aidTestGroupPreReqs', 'bmHourlySchedule', 'idTestSetGangLeader',
323	'tsConfig', 'tsLastScheduled' ];
324
325
326	def __init__(self):
327	ModelDataBase.__init__(self);
328
329	#
330	# Initialize with defaults.
331	# See the database for explanations of each of these fields.
332	#
333	self.idSchedGroup = None;
334	self.idItem = None;
335	self.offQueue = None;
336	self.idGenTestCaseArgs = None;
337	self.idTestGroup = None;
338	self.aidTestGroupPreReqs = None;
339	self.bmHourlySchedule = None;
340	self.tsConfig = None;
341	self.tsLastScheduled = None;
342	self.idTestSetGangLeader = None;
343	self.cMissingGangMembers = 1;
344
345	def initFromValues(self, idSchedGroup, idGenTestCaseArgs, idTestGroup, aidTestGroupPreReqs, # pylint: disable=R0913
346	bmHourlySchedule, cMissingGangMembers,
347	idItem = None, offQueue = None, tsConfig = None, tsLastScheduled = None, idTestSetGangLeader = None):
348	"""
349	Reinitialize with all attributes potentially given as inputs.
350	Return self.
351	"""
352	self.idSchedGroup = idSchedGroup;
353	self.idItem = idItem;
354	self.offQueue = offQueue;
355	self.idGenTestCaseArgs = idGenTestCaseArgs;
356	self.idTestGroup = idTestGroup;
357	self.aidTestGroupPreReqs = aidTestGroupPreReqs;
358	self.bmHourlySchedule = bmHourlySchedule;
359	self.tsConfig = tsConfig;
360	self.tsLastScheduled = tsLastScheduled;
361	self.idTestSetGangLeader = idTestSetGangLeader;
362	self.cMissingGangMembers = cMissingGangMembers;
363	return self;
364
365	def initFromDbRow(self, aoRow):
366	"""
367	Initialize from database row (SELECT * FROM SchedQueues).
368	Returns self.
369	Raises exception if no row is specfied.
370	"""
371	if aoRow is None:
372	raise TMExceptionBase('SchedQueueData not found.');
373
374	self.idSchedGroup = aoRow[0];
375	self.idItem = aoRow[1];
376	self.offQueue = aoRow[2];
377	self.idGenTestCaseArgs = aoRow[3];
378	self.idTestGroup = aoRow[4];
379	self.aidTestGroupPreReqs = aoRow[5];
380	self.bmHourlySchedule = aoRow[6];
381	self.tsConfig = aoRow[7];
382	self.tsLastScheduled = aoRow[8];
383	self.idTestSetGangLeader = aoRow[9];
384	self.cMissingGangMembers = aoRow[10];
385	return self;
386
387
388
389
390
391
392	class SchedulerBase(object):
393	"""
394	The scheduler base class.
395
396	The scheduler classes have two functions:
397	1. Recreate the scheduling queue.
398	2. Pick the next task from the queue.
399
400	The first is scheduler specific, the latter isn't.
401	"""
402
403	class BuildCache(object):
404	""" Build cache. """
405
406	class BuildCacheIterator(object):
407	""" Build class iterator. """
408	def __init__(self, oCache):
409	self.oCache = oCache;
410	self.iCur = 0;
411
412	def __iter__(self):
413	"""Returns self, required by the language."""
414	return self;
415
416	def next(self):
417	"""Returns the next build, raises StopIteration when the end has been reached."""
418	while True:
419	if self.iCur >= len(self.oCache.aoEntries):
420	oEntry = self.oCache.fetchFromCursor();
421	if oEntry is None:
422	raise StopIteration;
423	else:
424	oEntry = self.oCache.aoEntries[self.iCur];
425	self.iCur += 1;
426	if not oEntry.fRemoved:
427	return oEntry;
428	# end
429
430	class BuildCacheEntry(object):
431	""" Build cache entry. """
432
433	def __init__(self, oBuild, fMaybeBlacklisted):
434	self.oBuild = oBuild;
435	self._fBlacklisted = None if fMaybeBlacklisted is True else False;
436	self.fRemoved = False;
437	self._dPreReqDecisions = dict();
438
439	def remove(self):
440	"""
441	Marks the cache entry as removed.
442	This doesn't actually remove it from the cache array, only marks
443	it as removed. It has no effect on open iterators.
444	"""
445	self.fRemoved = True;
446
447	def getPreReqDecision(self, sPreReqSet):
448	"""
449	Retrieves a cached prerequisite decision.
450	Returns boolean if found, None if not.
451	"""
452	return self._dPreReqDecisions.get(sPreReqSet);
453
454	def setPreReqDecision(self, sPreReqSet, fDecision):
455	"""
456	Caches a prerequistie decision.
457	"""
458	self._dPreReqDecisions[sPreReqSet] = fDecision;
459	return fDecision;
460
461	def isBlacklisted(self, oDb):
462	""" Checks if the build is blacklisted. """
463	if self._fBlacklisted is None:
464	self._fBlacklisted = BuildLogic(oDb).isBuildBlacklisted(self.oBuild);
465	return self._fBlacklisted;
466
467
468	def __init__(self):
469	self.aoEntries = [];
470	self.oCursor = None;
471
472	def setupSource(self, oDb, idBuildSrc, sOs, sCpuArch, tsNow):
473	""" Configures the build cursor for the cache. """
474	if len(self.aoEntries) == 0 and self.oCursor is None:
475	oBuildSource = BuildSourceData().initFromDbWithId(oDb, idBuildSrc, tsNow);
476	self.oCursor = BuildSourceLogic(oDb).openBuildCursor(oBuildSource, sOs, sCpuArch, tsNow);
477	return True;
478
479	def __iter__(self):
480	"""Return an iterator."""
481	return self.BuildCacheIterator(self);
482
483	def fetchFromCursor(self):
484	""" Fetches a build from the cursor and adds it to the cache."""
485	if self.oCursor is None:
486	return None;
487
488	try:
489	aoRow = self.oCursor.fetchOne();
490	except:
491	return None;
492	if aoRow is None:
493	return None;
494
495	oBuild = BuildDataEx().initFromDbRow(aoRow);
496	oEntry = self.BuildCacheEntry(oBuild, aoRow[-1]);
497	self.aoEntries.append(oEntry);
498	return oEntry;
499
500	def __init__(self, oDb, oSchedGrpData, iVerbosity = 0, tsSecStart = None):
501	self._oDb = oDb;
502	self._oSchedGrpData = oSchedGrpData;
503	self._iVerbosity = iVerbosity;
504	self._asMessages = [];
505	self._tsSecStart = tsSecStart if tsSecStart is not None else utils.timestampSecond();
506	self.oBuildCache = self.BuildCache();
507	self.dTestGroupMembers = dict();
508
509	@staticmethod
510	def _instantiate(oDb, oSchedGrpData, iVerbosity = 0, tsSecStart = None):
511	"""
512	Instantiate the scheduler specified by the scheduling group.
513	Returns scheduler child class instance. May raise exception if
514	the input is invalid.
515	"""
516	if oSchedGrpData.enmScheduler == SchedGroupData.ksScheduler_BestEffortContinuousIntegration:
517	from testmanager.core.schedulerbeci import SchdulerBeci;
518	oScheduler = SchdulerBeci(oDb, oSchedGrpData, iVerbosity, tsSecStart);
519	else:
520	raise oDb.integrityException('Invalid scheduler "%s", idSchedGroup=%d' \
521	% (oSchedGrpData.enmScheduler, oSchedGrpData.idSchedGroup));
522	return oScheduler;
523
524
525	#
526	# Misc.
527	#
528
529	def msgDebug(self, sText):
530	"""Debug printing."""
531	if self._iVerbosity > 1:
532	self._asMessages.append('debug:' + sText);
533	return None;
534
535	def msgInfo(self, sText):
536	"""Info printing."""
537	if self._iVerbosity > 1:
538	self._asMessages.append('info: ' + sText);
539	return None;
540
541	def dprint(self, sMsg):
542	"""Prints a debug message to the srv glue log (see config.py). """
543	if config.g_kfSrvGlueDebugScheduler:
544	self._oDb.dprint(sMsg);
545	return None;
546
547	def getElapsedSecs(self):
548	""" Returns the number of seconds this scheduling task has been running. """
549	tsSecNow = utils.timestampSecond();
550	if tsSecNow < self._tsSecStart: # paranoia
551	self._tsSecStart = tsSecNow;
552	return tsSecNow - self._tsSecStart;
553
554
555	#
556	# Create schedule.
557	#
558
559	def _recreateQueueCancelGatherings(self):
560	"""
561	Cancels all pending gang gatherings on the current queue.
562	"""
563	self._oDb.execute('SELECT idTestSetGangLeader\n'
564	'FROM SchedQueues\n'
565	'WHERE idSchedGroup = %s\n'
566	' AND idTestSetGangLeader is not NULL\n'
567	, (self._oSchedGrpData.idSchedGroup,));
568	if self._oDb.getRowCount() > 0:
569	oTBStatusLogic = TestBoxStatusLogic(self._oDb);
570	for aoRow in self._oDb.fetchAll():
571	idTestSetGangLeader = aoRow[0];
572	oTBStatusLogic.updateGangStatus(idTestSetGangLeader,
573	TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut,
574	fCommit = False);
575	return True;
576
577	def _recreateQueueItems(self, oData):
578	"""
579	Returns an array of queue items (SchedQueueData).
580	Child classes must override this.
581	"""
582	_ = oData;
583	return [];
584
585	def recreateQueueWorker(self):
586	"""
587	Worker for recreateQueue.
588	"""
589
590	#
591	# Collect the necessary data and validate it.
592	#
593	oData = ReCreateQueueData(self._oDb, self._oSchedGrpData.idSchedGroup);
594	aoErrors = oData.checkForGroupDepCycles();
595	aoErrors.extend(oData.checkForMissingTestCaseDeps());
596	if len(aoErrors) == 0:
597	oData.deepTestGroupSort();
598
599	#
600	# The creation of the scheduling queue is done by the child class.
601	#
602	# We will try guess where in queue we're currently at and rotate
603	# the items such that we will resume execution in the approximately
604	# same position. The goal of the scheduler is to provide a 100%
605	# deterministic result so that if we regenerate the queue when there
606	# are no changes to the testcases, testgroups or scheduling groups
607	# involved, test execution will be unchanged (save for maybe just a
608	# little for gang gathering).
609	#
610	aoItems = list();
611	if len(oData.aoArgsVariations) > 0:
612	aoItems = self._recreateQueueItems(oData);
613	self.msgDebug('len(aoItems)=%s' % (len(aoItems),));
614	#for i in range(len(aoItems)):
615	# self.msgDebug('aoItems[%2d]=%s' % (i, aoItems[i]));
616	if len(aoItems) > 0:
617	self._oDb.execute('SELECT offQueue FROM SchedQueues WHERE idSchedGroup = %s ORDER BY idItem LIMIT 1'
618	, (self._oSchedGrpData.idSchedGroup,));
619	if self._oDb.getRowCount() > 0:
620	offQueue = self._oDb.fetchOne()[0];
621	self._oDb.execute('SELECT COUNT(*) FROM SchedQueues WHERE idSchedGroup = %s'
622	, (self._oSchedGrpData.idSchedGroup,));
623	cItems = self._oDb.fetchOne()[0];
624	offQueueNew = (offQueue * cItems) / len(aoItems);
625	if offQueueNew != 0:
626	aoItems = aoItems[offQueueNew:] + aoItems[:offQueueNew];
627
628	#
629	# Replace the scheduling queue.
630	# Care need to be take to first timeout/abort any gangs in the
631	# gathering state since these use the queue to set up the date.
632	#
633	self._recreateQueueCancelGatherings();
634	self._oDb.execute('DELETE FROM SchedQueues WHERE idSchedGroup = %s\n', (self._oSchedGrpData.idSchedGroup,));
635	self._oDb.insertList('INSERT INTO SchedQueues (\n'
636	' idSchedGroup,\n'
637	' offQueue,\n'
638	' idGenTestCaseArgs,\n'
639	' idTestGroup,\n'
640	' aidTestGroupPreReqs,\n'
641	' bmHourlySchedule,\n'
642	' cMissingGangMembers )\n',
643	aoItems, self._formatItemForInsert);
644	return (aoErrors, self._asMessages);
645
646	def _formatItemForInsert(self, oItem):
647	"""
648	Used by recreateQueueWorker together with TMDatabaseConnect::insertList
649	"""
650	return self._oDb.formatBindArgs('(%s,%s,%s,%s,%s,%s,%s)'
651	, ( oItem.idSchedGroup,
652	oItem.offQueue,
653	oItem.idGenTestCaseArgs,
654	oItem.idTestGroup,
655	oItem.aidTestGroupPreReqs if len(oItem.aidTestGroupPreReqs) > 0 else None,
656	oItem.bmHourlySchedule,
657	oItem.cMissingGangMembers
658	));
659
660	@staticmethod
661	def recreateQueue(oDb, uidAuthor, idSchedGroup, iVerbosity = 1):
662	"""
663	(Re-)creates the scheduling queue for the given group.
664
665	Returns (asMessages, asMessages). On success the array with the error
666	will be empty, on failure it will contain (sError, oRelatedObject)
667	entries. The messages is for debugging and are simple strings.
668
669	Raises exception database error.
670	"""
671
672	aoExtraMsgs = [];
673	if oDb.debugIsExplainEnabled():
674	aoExtraMsgs += ['Warning! Disabling SQL explain to avoid deadlocking against locked tables.'];
675	oDb.debugDisableExplain();
676
677	aoErrors = [];
678	asMessages = [];
679	try:
680	#
681	# To avoid concurrency issues (SchedQueues) and inconsistent data (*),
682	# we lock quite a few tables while doing this work. We access more
683	# data than scheduleNewTask so we lock some additional tables.
684	#
685	oDb.rollback();
686	oDb.begin();
687	oDb.execute('LOCK TABLE SchedGroups, SchedGroupMembers, TestGroups, TestGroupMembers IN SHARE MODE');
688	oDb.execute('LOCK TABLE TestBoxes, TestCaseArgs, TestCases IN SHARE MODE');
689	oDb.execute('LOCK TABLE TestBoxStatuses, SchedQueues IN EXCLUSIVE MODE');
690
691	#
692	# Instantiate the scheduler and call the worker function.
693	#
694	oSchedGrpData = SchedGroupData().initFromDbWithId(oDb, idSchedGroup);
695	oScheduler = SchedulerBase._instantiate(oDb, oSchedGrpData, iVerbosity);
696
697	(aoErrors, asMessages) = oScheduler.recreateQueueWorker();
698	if len(aoErrors) == 0:
699	SystemLogLogic(oDb).addEntry(SystemLogData.ksEvent_SchedQueueRecreate,
700	'User #%d recreated sched queue #%d.' % (uidAuthor, idSchedGroup,));
701	oDb.commit();
702	else:
703	oDb.rollback();
704
705	except:
706	oDb.rollback();
707	raise;
708
709	return (aoErrors, aoExtraMsgs + asMessages);
710
711
712
713	#
714	# Schedule Task.
715	#
716
717	def _composeGangArguments(self, idTestSet):
718	"""
719	Composes the gang specific testdriver arguments.
720	Returns command line string, including a leading space.
721	"""
722
723	oTestSet = TestSetData().initFromDbWithId(self._oDb, idTestSet);
724	aoGangMembers = TestSetLogic(self._oDb).getGang(oTestSet.idTestSetGangLeader);
725
726	sArgs = ' --gang-member-no %s --gang-members %s' % (oTestSet.iGangMemberNo, len(aoGangMembers));
727	for i, _ in enumerate(aoGangMembers):
728	sArgs = ' --gang-ipv4-%s %s' % (i, aoGangMembers[i].ip); ## @todo IPv6
729
730	return sArgs;
731
732
733	def composeExecResponseWorker(self, idTestSet, oTestEx, oTestBox, oBuild, oValidationKitBuild, sBaseUrl):
734	"""
735	Given all the bits of data, compose an EXEC command response to the testbox.
736	"""
737	sScriptZips = oTestEx.oTestCase.sValidationKitZips;
738	if sScriptZips is None or sScriptZips.find('@VALIDATIONKIT_ZIP@') >= 0:
739	assert oValidationKitBuild;
740	if sScriptZips is None:
741	sScriptZips = oValidationKitBuild.sBinaries;
742	else:
743	sScriptZips = sScriptZips.replace('@VALIDATIONKIT_ZIP@', oValidationKitBuild.sBinaries);
744	sScriptZips = sScriptZips.replace('@DOWNLOAD_BASE_URL@', sBaseUrl + config.g_ksTmDownloadBaseUrlRel);
745
746	sCmdLine = oTestEx.oTestCase.sBaseCmd + ' ' + oTestEx.sArgs;
747	sCmdLine = sCmdLine.replace('@BUILD_BINARIES@', oBuild.sBinaries);
748	sCmdLine = sCmdLine.strip();
749	if oTestEx.cGangMembers > 1:
750	sCmdLine += ' ' + self._composeGangArguments(idTestSet);
751
752	cSecTimeout = oTestEx.cSecTimeout if oTestEx.cSecTimeout is not None else oTestEx.oTestCase.cSecTimeout;
753	cSecTimeout = cSecTimeout * oTestBox.pctScaleTimeout / 100;
754
755	dResponse = \
756	{
757	constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.CMD_EXEC,
758	constants.tbresp.EXEC_PARAM_RESULT_ID: idTestSet,
759	constants.tbresp.EXEC_PARAM_SCRIPT_ZIPS: sScriptZips,
760	constants.tbresp.EXEC_PARAM_SCRIPT_CMD_LINE: sCmdLine,
761	constants.tbresp.EXEC_PARAM_TIMEOUT: cSecTimeout,
762	};
763	return dResponse;
764
765	@staticmethod
766	def composeExecResponse(oDb, idTestSet, sBaseUrl, iVerbosity = 0):
767	"""
768	Composes an EXEC response for a gang member (other than the last).
769	Returns a EXEC response or raises an exception (DB/input error).
770	"""
771	#
772	# Gather the necessary data.
773	#
774	oTestSet = TestSetData().initFromDbWithId(oDb, idTestSet);
775	oTestBox = TestBoxData().initFromDbWithGenId(oDb, oTestSet.idGenTestBox);
776	oTestEx = TestCaseArgsDataEx().initFromDbWithGenId(oDb, oTestSet.idGenTestCaseArgs);
777	oBuild = BuildDataEx().initFromDbWithId(oDb, oTestSet.idBuild);
778	oValidationKitBuild = None;
779	if oTestSet.idBuildTestSuite is not None:
780	oValidationKitBuild = BuildDataEx().initFromDbWithId(oDb, oTestSet.idBuildTestSuite);
781
782	#
783	# Instantiate the specified scheduler and let it do the rest.
784	#
785	oSchedGrpData = SchedGroupData().initFromDbWithId(oDb, oTestSet.idSchedGroup, oTestSet.tsCreated);
786	assert oSchedGrpData.fEnabled is True;
787	assert oSchedGrpData.idBuildSrc is not None;
788	oScheduler = SchedulerBase._instantiate(oDb, oSchedGrpData, iVerbosity);
789
790	return oScheduler.composeExecResponseWorker(idTestSet, oTestEx, oTestBox, oBuild, oValidationKitBuild, sBaseUrl);
791
792
793	def _updateTask(self, oTask, tsNow):
794	"""
795	Updates a gang schedule task.
796	"""
797	assert oTask.cMissingGangMembers >= 1;
798	assert oTask.idTestSetGangLeader is not None;
799	assert oTask.idTestSetGangLeader >= 1;
800	if tsNow is not None:
801	self._oDb.execute('UPDATE SchedQueues\n'
802	' SET idTestSetGangLeader = %s,\n'
803	' cMissingGangMembers = %s,\n'
804	' tsLastScheduled = %s\n'
805	'WHERE idItem = %s\n'
806	, (oTask.idTestSetGangLeader, oTask.cMissingGangMembers, tsNow, oTask.idItem,) );
807	else:
808	self._oDb.execute('UPDATE SchedQueues\n'
809	' SET cMissingGangMembers = %s\n'
810	'WHERE idItem = %s\n'
811	, (oTask.cMissingGangMembers, oTask.idItem,) );
812	return True;
813
814	def _moveTaskToEndOfQueue(self, oTask, cGangMembers, tsNow):
815	"""
816	The task has been scheduled successfully, reset it's data move it to
817	the end of the queue.
818	"""
819	if cGangMembers > 1:
820	self._oDb.execute('UPDATE SchedQueues\n'
821	' SET idItem = NEXTVAL(\'SchedQueueItemIdSeq\'),\n'
822	' idTestSetGangLeader = NULL,\n'
823	' cMissingGangMembers = %s\n'
824	'WHERE idItem = %s\n'
825	, (cGangMembers, oTask.idItem,) );
826	else:
827	self._oDb.execute('UPDATE SchedQueues\n'
828	' SET idItem = NEXTVAL(\'SchedQueueItemIdSeq\'),\n'
829	' idTestSetGangLeader = NULL,\n'
830	' cMissingGangMembers = 1,\n'
831	' tsLastScheduled = %s\n'
832	'WHERE idItem = %s\n'
833	, (tsNow, oTask.idItem,) );
834	return True;
835
836
837
838
839	def _createTestSet(self, oTask, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, tsNow):
840	# type: (SchedQueueData, TestCaseArgsDataEx, TestBoxData, BuildDataEx, BuildDataEx, datetime.datetime) -> int
841	"""
842	Creates a test set for using the given data.
843	Will not commit, someone up the callstack will that later on.
844
845	Returns the test set ID, may raise an exception on database error.
846	"""
847	# Lazy bird doesn't want to write testset.py and does it all here.
848
849	#
850	# We're getting the TestSet ID first in order to include it in the base
851	# file name (that way we can directly relate files on the disk to the
852	# test set when doing batch work), and also for idTesetSetGangLeader.
853	#
854	self._oDb.execute('SELECT NEXTVAL(\'TestSetIdSeq\')');
855	idTestSet = self._oDb.fetchOne()[0];
856
857	sBaseFilename = '%04d/%02d/%02d/%02d/TestSet-%s' \
858	% (tsNow.year, tsNow.month, tsNow.day, (tsNow.hour / 6) * 6, idTestSet);
859
860	#
861	# Gang scheduling parameters. Changes the oTask data for updating by caller.
862	#
863	iGangMemberNo = 0;
864
865	if oTestEx.cGangMembers <= 1:
866	assert oTask.idTestSetGangLeader is None;
867	assert oTask.cMissingGangMembers <= 1;
868	elif oTask.idTestSetGangLeader is None:
869	assert oTask.cMissingGangMembers == oTestEx.cGangMembers;
870	oTask.cMissingGangMembers = oTestEx.cGangMembers - 1;
871	oTask.idTestSetGangLeader = idTestSet;
872	else:
873	assert oTask.cMissingGangMembers > 0 and oTask.cMissingGangMembers < oTestEx.cGangMembers;
874	oTask.cMissingGangMembers -= 1;
875
876	#
877	# Do the database stuff.
878	#
879	self._oDb.execute('INSERT INTO TestSets (\n'
880	' idTestSet,\n'
881	' tsConfig,\n'
882	' tsCreated,\n'
883	' idBuild,\n'
884	' idBuildCategory,\n'
885	' idBuildTestSuite,\n'
886	' idGenTestBox,\n'
887	' idTestBox,\n'
888	' idSchedGroup,\n'
889	' idTestGroup,\n'
890	' idGenTestCase,\n'
891	' idTestCase,\n'
892	' idGenTestCaseArgs,\n'
893	' idTestCaseArgs,\n'
894	' sBaseFilename,\n'
895	' iGangMemberNo,\n'
896	' idTestSetGangLeader )\n'
897	'VALUES ( %s,\n' # idTestSet
898	' %s,\n' # tsConfig
899	' %s,\n' # tsCreated
900	' %s,\n' # idBuild
901	' %s,\n' # idBuildCategory
902	' %s,\n' # idBuildTestSuite
903	' %s,\n' # idGenTestBox
904	' %s,\n' # idTestBox
905	' %s,\n' # idSchedGroup
906	' %s,\n' # idTestGroup
907	' %s,\n' # idGenTestCase
908	' %s,\n' # idTestCase
909	' %s,\n' # idGenTestCaseArgs
910	' %s,\n' # idTestCaseArgs
911	' %s,\n' # sBaseFilename
912	' %s,\n' # iGangMemberNo
913	' %s)\n' # idTestSetGangLeader
914	, ( idTestSet,
915	oTask.tsConfig,
916	tsNow,
917	oBuild.idBuild,
918	oBuild.idBuildCategory,
919	oValidationKitBuild.idBuild if oValidationKitBuild is not None else None,
920	oTestBoxData.idGenTestBox,
921	oTestBoxData.idTestBox,
922	oTask.idSchedGroup,
923	oTask.idTestGroup,
924	oTestEx.oTestCase.idGenTestCase,
925	oTestEx.oTestCase.idTestCase,
926	oTestEx.idGenTestCaseArgs,
927	oTestEx.idTestCaseArgs,
928	sBaseFilename,
929	iGangMemberNo,
930	oTask.idTestSetGangLeader,
931	));
932
933	self._oDb.execute('INSERT INTO TestResults (\n'
934	' idTestResultParent,\n'
935	' idTestSet,\n'
936	' tsCreated,\n'
937	' idStrName,\n'
938	' cErrors,\n'
939	' enmStatus,\n'
940	' iNestingDepth)\n'
941	'VALUES ( NULL,\n' # idTestResultParent
942	' %s,\n' # idTestSet
943	' %s,\n' # tsCreated
944	' 0,\n' # idStrName
945	' 0,\n' # cErrors
946	' \'running\'::TestStatus_T,\n'
947	' 0)\n' # iNestingDepth
948	'RETURNING idTestResult'
949	, ( idTestSet, tsNow, ));
950	idTestResult = self._oDb.fetchOne()[0];
951
952	self._oDb.execute('UPDATE TestSets\n'
953	' SET idTestResult = %s\n'
954	'WHERE idTestSet = %s\n'
955	, (idTestResult, idTestSet, ));
956
957	return idTestSet;
958
959	def _tryFindValidationKitBit(self, oTestBoxData, tsNow):
960	"""
961	Tries to find the most recent validation kit build suitable for the given testbox.
962	Returns BuildDataEx or None. Raise exception on database error.
963
964	Can be overridden by child classes to change the default build requirements.
965	"""
966	oBuildLogic = BuildLogic(self._oDb);
967	oBuildSource = BuildSourceData().initFromDbWithId(self._oDb, self._oSchedGrpData.idBuildSrcTestSuite, tsNow);
968	oCursor = BuildSourceLogic(self._oDb).openBuildCursor(oBuildSource, oTestBoxData.sOs, oTestBoxData.sCpuArch, tsNow);
969	for _ in range(oCursor.getRowCount()):
970	oBuild = BuildDataEx().initFromDbRow(oCursor.fetchOne());
971	if not oBuildLogic.isBuildBlacklisted(oBuild):
972	return oBuild;
973	return None;
974
975	def _tryFindBuild(self, oTask, oTestEx, oTestBoxData, tsNow):
976	"""
977	Tries to find a fitting build.
978	Returns BuildDataEx or None. Raise exception on database error.
979
980	Can be overridden by child classes to change the default build requirements.
981	"""
982
983	#
984	# Gather the set of prerequisites we have and turn them into a value
985	# set for use in the loop below.
986	#
987	# Note! We're scheduling on testcase level and ignoring argument variation
988	# selections in TestGroupMembers is intentional.
989	#
990	dPreReqs = {};
991
992	# Direct prerequisites. We assume they're all enabled as this can be
993	# checked at queue creation time.
994	for oPreReq in oTestEx.aoTestCasePreReqs:
995	dPreReqs[oPreReq.idTestCase] = 1;
996
997	# Testgroup dependencies from the scheduling group config.
998	if oTask.aidTestGroupPreReqs is not None:
999	for iTestGroup in oTask.aidTestGroupPreReqs:
1000	# Make sure the _active_ test group members are in the cache.
1001	if iTestGroup not in self.dTestGroupMembers:
1002	self._oDb.execute('SELECT DISTINCT TestGroupMembers.idTestCase\n'
1003	'FROM TestGroupMembers, TestCases\n'
1004	'WHERE TestGroupMembers.idTestGroup = %s\n'
1005	' AND TestGroupMembers.tsExpire > %s\n'
1006	' AND TestGroupMembers.tsEffective <= %s\n'
1007	' AND TestCases.idTestCase = TestGroupMembers.idTestCase\n'
1008	' AND TestCases.tsExpire > %s\n'
1009	' AND TestCases.tsEffective <= %s\n'
1010	' AND TestCases.fEnabled is TRUE\n'
1011	, (iTestGroup, oTask.tsConfig, oTask.tsConfig, oTask.tsConfig, oTask.tsConfig,));
1012	aidTestCases = [];
1013	for aoRow in self._oDb.fetchAll():
1014	aidTestCases.append(aoRow[0]);
1015	self.dTestGroupMembers[iTestGroup] = aidTestCases;
1016
1017	# Add the testgroup members to the prerequisites.
1018	for idTestCase in self.dTestGroupMembers[iTestGroup]:
1019	dPreReqs[idTestCase] = 1;
1020
1021	# Create a SQL values table out of them.
1022	sPreReqSet = ''
1023	if len(dPreReqs) > 0:
1024	for idPreReq in sorted(dPreReqs.keys()):
1025	sPreReqSet += ', (' + str(idPreReq) + ')';
1026	sPreReqSet = sPreReqSet[2:]; # drop the leading ', '.
1027
1028	#
1029	# Try the builds.
1030	#
1031	self.oBuildCache.setupSource(self._oDb, self._oSchedGrpData.idBuildSrc, oTestBoxData.sOs, oTestBoxData.sCpuArch, tsNow);
1032	for oEntry in self.oBuildCache:
1033	#
1034	# Check build requirements set by the test.
1035	#
1036	if not oTestEx.matchesBuildProps(oEntry.oBuild):
1037	continue;
1038
1039	if oEntry.isBlacklisted(self._oDb):
1040	oEntry.remove();
1041	continue;
1042
1043	#
1044	# Check prerequisites. The default scheduler is satisfied if one
1045	# argument variation has been executed successfully. It is not
1046	# satisfied if there are any failure runs.
1047	#
1048	if len(sPreReqSet) > 0:
1049	fDecision = oEntry.getPreReqDecision(sPreReqSet);
1050	if fDecision is None:
1051	# Check for missing prereqs.
1052	self._oDb.execute('SELECT COUNT(*)\n'
1053	'FROM (VALUES ' + sPreReqSet + ') AS PreReqs(idTestCase)\n'
1054	'LEFT OUTER JOIN (SELECT idTestSet\n'
1055	' FROM TestSets\n'
1056	' WHERE enmStatus IN (%s, %s)\n'
1057	' AND idBuild = %s\n'
1058	' ) AS TestSets\n'
1059	' ON (PreReqs.idTestCase = TestSets.idTestCase)\n'
1060	'WHERE TestSets.idTestSet is NULL\n'
1061	, ( TestSetData.ksTestStatus_Success, TestSetData.ksTestStatus_Skipped,
1062	oEntry.oBuild.idBuild, ));
1063	cMissingPreReqs = self._oDb.fetchOne()[0];
1064	if cMissingPreReqs > 0:
1065	self.dprint('build %s is missing %u prerequisites (out of %s)'
1066	% (oEntry.oBuild.idBuild, cMissingPreReqs, sPreReqSet,));
1067	oEntry.setPreReqDecision(sPreReqSet, False);
1068	continue;
1069
1070	# Check for failed prereq runs.
1071	self._oDb.execute('SELECT COUNT(*)\n'
1072	'FROM (VALUES ' + sPreReqSet + ') AS PreReqs(idTestCase),\n'
1073	' TestSets\n'
1074	'WHERE PreReqs.idTestCase = TestSets.idTestCase\n'
1075	' AND TestSets.idBuild = %s\n'
1076	' AND TestSets.enmStatus IN (%s, %s, %s)\n'
1077	, ( oEntry.oBuild.idBuild,
1078	TestSetData.ksTestStatus_Failure,
1079	TestSetData.ksTestStatus_TimedOut,
1080	TestSetData.ksTestStatus_Rebooted,
1081	)
1082	);
1083	cFailedPreReqs = self._oDb.fetchOne()[0];
1084	if cFailedPreReqs > 0:
1085	self.dprint('build %s is has %u prerequisite failures (out of %s)'
1086	% (oEntry.oBuild.idBuild, cFailedPreReqs, sPreReqSet,));
1087	oEntry.setPreReqDecision(sPreReqSet, False);
1088	continue;
1089
1090	oEntry.setPreReqDecision(sPreReqSet, True);
1091	elif not fDecision:
1092	continue;
1093
1094	#
1095	# If we can, check if the build files still exist.
1096	#
1097	if oEntry.oBuild.areFilesStillThere() is False:
1098	self.dprint('build %s no longer exists' % (oEntry.oBuild.idBuild,));
1099	oEntry.remove();
1100	continue;
1101
1102	self.dprint('found oBuild=%s' % (oEntry.oBuild,));
1103	return oEntry.oBuild;
1104	return None;
1105
1106	def _tryFindMatchingBuild(self, oLeaderBuild, oTestBoxData, idBuildSrc):
1107	"""
1108	Tries to find a matching build for gang scheduling.
1109	Returns BuildDataEx or None. Raise exception on database error.
1110
1111	Can be overridden by child classes to change the default build requirements.
1112	"""
1113	#
1114	# Note! Should probably check build prerequisites if we get a different
1115	# build back, so that we don't use a build which hasn't passed
1116	# the smoke test.
1117	#
1118	_ = idBuildSrc;
1119	return BuildLogic(self._oDb).tryFindSameBuildForOsArch(oLeaderBuild, oTestBoxData.sOs, oTestBoxData.sCpuArch);
1120
1121
1122	def _tryAsLeader(self, oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl):
1123	"""
1124	Try schedule the task as a gang leader (can be a gang of one).
1125	Returns response or None. May raise exception on DB error.
1126	"""
1127
1128	# We don't wait for busy resources, we just try the next test.
1129	oTestArgsLogic = TestCaseArgsLogic(self._oDb);
1130	if not oTestArgsLogic.areResourcesFree(oTestEx):
1131	self.dprint('Cannot get global test resources!');
1132	return None;
1133
1134	#
1135	# Find a matching build (this is the difficult bit).
1136	#
1137	oBuild = self._tryFindBuild(oTask, oTestEx, oTestBoxData, tsNow);
1138	if oBuild is None:
1139	self.dprint('No build!');
1140	return None;
1141	if oTestEx.oTestCase.needValidationKitBit():
1142	oValidationKitBuild = self._tryFindValidationKitBit(oTestBoxData, tsNow);
1143	if oValidationKitBuild is None:
1144	self.dprint('No validation kit build!');
1145	return None;
1146	else:
1147	oValidationKitBuild = None;
1148
1149	#
1150	# Create a testset, allocate the resources and update the state.
1151	# Note! Since resource allocation may still fail, we create a nested
1152	# transaction so we can roll back. (Heed lock warning in docs!)
1153	#
1154	self._oDb.execute('SAVEPOINT tryAsLeader');
1155	idTestSet = self._createTestSet(oTask, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, tsNow);
1156
1157	if GlobalResourceLogic(self._oDb).allocateResources(oTestBoxData.idTestBox, oTestEx.aoGlobalRsrc, fCommit = False) \
1158	is not True:
1159	self._oDb.execute('ROLLBACK TO SAVEPOINT tryAsLeader');
1160	self.dprint('Failed to allocate global resources!');
1161	return False;
1162
1163	if oTestEx.cGangMembers <= 1:
1164	# We're alone, put the task back at the end of the queue and issue EXEC cmd.
1165	self._moveTaskToEndOfQueue(oTask, oTestEx.cGangMembers, tsNow);
1166	dResponse = self.composeExecResponseWorker(idTestSet, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, sBaseUrl);
1167	sTBState = TestBoxStatusData.ksTestBoxState_Testing;
1168	else:
1169	# We're missing gang members, issue WAIT cmd.
1170	self._updateTask(oTask, tsNow if idTestSet == oTask.idTestSetGangLeader else None);
1171	dResponse = { constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.CMD_WAIT, };
1172	sTBState = TestBoxStatusData.ksTestBoxState_GangGathering;
1173
1174	TestBoxStatusLogic(self._oDb).updateState(oTestBoxData.idTestBox, sTBState, idTestSet, fCommit = False);
1175	self._oDb.execute('RELEASE SAVEPOINT tryAsLeader');
1176	return dResponse;
1177
1178	def _tryAsGangMember(self, oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl):
1179	"""
1180	Try schedule the task as a gang member.
1181	Returns response or None. May raise exception on DB error.
1182	"""
1183
1184	#
1185	# The leader has choosen a build, we need to find a matching one for our platform.
1186	# (It's up to the scheduler decide upon how strict dependencies are to be enforced
1187	# upon subordinate group members.)
1188	#
1189	oLeaderTestSet = TestSetData().initFromDbWithId(self._oDb, oTestBoxData.idTestSetGangLeader);
1190
1191	oLeaderBuild = BuildDataEx().initFromDbWithId(self._oDb, oLeaderTestSet.idBuild);
1192	oBuild = self._tryFindMatchingBuild(oLeaderBuild, oTestBoxData, self._oSchedGrpData.idBuildSrc);
1193	if oBuild is None:
1194	return None;
1195
1196	oValidationKitBuild = None;
1197	if oLeaderTestSet.idBuildTestSuite is not None:
1198	oLeaderValidationKitBit = BuildDataEx().initFromDbWithId(self._oDb, oLeaderTestSet.idBuildTestSuite);
1199	oValidationKitBuild = self._tryFindMatchingBuild(oLeaderValidationKitBit, oTestBoxData,
1200	self._oSchedGrpData.idBuildSrcTestSuite);
1201
1202	#
1203	# Create a testset and update the state(s).
1204	#
1205	idTestSet = self._createTestSet(oTask, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, tsNow);
1206
1207	oTBStatusLogic = TestBoxStatusLogic(self._oDb);
1208	if oTask.cMissingGangMembers < 1:
1209	# The whole gang is there, move the task to the end of the queue
1210	# and update the status on the other gang members.
1211	self._moveTaskToEndOfQueue(oTask, oTestEx.cGangMembers, tsNow);
1212	dResponse = self.composeExecResponseWorker(idTestSet, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, sBaseUrl);
1213	sTBState = TestBoxStatusData.ksTestBoxState_GangTesting;
1214	oTBStatusLogic.updateGangStatus(oTask.idTestSetGangLeader, sTBState, fCommit = False);
1215	else:
1216	# We're still missing some gang members, issue WAIT cmd.
1217	self._updateTask(oTask, tsNow if idTestSet == oTask.idTestSetGangLeader else None);
1218	dResponse = { constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.CMD_WAIT, };
1219	sTBState = TestBoxStatusData.ksTestBoxState_GangGathering;
1220
1221	oTBStatusLogic.updateState(oTestBoxData.idTestBox, sTBState, idTestSet, fCommit = False);
1222	return dResponse;
1223
1224
1225	def scheduleNewTaskWorker(self, oTestBoxData, tsNow, sBaseUrl):
1226	"""
1227	Worker for schduling a new task.
1228	"""
1229
1230	#
1231	# Iterate the scheduler queue (fetch all to avoid having to concurrent
1232	# queries), trying out each task to see if the testbox can execute it.
1233	#
1234	dRejected = {}; # variations we've already checked out and rejected.
1235	self._oDb.execute('SELECT *\n'
1236	'FROM SchedQueues\n'
1237	'WHERE idSchedGroup = %s\n'
1238	' AND ( bmHourlySchedule IS NULL\n'
1239	' OR get_bit(bmHourlySchedule, %s) = 1 )\n'
1240	'ORDER BY idItem ASC\n'
1241	, (self._oSchedGrpData.idSchedGroup, utils.getLocalHourOfWeek()) );
1242	aaoRows = self._oDb.fetchAll();
1243	for aoRow in aaoRows:
1244	# Don't loop forever.
1245	if self.getElapsedSecs() >= config.g_kcSecMaxNewTask:
1246	break;
1247
1248	# Unpack the data and check if we've rejected the testcasevar/group variation already (they repeat).
1249	oTask = SchedQueueData().initFromDbRow(aoRow);
1250	if config.g_kfSrvGlueDebugScheduler:
1251	self.dprint('** Considering: idItem=%s idGenTestCaseArgs=%s idTestGroup=%s Deps=%s last=%s cfg=%s\n'
1252	% ( oTask.idItem, oTask.idGenTestCaseArgs, oTask.idTestGroup, oTask.aidTestGroupPreReqs,
1253	oTask.tsLastScheduled, oTask.tsConfig,));
1254
1255	sRejectNm = '%s:%s' % (oTask.idGenTestCaseArgs, oTask.idTestGroup,);
1256	if sRejectNm in dRejected:
1257	self.dprint('Duplicate, already rejected! (%s)' % (sRejectNm,));
1258	continue;
1259	dRejected[sRejectNm] = 1;
1260
1261	# Fetch all the test case info (too much, but who cares right now).
1262	oTestEx = TestCaseArgsDataEx().initFromDbWithGenIdEx(self._oDb, oTask.idGenTestCaseArgs,
1263	tsConfigEff = oTask.tsConfig,
1264	tsRsrcEff = oTask.tsConfig);
1265	if config.g_kfSrvGlueDebugScheduler:
1266	self.dprint('TestCase "%s": %s %s' % (oTestEx.oTestCase.sName, oTestEx.oTestCase.sBaseCmd, oTestEx.sArgs,));
1267
1268	# This shouldn't happen, but just in case it does...
1269	if oTestEx.oTestCase.fEnabled is not True:
1270	self.dprint('Testcase is not enabled!!');
1271	continue;
1272
1273	# Check if the testbox properties matches the test.
1274	if not oTestEx.matchesTestBoxProps(oTestBoxData):
1275	self.dprint('Testbox mismatch!');
1276	continue;
1277
1278	# Try schedule it.
1279	if oTask.idTestSetGangLeader is None or oTestEx.cGangMembers <= 1:
1280	dResponse = self._tryAsLeader(oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl);
1281	elif oTask.cMissingGangMembers > 1:
1282	dResponse = self._tryAsGangMember(oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl);
1283	else:
1284	dResponse = None; # Shouldn't happen!
1285	if dResponse is not None:
1286	self.dprint('Found a task! dResponse=%s' % (dResponse,));
1287	return dResponse;
1288
1289	# Found no suitable task.
1290	return None;
1291
1292	@staticmethod
1293	def _pickSchedGroup(oTestBoxDataEx, iWorkItem, dIgnoreSchedGroupIds):
1294	"""
1295	Picks the next scheduling group for the given testbox.
1296	"""
1297	if len(oTestBoxDataEx.aoInSchedGroups) == 1:
1298	oSchedGroup = oTestBoxDataEx.aoInSchedGroups[0].oSchedGroup;
1299	if oSchedGroup.fEnabled \
1300	and oSchedGroup.idBuildSrc is not None \
1301	and oSchedGroup.idSchedGroup not in dIgnoreSchedGroupIds:
1302	return (oSchedGroup, 0);
1303	iWorkItem = 0;
1304
1305	elif len(oTestBoxDataEx.aoInSchedGroups) > 0:
1306	# Construct priority table of currently enabled scheduling groups.
1307	aaoList1 = [];
1308	for oInGroup in oTestBoxDataEx.aoInSchedGroups:
1309	oSchedGroup = oInGroup.oSchedGroup;
1310	if oSchedGroup.fEnabled and oSchedGroup.idBuildSrc is not None:
1311	iSchedPriority = oInGroup.iSchedPriority;
1312	if iSchedPriority > 31: # paranoia
1313	iSchedPriority = 31;
1314	elif iSchedPriority < 0: # paranoia
1315	iSchedPriority = 0;
1316
1317	for iSchedPriority in xrange(min(iSchedPriority, len(aaoList1))):
1318	aaoList1[iSchedPriority].append(oSchedGroup);
1319	while len(aaoList1) <= iSchedPriority:
1320	aaoList1.append([oSchedGroup,]);
1321
1322	# Flatten it into a single list, mixing the priorities a little so it doesn't
1323	# take forever before low priority stuff is executed.
1324	aoFlat = [];
1325	iLo = 0;
1326	iHi = len(aaoList1) - 1;
1327	while iHi >= iLo:
1328	aoFlat += aaoList1[iHi];
1329	if iLo < iHi:
1330	aoFlat += aaoList1[iLo];
1331	iLo += 1;
1332	iHi -= 1;
1333
1334	# Pick the next one.
1335	cLeft = len(aoFlat);
1336	while cLeft > 0:
1337	cLeft -= 1;
1338	iWorkItem += 1;
1339	if iWorkItem >= len(aoFlat) or iWorkItem < 0:
1340	iWorkItem = 0;
1341	if aoFlat[iWorkItem].idSchedGroup not in dIgnoreSchedGroupIds:
1342	return (aoFlat[iWorkItem], iWorkItem);
1343	else:
1344	iWorkItem = 0;
1345
1346	# No active group.
1347	return (None, iWorkItem);
1348
1349	@staticmethod
1350	def scheduleNewTask(oDb, oTestBoxData, iWorkItem, sBaseUrl, iVerbosity = 0):
1351	# type: (TMDatabaseConnection, TestBoxData, int, str, int) -> None
1352	"""
1353	Schedules a new task for a testbox.
1354	"""
1355	oTBStatusLogic = TestBoxStatusLogic(oDb);
1356
1357	try:
1358	#
1359	# To avoid concurrency issues in SchedQueues we lock all the rows
1360	# related to our scheduling queue. Also, since this is a very
1361	# expensive operation we lock the testbox status row to fend of
1362	# repeated retires by faulty testbox scripts.
1363	#
1364	tsSecStart = utils.timestampSecond();
1365	oDb.rollback();
1366	oDb.begin();
1367	oDb.execute('SELECT idTestBox FROM TestBoxStatuses WHERE idTestBox = %s FOR UPDATE NOWAIT'
1368	% (oTestBoxData.idTestBox,));
1369	oDb.execute('SELECT SchedQueues.idSchedGroup\n'
1370	' FROM SchedQueues, TestBoxesInSchedGroups\n'
1371	'WHERE TestBoxesInSchedGroups.idTestBox = %s\n'
1372	' AND TestBoxesInSchedGroups.tsExpire = \'infinity\'::TIMESTAMP\n'
1373	' AND TestBoxesInSchedGroups.idSchedGroup = SchedQueues.idSchedGroup\n'
1374	' FOR UPDATE'
1375	% (oTestBoxData.idTestBox,));
1376
1377	# We need the current timestamp.
1378	tsNow = oDb.getCurrentTimestamp();
1379
1380	# Re-read the testbox data with scheduling group relations.
1381	oTestBoxDataEx = TestBoxDataEx().initFromDbWithId(oDb, oTestBoxData.idTestBox, tsNow);
1382	if oTestBoxDataEx.fEnabled \
1383	and oTestBoxDataEx.idGenTestBox == oTestBoxData.idGenTestBox:
1384
1385	# We may have to skip scheduling groups that are out of work (e.g. 'No build').
1386	iInitialWorkItem = iWorkItem;
1387	dIgnoreSchedGroupIds = {};
1388	while True:
1389	# Now, pick the scheduling group.
1390	(oSchedGroup, iWorkItem) = SchedulerBase._pickSchedGroup(oTestBoxDataEx, iWorkItem, dIgnoreSchedGroupIds);
1391	if oSchedGroup is None:
1392	break;
1393	assert oSchedGroup.fEnabled and oSchedGroup.idBuildSrc is not None;
1394
1395	# Instantiate the specified scheduler and let it do the rest.
1396	oScheduler = SchedulerBase._instantiate(oDb, oSchedGroup, iVerbosity, tsSecStart);
1397	dResponse = oScheduler.scheduleNewTaskWorker(oTestBoxDataEx, tsNow, sBaseUrl);
1398	if dResponse is not None:
1399	oTBStatusLogic.updateWorkItem(oTestBoxDataEx.idTestBox, iWorkItem);
1400	oDb.commit();
1401	return dResponse;
1402
1403	# Check out the next work item?
1404	if oScheduler.getElapsedSecs() > config.g_kcSecMaxNewTask:
1405	break;
1406	dIgnoreSchedGroupIds[oSchedGroup.idSchedGroup] = oSchedGroup;
1407
1408	# No luck, but best if we update the work item if we've made progress.
1409	# Note! In case of a config.g_kcSecMaxNewTask timeout, this may accidentally skip
1410	# a work item with actually work to do. But that's a small price to pay.
1411	if iWorkItem != iInitialWorkItem:
1412	oTBStatusLogic.updateWorkItem(oTestBoxDataEx.idTestBox, iWorkItem);
1413	oDb.commit();
1414	return None;
1415	except:
1416	oDb.rollback();
1417	raise;
1418
1419	# Not enabled, rollback and return no task.
1420	oDb.rollback();
1421	return None;
1422
1423	@staticmethod
1424	def tryCancelGangGathering(oDb, oStatusData):
1425	"""
1426	Try canceling a gang gathering.
1427
1428	Returns True if successfully cancelled.
1429	Returns False if not (someone raced us to the SchedQueue table).
1430
1431	Note! oStatusData is re-initialized.
1432	"""
1433	assert oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGathering;
1434	try:
1435	#
1436	# Lock the tables we're updating so we don't run into concurrency
1437	# issues (we're racing both scheduleNewTask and other callers of
1438	# this method).
1439	#
1440	oDb.rollback();
1441	oDb.begin();
1442	oDb.execute('LOCK TABLE TestBoxStatuses, SchedQueues IN EXCLUSIVE MODE');
1443
1444	#
1445	# Re-read the testbox data and check that we're still in the same state.
1446	#
1447	oStatusData.initFromDbWithId(oDb, oStatusData.idTestBox);
1448	if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGathering:
1449	#
1450	# Get the leader thru the test set and change the state of the whole gang.
1451	#
1452	oTestSetData = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet);
1453
1454	oTBStatusLogic = TestBoxStatusLogic(oDb);
1455	oTBStatusLogic.updateGangStatus(oTestSetData.idTestSetGangLeader,
1456	TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut,
1457	fCommit = False);
1458
1459	#
1460	# Move the scheduling queue item to the end.
1461	#
1462	oDb.execute('SELECT *\n'
1463	'FROM SchedQueues\n'
1464	'WHERE idTestSetGangLeader = %s\n'
1465	, (oTestSetData.idTestSetGangLeader,) );
1466	oTask = SchedQueueData().initFromDbRow(oDb.fetchOne());
1467	oTestEx = TestCaseArgsDataEx().initFromDbWithGenId(oDb, oTask.idGenTestCaseArgs);
1468
1469	oDb.execute('UPDATE SchedQueues\n'
1470	' SET idItem = NEXTVAL(\'SchedQueueItemIdSeq\'),\n'
1471	' idTestSetGangLeader = NULL,\n'
1472	' cMissingGangMembers = %s\n'
1473	'WHERE idItem = %s\n'
1474	, (oTestEx.cGangMembers, oTask.idItem,) );
1475
1476	oDb.commit();
1477	return True;
1478
1479	elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut:
1480	oDb.rollback();
1481	return True;
1482	except:
1483	oDb.rollback();
1484	raise;
1485
1486	# Not enabled, rollback and return no task.
1487	oDb.rollback();
1488	return False;
1489
1490
1491	#
1492	# Unit testing.
1493	#
1494
1495	# pylint: disable=C0111
1496	class SchedQueueDataTestCase(ModelDataBaseTestCase):
1497	def setUp(self):
1498	self.aoSamples = [SchedQueueData(),];
1499
1500	if __name__ == '__main__':
1501	unittest.main();
1502	# not reached.
1503

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/core/schedulerbase.py@ 65040

Download in other formats: