1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 import sys
17 import re
18 import exceptions
19 from twisted.python import log
20 from twisted.internet import defer
21 from twisted.enterprise import adbapi
22 from buildbot.process.buildstep import LogLineObserver
23 from buildbot.steps.shell import Test
24
26 """This class works the same way as
27 twisted.enterprise.adbapi.ConnectionPool. But it adds the ability to
28 compare connection pools for equality (by comparing the arguments
29 passed to the constructor).
30
31 This is useful when passing the ConnectionPool to a BuildStep, as
32 otherwise Buildbot will consider the buildstep (and hence the
33 containing buildfactory) to have changed every time the configuration
34 is reloaded.
35
36 It also sets some defaults differently from adbapi.ConnectionPool that
37 are more suitable for use in MTR.
38 """
40 self._eqKey = (args, kwargs)
41 return adbapi.ConnectionPool.__init__(self,
42 cp_reconnect=True, cp_min=1, cp_max=3,
43 *args, **kwargs)
44
46 if isinstance(other, EqConnectionPool):
47 return self._eqKey == other._eqKey
48 else:
49 return False
50
52 return not self.__eq__(other)
53
54
56 - def __init__(self, testname, variant, result, info, text, callback):
57 self.testname = testname
58 self.variant = variant
59 self.result = result
60 self.info = info
61 self.text = text
62 self.callback = callback
63
64 - def add(self, line):
66
68 return self.callback(self.testname, self.variant, self.result, self.info, self.text)
72 """
73 Class implementing a log observer (can be passed to
74 BuildStep.addLogObserver().
75
76 It parses the output of mysql-test-run.pl as used in MySQL,
77 MariaDB, Drizzle, etc.
78
79 It counts number of tests run and uses it to provide more accurate
80 completion estimates.
81
82 It parses out test failures from the output and summarises the results on
83 the Waterfall page. It also passes the information to methods that can be
84 overridden in a subclass to do further processing on the information."""
85
86 _line_re = re.compile(r"^([-._0-9a-zA-z]+)( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ (fail|pass) \]\s*(.*)$")
87 _line_re2 = re.compile(r"^[-._0-9a-zA-z]+( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ [-a-z]+ \]")
88 _line_re3 = re.compile(r"^\*\*\*Warnings generated in error logs during shutdown after running tests: (.*)")
89 _line_re4 = re.compile(r"^The servers were restarted [0-9]+ times$")
90 _line_re5 = re.compile(r"^Only\s+[0-9]+\s+of\s+[0-9]+\s+completed.$")
91
92 - def __init__(self, textLimit=5, testNameLimit=16, testType=None):
93 self.textLimit = textLimit
94 self.testNameLimit = testNameLimit
95 self.testType = testType
96 self.numTests = 0
97 self.testFail = None
98 self.failList = []
99 self.warnList = []
100 LogLineObserver.__init__(self)
101
106
108 stripLine = line.strip("\r\n")
109 m = self._line_re.search(stripLine)
110 if m:
111 testname, variant, worker, result, info = m.groups()
112 self.closeTestFail()
113 self.numTests += 1
114 self.step.setProgress('tests', self.numTests)
115
116 if result == "fail":
117 if variant == None:
118 variant = ""
119 else:
120 variant = variant[2:-1]
121 self.openTestFail(testname, variant, result, info, stripLine + "\n")
122
123 else:
124 m = self._line_re3.search(stripLine)
125 if m:
126 stuff = m.group(1)
127 self.closeTestFail()
128 testList = stuff.split(" ")
129 self.doCollectWarningTests(testList)
130
131 elif (self._line_re2.search(stripLine) or
132 self._line_re4.search(stripLine) or
133 self._line_re5.search(stripLine) or
134 stripLine == "Test suite timeout! Terminating..." or
135 stripLine.startswith("mysql-test-run: *** ERROR: Not all tests completed") or
136 (stripLine.startswith("------------------------------------------------------------")
137 and self.testFail != None)):
138 self.closeTestFail()
139
140 else:
141 self.addTestFailOutput(stripLine + "\n")
142
143 - def openTestFail(self, testname, variant, result, info, line):
145
147 if self.testFail != None:
148 self.testFail.add(line)
149
151 if self.testFail != None:
152 self.testFail.fireCallback()
153 self.testFail = None
154
155 - def addToText(self, src, dst):
156 lastOne = None
157 count = 0
158 for t in src:
159 if t != lastOne:
160 dst.append(t)
161 count += 1
162 if count >= self.textLimit:
163 break
164
165 - def makeText(self, done):
166 if done:
167 text = ["test"]
168 else:
169 text = ["testing"]
170 if self.testType:
171 text.append(self.testType)
172 fails = self.failList[:]
173 fails.sort()
174 self.addToText(fails, text)
175 warns = self.warnList[:]
176 warns.sort()
177 self.addToText(warns, text)
178 return text
179
180
181 - def updateText(self):
183
184 strip_re = re.compile(r"^[a-z]+\.")
185
193
198
204
205
210
212 """
213 Build step that runs mysql-test-run.pl, as used in MySQL, Drizzle,
214 MariaDB, etc.
215
216 It uses class MtrLogObserver to parse test results out from the
217 output of mysql-test-run.pl, providing better completion time
218 estimates and summarising test failures on the waterfall page.
219
220 It also provides access to mysqld server error logs from the test
221 run to help debugging any problems.
222
223 Optionally, it can insert into a database data about the test run,
224 including details of any test failures.
225
226 Parameters:
227
228 textLimit
229 Maximum number of test failures to show on the waterfall page
230 (to not flood the page in case of a large number of test
231 failures. Defaults to 5.
232
233 testNameLimit
234 Maximum length of test names to show unabbreviated in the
235 waterfall page, to avoid excessive column width. Defaults to 16.
236
237 parallel
238 Value of --parallel option used for mysql-test-run.pl (number
239 of processes used to run the test suite in parallel). Defaults
240 to 4. This is used to determine the number of server error log
241 files to download from the slave. Specifying a too high value
242 does not hurt (as nonexisting error logs will be ignored),
243 however if using --parallel value greater than the default it
244 needs to be specified, or some server error logs will be
245 missing.
246
247 dbpool
248 An instance of twisted.enterprise.adbapi.ConnectionPool, or None.
249 Defaults to None. If specified, results are inserted into the database
250 using the ConnectionPool.
251
252 The class process.mtrlogobserver.EqConnectionPool subclass of
253 ConnectionPool can be useful to pass as value for dbpool, to
254 avoid having config reloads think the Buildstep is changed
255 just because it gets a new ConnectionPool instance (even
256 though connection parameters are unchanged).
257
258 autoCreateTables
259 Boolean, defaults to False. If True (and dbpool is specified), the
260 necessary database tables will be created automatically if they do
261 not exist already. Alternatively, the tables can be created manually
262 from the SQL statements found in the mtrlogobserver.py source file.
263
264 test_type
265 test_info
266 Two descriptive strings that will be inserted in the database tables if
267 dbpool is specified. The test_type string, if specified, will also
268 appear on the waterfall page."""
269
270 - def __init__(self, dbpool=None, test_type=None, test_info="",
271 description=None, descriptionDone=None,
272 autoCreateTables=False, textLimit=5, testNameLimit=16,
273 parallel=4, logfiles = {}, lazylogfiles = True,
274 warningPattern="MTR's internal check of the test case '.*' failed",
275 mtr_subdir="mysql-test", **kwargs):
276
277 if description is None:
278 description = ["testing"]
279 if test_type:
280 description.append(test_type)
281 if descriptionDone is None:
282 descriptionDone = ["test"]
283 if test_type:
284 descriptionDone.append(test_type)
285 Test.__init__(self, logfiles=logfiles, lazylogfiles=lazylogfiles,
286 description=description, descriptionDone=descriptionDone,
287 warningPattern=warningPattern, **kwargs)
288 self.dbpool = dbpool
289 self.test_type = test_type
290 self.test_info = test_info
291 self.autoCreateTables = autoCreateTables
292 self.textLimit = textLimit
293 self.testNameLimit = testNameLimit
294 self.parallel = parallel
295 self.mtr_subdir = mtr_subdir
296 self.progressMetrics += ('tests',)
297
298 self.addFactoryArguments(dbpool=self.dbpool,
299 test_type=self.test_type,
300 test_info=self.test_info,
301 autoCreateTables=self.autoCreateTables,
302 textLimit=self.textLimit,
303 testNameLimit=self.testNameLimit,
304 parallel=self.parallel,
305 mtr_subdir=self.mtr_subdir)
306
308 properties = self.build.getProperties()
309 subdir = properties.render(self.mtr_subdir)
310
311
312 for mtr in range(0, self.parallel+1):
313 for mysqld in range(1, 4+1):
314 if mtr == 0:
315 logname = "mysqld.%d.err" % mysqld
316 filename = "var/log/mysqld.%d.err" % mysqld
317 else:
318 logname = "mysqld.%d.err.%d" % (mysqld, mtr)
319 filename = "var/%d/log/mysqld.%d.err" % (mtr, mysqld)
320 self.addLogFile(logname, subdir + "/" + filename)
321
322 self.myMtr = self.MyMtrLogObserver(textLimit=self.textLimit,
323 testNameLimit=self.testNameLimit,
324 testType=self.test_type)
325 self.addLogObserver("stdio", self.myMtr)
326
327
328 d = self.registerInDB()
329 d.addCallback(self.afterRegisterInDB)
330 d.addErrback(self.failed)
331
332 - def getText(self, command, results):
333 return self.myMtr.makeText(True)
334
336 """
337 Run a database transaction with dbpool.runInteraction, but retry the
338 transaction in case of a temporary error (like connection lost).
339
340 This is needed to be robust against things like database connection
341 idle timeouts.
342
343 The passed callable that implements the transaction must be retryable,
344 ie. it must not have any destructive side effects in the case where
345 an exception is thrown and/or rollback occurs that would prevent it
346 from functioning correctly when called again."""
347
348 def runWithRetry(txn, *args, **kw):
349 retryCount = 0
350 while(True):
351 try:
352 return actionFn(txn, *args, **kw)
353 except txn.OperationalError:
354 retryCount += 1
355 if retryCount >= 5:
356 raise
357 excType, excValue, excTraceback = sys.exc_info()
358 log.msg("Database transaction failed (caught exception %s(%s)), retrying ..." % (excType, excValue))
359 txn.close()
360 txn.reconnect()
361 txn.reopen()
362
363 return self.dbpool.runInteraction(runWithRetry, *args, **kw)
364
366 """
367 Run a database query, like with dbpool.runQuery, but retry the query in
368 case of a temporary error (like connection lost).
369
370 This is needed to be robust against things like database connection
371 idle timeouts."""
372
373 def runQuery(txn, *args, **kw):
374 txn.execute(*args, **kw)
375 return txn.fetchall()
376
377 return self.runInteractionWithRetry(runQuery, *args, **kw)
378
384
385
387
388
389
390
391 if self.autoCreateTables:
392 txn.execute("""
393 CREATE TABLE IF NOT EXISTS test_run(
394 id INT PRIMARY KEY AUTO_INCREMENT,
395 branch VARCHAR(100),
396 revision VARCHAR(32) NOT NULL,
397 platform VARCHAR(100) NOT NULL,
398 dt TIMESTAMP NOT NULL,
399 bbnum INT NOT NULL,
400 typ VARCHAR(32) NOT NULL,
401 info VARCHAR(255),
402 KEY (branch, revision),
403 KEY (dt),
404 KEY (platform, bbnum)
405 ) ENGINE=innodb
406 """)
407 txn.execute("""
408 CREATE TABLE IF NOT EXISTS test_failure(
409 test_run_id INT NOT NULL,
410 test_name VARCHAR(100) NOT NULL,
411 test_variant VARCHAR(16) NOT NULL,
412 info_text VARCHAR(255),
413 failure_text TEXT,
414 PRIMARY KEY (test_run_id, test_name, test_variant)
415 ) ENGINE=innodb
416 """)
417 txn.execute("""
418 CREATE TABLE IF NOT EXISTS test_warnings(
419 test_run_id INT NOT NULL,
420 list_id INT NOT NULL,
421 list_idx INT NOT NULL,
422 test_name VARCHAR(100) NOT NULL,
423 PRIMARY KEY (test_run_id, list_id, list_idx)
424 ) ENGINE=innodb
425 """)
426
427 revision = None
428 try:
429 revision = self.getProperty("got_revision")
430 except exceptions.KeyError:
431 revision = self.getProperty("revision")
432 typ = "mtr"
433 if self.test_type:
434 typ = self.test_type
435 txn.execute("""
436 INSERT INTO test_run(branch, revision, platform, dt, bbnum, typ, info)
437 VALUES (%s, %s, %s, CURRENT_TIMESTAMP(), %s, %s, %s)
438 """, (self.getProperty("branch"), revision,
439 self.getProperty("buildername"), self.getProperty("buildnumber"),
440 typ, self.test_info))
441
442 return txn.lastrowid
443
449
451 log.msg("Error in async insert into database: %s" % err)
452
455
456 dbpool = self.step.dbpool
457 run_id = self.step.getProperty("mtr_id")
458 if dbpool == None:
459 return defer.succeed(None)
460 if variant == None:
461 variant = ""
462 d = self.step.runQueryWithRetry("""
463 INSERT INTO test_failure(test_run_id, test_name, test_variant, info_text, failure_text)
464 VALUES (%s, %s, %s, %s, %s)
465 """, (run_id, testname, variant, info, text))
466
467 d.addErrback(self.step.reportError)
468 return d
469
471
472 dbpool = self.step.dbpool
473 if dbpool == None:
474 return defer.succeed(None)
475 run_id = self.step.getProperty("mtr_id")
476 warn_id = self.step.getProperty("mtr_warn_id")
477 self.step.setProperty("mtr_warn_id", warn_id + 1)
478 q = ("INSERT INTO test_warnings(test_run_id, list_id, list_idx, test_name) " +
479 "VALUES " + ", ".join(map(lambda x: "(%s, %s, %s, %s)", testList)))
480 v = []
481 idx = 0
482 for t in testList:
483 v.extend([run_id, warn_id, idx, t])
484 idx = idx + 1
485 d = self.step.runQueryWithRetry(q, tuple(v))
486 d.addErrback(self.step.reportError)
487 return d