1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 import sys
17 import re
18 from twisted.python import log
19 from twisted.internet import defer
20 from twisted.enterprise import adbapi
21 from buildbot.process.buildstep import LogLineObserver
22 from buildbot.steps.shell import Test
23
25 """This class works the same way as
26 twisted.enterprise.adbapi.ConnectionPool. But it adds the ability to
27 compare connection pools for equality (by comparing the arguments
28 passed to the constructor).
29
30 This is useful when passing the ConnectionPool to a BuildStep, as
31 otherwise Buildbot will consider the buildstep (and hence the
32 containing buildfactory) to have changed every time the configuration
33 is reloaded.
34
35 It also sets some defaults differently from adbapi.ConnectionPool that
36 are more suitable for use in MTR.
37 """
39 self._eqKey = (args, kwargs)
40 return adbapi.ConnectionPool.__init__(self,
41 cp_reconnect=True, cp_min=1, cp_max=3,
42 *args, **kwargs)
43
45 if isinstance(other, EqConnectionPool):
46 return self._eqKey == other._eqKey
47 else:
48 return False
49
51 return not self.__eq__(other)
52
53
55 - def __init__(self, testname, variant, result, info, text, callback):
56 self.testname = testname
57 self.variant = variant
58 self.result = result
59 self.info = info
60 self.text = text
61 self.callback = callback
62
63 - def add(self, line):
65
67 return self.callback(self.testname, self.variant, self.result, self.info, self.text)
71 """
72 Class implementing a log observer (can be passed to
73 BuildStep.addLogObserver().
74
75 It parses the output of mysql-test-run.pl as used in MySQL,
76 MariaDB, Drizzle, etc.
77
78 It counts number of tests run and uses it to provide more accurate
79 completion estimates.
80
81 It parses out test failures from the output and summarises the results on
82 the Waterfall page. It also passes the information to methods that can be
83 overridden in a subclass to do further processing on the information."""
84
85 _line_re = re.compile(r"^([-._0-9a-zA-z]+)( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ (fail|pass) \]\s*(.*)$")
86 _line_re2 = re.compile(r"^[-._0-9a-zA-z]+( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ [-a-z]+ \]")
87 _line_re3 = re.compile(r"^\*\*\*Warnings generated in error logs during shutdown after running tests: (.*)")
88 _line_re4 = re.compile(r"^The servers were restarted [0-9]+ times$")
89 _line_re5 = re.compile(r"^Only\s+[0-9]+\s+of\s+[0-9]+\s+completed.$")
90
91 - def __init__(self, textLimit=5, testNameLimit=16, testType=None):
92 self.textLimit = textLimit
93 self.testNameLimit = testNameLimit
94 self.testType = testType
95 self.numTests = 0
96 self.testFail = None
97 self.failList = []
98 self.warnList = []
99 LogLineObserver.__init__(self)
100
105
107 stripLine = line.strip("\r\n")
108 m = self._line_re.search(stripLine)
109 if m:
110 testname, variant, worker, result, info = m.groups()
111 self.closeTestFail()
112 self.numTests += 1
113 self.step.setProgress('tests', self.numTests)
114
115 if result == "fail":
116 if variant == None:
117 variant = ""
118 else:
119 variant = variant[2:-1]
120 self.openTestFail(testname, variant, result, info, stripLine + "\n")
121
122 else:
123 m = self._line_re3.search(stripLine)
124 if m:
125 stuff = m.group(1)
126 self.closeTestFail()
127 testList = stuff.split(" ")
128 self.doCollectWarningTests(testList)
129
130 elif (self._line_re2.search(stripLine) or
131 self._line_re4.search(stripLine) or
132 self._line_re5.search(stripLine) or
133 stripLine == "Test suite timeout! Terminating..." or
134 stripLine.startswith("mysql-test-run: *** ERROR: Not all tests completed") or
135 (stripLine.startswith("------------------------------------------------------------")
136 and self.testFail != None)):
137 self.closeTestFail()
138
139 else:
140 self.addTestFailOutput(stripLine + "\n")
141
142 - def openTestFail(self, testname, variant, result, info, line):
144
146 if self.testFail != None:
147 self.testFail.add(line)
148
150 if self.testFail != None:
151 self.testFail.fireCallback()
152 self.testFail = None
153
154 - def addToText(self, src, dst):
155 lastOne = None
156 count = 0
157 for t in src:
158 if t != lastOne:
159 dst.append(t)
160 count += 1
161 if count >= self.textLimit:
162 break
163
164 - def makeText(self, done):
165 if done:
166 text = ["test"]
167 else:
168 text = ["testing"]
169 if self.testType:
170 text.append(self.testType)
171 fails = self.failList[:]
172 fails.sort()
173 self.addToText(fails, text)
174 warns = self.warnList[:]
175 warns.sort()
176 self.addToText(warns, text)
177 return text
178
179
180 - def updateText(self):
182
183 strip_re = re.compile(r"^[a-z]+\.")
184
192
197
203
204
209
211 """
212 Build step that runs mysql-test-run.pl, as used in MySQL, Drizzle,
213 MariaDB, etc.
214
215 It uses class MtrLogObserver to parse test results out from the
216 output of mysql-test-run.pl, providing better completion time
217 estimates and summarising test failures on the waterfall page.
218
219 It also provides access to mysqld server error logs from the test
220 run to help debugging any problems.
221
222 Optionally, it can insert into a database data about the test run,
223 including details of any test failures.
224
225 Parameters:
226
227 textLimit
228 Maximum number of test failures to show on the waterfall page
229 (to not flood the page in case of a large number of test
230 failures. Defaults to 5.
231
232 testNameLimit
233 Maximum length of test names to show unabbreviated in the
234 waterfall page, to avoid excessive column width. Defaults to 16.
235
236 parallel
237 Value of --parallel option used for mysql-test-run.pl (number
238 of processes used to run the test suite in parallel). Defaults
239 to 4. This is used to determine the number of server error log
240 files to download from the slave. Specifying a too high value
241 does not hurt (as nonexisting error logs will be ignored),
242 however if using --parallel value greater than the default it
243 needs to be specified, or some server error logs will be
244 missing.
245
246 dbpool
247 An instance of twisted.enterprise.adbapi.ConnectionPool, or None.
248 Defaults to None. If specified, results are inserted into the database
249 using the ConnectionPool.
250
251 The class process.mtrlogobserver.EqConnectionPool subclass of
252 ConnectionPool can be useful to pass as value for dbpool, to
253 avoid having config reloads think the Buildstep is changed
254 just because it gets a new ConnectionPool instance (even
255 though connection parameters are unchanged).
256
257 autoCreateTables
258 Boolean, defaults to False. If True (and dbpool is specified), the
259 necessary database tables will be created automatically if they do
260 not exist already. Alternatively, the tables can be created manually
261 from the SQL statements found in the mtrlogobserver.py source file.
262
263 test_type
264 test_info
265 Two descriptive strings that will be inserted in the database tables if
266 dbpool is specified. The test_type string, if specified, will also
267 appear on the waterfall page."""
268
269 renderables = [ 'mtr_subdir' ]
270
271 - def __init__(self, dbpool=None, test_type=None, test_info="",
272 description=None, descriptionDone=None,
273 autoCreateTables=False, textLimit=5, testNameLimit=16,
274 parallel=4, logfiles = {}, lazylogfiles = True,
275 warningPattern="MTR's internal check of the test case '.*' failed",
276 mtr_subdir="mysql-test", **kwargs):
277
278 if description is None:
279 description = ["testing"]
280 if test_type:
281 description.append(test_type)
282 if descriptionDone is None:
283 descriptionDone = ["test"]
284 if test_type:
285 descriptionDone.append(test_type)
286 Test.__init__(self, logfiles=logfiles, lazylogfiles=lazylogfiles,
287 description=description, descriptionDone=descriptionDone,
288 warningPattern=warningPattern, **kwargs)
289 self.dbpool = dbpool
290 self.test_type = test_type
291 self.test_info = test_info
292 self.autoCreateTables = autoCreateTables
293 self.textLimit = textLimit
294 self.testNameLimit = testNameLimit
295 self.parallel = parallel
296 self.mtr_subdir = mtr_subdir
297 self.progressMetrics += ('tests',)
298
299 self.addFactoryArguments(dbpool=self.dbpool,
300 test_type=self.test_type,
301 test_info=self.test_info,
302 autoCreateTables=self.autoCreateTables,
303 textLimit=self.textLimit,
304 testNameLimit=self.testNameLimit,
305 parallel=self.parallel,
306 mtr_subdir=self.mtr_subdir)
307
309
310 for mtr in range(0, self.parallel+1):
311 for mysqld in range(1, 4+1):
312 if mtr == 0:
313 logname = "mysqld.%d.err" % mysqld
314 filename = "var/log/mysqld.%d.err" % mysqld
315 else:
316 logname = "mysqld.%d.err.%d" % (mysqld, mtr)
317 filename = "var/%d/log/mysqld.%d.err" % (mtr, mysqld)
318 self.addLogFile(logname, self.mtr_subdir + "/" + filename)
319
320 self.myMtr = self.MyMtrLogObserver(textLimit=self.textLimit,
321 testNameLimit=self.testNameLimit,
322 testType=self.test_type)
323 self.addLogObserver("stdio", self.myMtr)
324
325
326 d = self.registerInDB()
327 d.addCallback(self.afterRegisterInDB)
328 d.addErrback(self.failed)
329
330 - def getText(self, command, results):
331 return self.myMtr.makeText(True)
332
334 """
335 Run a database transaction with dbpool.runInteraction, but retry the
336 transaction in case of a temporary error (like connection lost).
337
338 This is needed to be robust against things like database connection
339 idle timeouts.
340
341 The passed callable that implements the transaction must be retryable,
342 ie. it must not have any destructive side effects in the case where
343 an exception is thrown and/or rollback occurs that would prevent it
344 from functioning correctly when called again."""
345
346 def runWithRetry(txn, *args, **kw):
347 retryCount = 0
348 while(True):
349 try:
350 return actionFn(txn, *args, **kw)
351 except txn.OperationalError:
352 retryCount += 1
353 if retryCount >= 5:
354 raise
355 excType, excValue, excTraceback = sys.exc_info()
356 log.msg("Database transaction failed (caught exception %s(%s)), retrying ..." % (excType, excValue))
357 txn.close()
358 txn.reconnect()
359 txn.reopen()
360
361 return self.dbpool.runInteraction(runWithRetry, *args, **kw)
362
364 """
365 Run a database query, like with dbpool.runQuery, but retry the query in
366 case of a temporary error (like connection lost).
367
368 This is needed to be robust against things like database connection
369 idle timeouts."""
370
371 def runQuery(txn, *args, **kw):
372 txn.execute(*args, **kw)
373 return txn.fetchall()
374
375 return self.runInteractionWithRetry(runQuery, *args, **kw)
376
382
383
385
386
387
388
389 if self.autoCreateTables:
390 txn.execute("""
391 CREATE TABLE IF NOT EXISTS test_run(
392 id INT PRIMARY KEY AUTO_INCREMENT,
393 branch VARCHAR(100),
394 revision VARCHAR(32) NOT NULL,
395 platform VARCHAR(100) NOT NULL,
396 dt TIMESTAMP NOT NULL,
397 bbnum INT NOT NULL,
398 typ VARCHAR(32) NOT NULL,
399 info VARCHAR(255),
400 KEY (branch, revision),
401 KEY (dt),
402 KEY (platform, bbnum)
403 ) ENGINE=innodb
404 """)
405 txn.execute("""
406 CREATE TABLE IF NOT EXISTS test_failure(
407 test_run_id INT NOT NULL,
408 test_name VARCHAR(100) NOT NULL,
409 test_variant VARCHAR(16) NOT NULL,
410 info_text VARCHAR(255),
411 failure_text TEXT,
412 PRIMARY KEY (test_run_id, test_name, test_variant)
413 ) ENGINE=innodb
414 """)
415 txn.execute("""
416 CREATE TABLE IF NOT EXISTS test_warnings(
417 test_run_id INT NOT NULL,
418 list_id INT NOT NULL,
419 list_idx INT NOT NULL,
420 test_name VARCHAR(100) NOT NULL,
421 PRIMARY KEY (test_run_id, list_id, list_idx)
422 ) ENGINE=innodb
423 """)
424
425 revision = self.getProperty("got_revision")
426 if revision is None:
427 revision = self.getProperty("revision")
428 typ = "mtr"
429 if self.test_type:
430 typ = self.test_type
431 txn.execute("""
432 INSERT INTO test_run(branch, revision, platform, dt, bbnum, typ, info)
433 VALUES (%s, %s, %s, CURRENT_TIMESTAMP(), %s, %s, %s)
434 """, (self.getProperty("branch"), revision,
435 self.getProperty("buildername"), self.getProperty("buildnumber"),
436 typ, self.test_info))
437
438 return txn.lastrowid
439
445
447 log.msg("Error in async insert into database: %s" % err)
448
451
452 dbpool = self.step.dbpool
453 run_id = self.step.getProperty("mtr_id")
454 if dbpool == None:
455 return defer.succeed(None)
456 if variant == None:
457 variant = ""
458 d = self.step.runQueryWithRetry("""
459 INSERT INTO test_failure(test_run_id, test_name, test_variant, info_text, failure_text)
460 VALUES (%s, %s, %s, %s, %s)
461 """, (run_id, testname, variant, info, text))
462
463 d.addErrback(self.step.reportError)
464 return d
465
467
468 dbpool = self.step.dbpool
469 if dbpool == None:
470 return defer.succeed(None)
471 run_id = self.step.getProperty("mtr_id")
472 warn_id = self.step.getProperty("mtr_warn_id")
473 self.step.setProperty("mtr_warn_id", warn_id + 1)
474 q = ("INSERT INTO test_warnings(test_run_id, list_id, list_idx, test_name) " +
475 "VALUES " + ", ".join(map(lambda x: "(%s, %s, %s, %s)", testList)))
476 v = []
477 idx = 0
478 for t in testList:
479 v.extend([run_id, warn_id, idx, t])
480 idx = idx + 1
481 d = self.step.runQueryWithRetry(q, tuple(v))
482 d.addErrback(self.step.reportError)
483 return d