1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 import sys
17 import re
18 from twisted.python import log
19 from twisted.internet import defer
20 from twisted.enterprise import adbapi
21 from buildbot.process.buildstep import LogLineObserver
22 from buildbot.steps.shell import Test
23
25 """This class works the same way as
26 twisted.enterprise.adbapi.ConnectionPool. But it adds the ability to
27 compare connection pools for equality (by comparing the arguments
28 passed to the constructor).
29
30 This is useful when passing the ConnectionPool to a BuildStep, as
31 otherwise Buildbot will consider the buildstep (and hence the
32 containing buildfactory) to have changed every time the configuration
33 is reloaded.
34
35 It also sets some defaults differently from adbapi.ConnectionPool that
36 are more suitable for use in MTR.
37 """
39 self._eqKey = (args, kwargs)
40 return adbapi.ConnectionPool.__init__(self,
41 cp_reconnect=True, cp_min=1, cp_max=3,
42 *args, **kwargs)
43
45 if isinstance(other, EqConnectionPool):
46 return self._eqKey == other._eqKey
47 else:
48 return False
49
51 return not self.__eq__(other)
52
53
55 - def __init__(self, testname, variant, result, info, text, callback):
56 self.testname = testname
57 self.variant = variant
58 self.result = result
59 self.info = info
60 self.text = text
61 self.callback = callback
62
63 - def add(self, line):
65
67 return self.callback(self.testname, self.variant, self.result, self.info, self.text)
71 """
72 Class implementing a log observer (can be passed to
73 BuildStep.addLogObserver().
74
75 It parses the output of mysql-test-run.pl as used in MySQL,
76 MariaDB, Drizzle, etc.
77
78 It counts number of tests run and uses it to provide more accurate
79 completion estimates.
80
81 It parses out test failures from the output and summarises the results on
82 the Waterfall page. It also passes the information to methods that can be
83 overridden in a subclass to do further processing on the information."""
84
85 _line_re = re.compile(r"^([-._0-9a-zA-z]+)( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ (fail|pass) \]\s*(.*)$")
86 _line_re2 = re.compile(r"^[-._0-9a-zA-z]+( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ [-a-z]+ \]")
87 _line_re3 = re.compile(r"^\*\*\*Warnings generated in error logs during shutdown after running tests: (.*)")
88 _line_re4 = re.compile(r"^The servers were restarted [0-9]+ times$")
89 _line_re5 = re.compile(r"^Only\s+[0-9]+\s+of\s+[0-9]+\s+completed.$")
90
91 - def __init__(self, textLimit=5, testNameLimit=16, testType=None):
92 self.textLimit = textLimit
93 self.testNameLimit = testNameLimit
94 self.testType = testType
95 self.numTests = 0
96 self.testFail = None
97 self.failList = []
98 self.warnList = []
99 LogLineObserver.__init__(self)
100
105
107 stripLine = line.strip("\r\n")
108 m = self._line_re.search(stripLine)
109 if m:
110 testname, variant, worker, result, info = m.groups()
111 self.closeTestFail()
112 self.numTests += 1
113 self.step.setProgress('tests', self.numTests)
114
115 if result == "fail":
116 if variant == None:
117 variant = ""
118 else:
119 variant = variant[2:-1]
120 self.openTestFail(testname, variant, result, info, stripLine + "\n")
121
122 else:
123 m = self._line_re3.search(stripLine)
124 if m:
125 stuff = m.group(1)
126 self.closeTestFail()
127 testList = stuff.split(" ")
128 self.doCollectWarningTests(testList)
129
130 elif (self._line_re2.search(stripLine) or
131 self._line_re4.search(stripLine) or
132 self._line_re5.search(stripLine) or
133 stripLine == "Test suite timeout! Terminating..." or
134 stripLine.startswith("mysql-test-run: *** ERROR: Not all tests completed") or
135 (stripLine.startswith("------------------------------------------------------------")
136 and self.testFail != None)):
137 self.closeTestFail()
138
139 else:
140 self.addTestFailOutput(stripLine + "\n")
141
142 - def openTestFail(self, testname, variant, result, info, line):
144
146 if self.testFail != None:
147 self.testFail.add(line)
148
150 if self.testFail != None:
151 self.testFail.fireCallback()
152 self.testFail = None
153
154 - def addToText(self, src, dst):
155 lastOne = None
156 count = 0
157 for t in src:
158 if t != lastOne:
159 dst.append(t)
160 count += 1
161 if count >= self.textLimit:
162 break
163
164 - def makeText(self, done):
165 if done:
166 text = ["test"]
167 else:
168 text = ["testing"]
169 if self.testType:
170 text.append(self.testType)
171 fails = self.failList[:]
172 fails.sort()
173 self.addToText(fails, text)
174 warns = self.warnList[:]
175 warns.sort()
176 self.addToText(warns, text)
177 return text
178
179
180 - def updateText(self):
182
183 strip_re = re.compile(r"^[a-z]+\.")
184
192
197
203
204
209
211 """
212 Build step that runs mysql-test-run.pl, as used in MySQL, Drizzle,
213 MariaDB, etc.
214
215 It uses class MtrLogObserver to parse test results out from the
216 output of mysql-test-run.pl, providing better completion time
217 estimates and summarising test failures on the waterfall page.
218
219 It also provides access to mysqld server error logs from the test
220 run to help debugging any problems.
221
222 Optionally, it can insert into a database data about the test run,
223 including details of any test failures.
224
225 Parameters:
226
227 textLimit
228 Maximum number of test failures to show on the waterfall page
229 (to not flood the page in case of a large number of test
230 failures. Defaults to 5.
231
232 testNameLimit
233 Maximum length of test names to show unabbreviated in the
234 waterfall page, to avoid excessive column width. Defaults to 16.
235
236 parallel
237 Value of --parallel option used for mysql-test-run.pl (number
238 of processes used to run the test suite in parallel). Defaults
239 to 4. This is used to determine the number of server error log
240 files to download from the slave. Specifying a too high value
241 does not hurt (as nonexisting error logs will be ignored),
242 however if using --parallel value greater than the default it
243 needs to be specified, or some server error logs will be
244 missing.
245
246 dbpool
247 An instance of twisted.enterprise.adbapi.ConnectionPool, or None.
248 Defaults to None. If specified, results are inserted into the database
249 using the ConnectionPool.
250
251 The class process.mtrlogobserver.EqConnectionPool subclass of
252 ConnectionPool can be useful to pass as value for dbpool, to
253 avoid having config reloads think the Buildstep is changed
254 just because it gets a new ConnectionPool instance (even
255 though connection parameters are unchanged).
256
257 autoCreateTables
258 Boolean, defaults to False. If True (and dbpool is specified), the
259 necessary database tables will be created automatically if they do
260 not exist already. Alternatively, the tables can be created manually
261 from the SQL statements found in the mtrlogobserver.py source file.
262
263 test_type
264 test_info
265 Two descriptive strings that will be inserted in the database tables if
266 dbpool is specified. The test_type string, if specified, will also
267 appear on the waterfall page."""
268
269 renderables = [ 'mtr_subdir' ]
270
271 - def __init__(self, dbpool=None, test_type=None, test_info="",
272 description=None, descriptionDone=None,
273 autoCreateTables=False, textLimit=5, testNameLimit=16,
274 parallel=4, logfiles = {}, lazylogfiles = True,
275 warningPattern="MTR's internal check of the test case '.*' failed",
276 mtr_subdir="mysql-test", **kwargs):
298
300
301 for mtr in range(0, self.parallel+1):
302 for mysqld in range(1, 4+1):
303 if mtr == 0:
304 logname = "mysqld.%d.err" % mysqld
305 filename = "var/log/mysqld.%d.err" % mysqld
306 else:
307 logname = "mysqld.%d.err.%d" % (mysqld, mtr)
308 filename = "var/%d/log/mysqld.%d.err" % (mtr, mysqld)
309 self.addLogFile(logname, self.mtr_subdir + "/" + filename)
310
311 self.myMtr = self.MyMtrLogObserver(textLimit=self.textLimit,
312 testNameLimit=self.testNameLimit,
313 testType=self.test_type)
314 self.addLogObserver("stdio", self.myMtr)
315
316
317 d = self.registerInDB()
318 d.addCallback(self.afterRegisterInDB)
319 d.addErrback(self.failed)
320
321 - def getText(self, command, results):
322 return self.myMtr.makeText(True)
323
325 """
326 Run a database transaction with dbpool.runInteraction, but retry the
327 transaction in case of a temporary error (like connection lost).
328
329 This is needed to be robust against things like database connection
330 idle timeouts.
331
332 The passed callable that implements the transaction must be retryable,
333 ie. it must not have any destructive side effects in the case where
334 an exception is thrown and/or rollback occurs that would prevent it
335 from functioning correctly when called again."""
336
337 def runWithRetry(txn, *args, **kw):
338 retryCount = 0
339 while(True):
340 try:
341 return actionFn(txn, *args, **kw)
342 except txn.OperationalError:
343 retryCount += 1
344 if retryCount >= 5:
345 raise
346 excType, excValue, excTraceback = sys.exc_info()
347 log.msg("Database transaction failed (caught exception %s(%s)), retrying ..." % (excType, excValue))
348 txn.close()
349 txn.reconnect()
350 txn.reopen()
351
352 return self.dbpool.runInteraction(runWithRetry, *args, **kw)
353
355 """
356 Run a database query, like with dbpool.runQuery, but retry the query in
357 case of a temporary error (like connection lost).
358
359 This is needed to be robust against things like database connection
360 idle timeouts."""
361
362 def runQuery(txn, *args, **kw):
363 txn.execute(*args, **kw)
364 return txn.fetchall()
365
366 return self.runInteractionWithRetry(runQuery, *args, **kw)
367
373
374
376
377
378
379
380 if self.autoCreateTables:
381 txn.execute("""
382 CREATE TABLE IF NOT EXISTS test_run(
383 id INT PRIMARY KEY AUTO_INCREMENT,
384 branch VARCHAR(100),
385 revision VARCHAR(32) NOT NULL,
386 platform VARCHAR(100) NOT NULL,
387 dt TIMESTAMP NOT NULL,
388 bbnum INT NOT NULL,
389 typ VARCHAR(32) NOT NULL,
390 info VARCHAR(255),
391 KEY (branch, revision),
392 KEY (dt),
393 KEY (platform, bbnum)
394 ) ENGINE=innodb
395 """)
396 txn.execute("""
397 CREATE TABLE IF NOT EXISTS test_failure(
398 test_run_id INT NOT NULL,
399 test_name VARCHAR(100) NOT NULL,
400 test_variant VARCHAR(16) NOT NULL,
401 info_text VARCHAR(255),
402 failure_text TEXT,
403 PRIMARY KEY (test_run_id, test_name, test_variant)
404 ) ENGINE=innodb
405 """)
406 txn.execute("""
407 CREATE TABLE IF NOT EXISTS test_warnings(
408 test_run_id INT NOT NULL,
409 list_id INT NOT NULL,
410 list_idx INT NOT NULL,
411 test_name VARCHAR(100) NOT NULL,
412 PRIMARY KEY (test_run_id, list_id, list_idx)
413 ) ENGINE=innodb
414 """)
415
416 revision = self.getProperty("got_revision")
417 if revision is None:
418 revision = self.getProperty("revision")
419 typ = "mtr"
420 if self.test_type:
421 typ = self.test_type
422 txn.execute("""
423 INSERT INTO test_run(branch, revision, platform, dt, bbnum, typ, info)
424 VALUES (%s, %s, %s, CURRENT_TIMESTAMP(), %s, %s, %s)
425 """, (self.getProperty("branch"), revision,
426 self.getProperty("buildername"), self.getProperty("buildnumber"),
427 typ, self.test_info))
428
429 return txn.lastrowid
430
436
438 log.msg("Error in async insert into database: %s" % err)
439
442
443 dbpool = self.step.dbpool
444 run_id = self.step.getProperty("mtr_id")
445 if dbpool == None:
446 return defer.succeed(None)
447 if variant == None:
448 variant = ""
449 d = self.step.runQueryWithRetry("""
450 INSERT INTO test_failure(test_run_id, test_name, test_variant, info_text, failure_text)
451 VALUES (%s, %s, %s, %s, %s)
452 """, (run_id, testname, variant, info, text))
453
454 d.addErrback(self.step.reportError)
455 return d
456
458
459 dbpool = self.step.dbpool
460 if dbpool == None:
461 return defer.succeed(None)
462 run_id = self.step.getProperty("mtr_id")
463 warn_id = self.step.getProperty("mtr_warn_id")
464 self.step.setProperty("mtr_warn_id", warn_id + 1)
465 q = ("INSERT INTO test_warnings(test_run_id, list_id, list_idx, test_name) " +
466 "VALUES " + ", ".join(map(lambda x: "(%s, %s, %s, %s)", testList)))
467 v = []
468 idx = 0
469 for t in testList:
470 v.extend([run_id, warn_id, idx, t])
471 idx = idx + 1
472 d = self.step.runQueryWithRetry(q, tuple(v))
473 d.addErrback(self.step.reportError)
474 return d