1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 import os
18 from base64 import b64encode
19 import sys
20 import shutil
21
22 from zope.interface import implements
23 from twisted.internet import reactor, defer
24 from twisted.python import log, failure, runtime
25
26 from buildslave.interfaces import ISlaveCommand
27 from buildslave import runprocess
28 from buildslave.exceptions import AbandonChain
29 from buildslave.commands import utils
30 from buildslave import util
31
32
33
34
35 command_version = "2.14"
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
68 implements(ISlaveCommand)
69
70 """This class defines one command that can be invoked by the build master.
71 The command is executed on the slave side, and always sends back a
72 completion message when it finishes. It may also send intermediate status
73 as it runs (by calling builder.sendStatus). Some commands can be
74 interrupted (either by the build master or a local timeout), in which
75 case the step is expected to complete normally with a status message that
76 indicates an error occurred.
77
78 These commands are used by BuildSteps on the master side. Each kind of
79 BuildStep uses a single Command. The slave must implement all the
80 Commands required by the set of BuildSteps used for any given build:
81 this is checked at startup time.
82
83 All Commands are constructed with the same signature:
84 c = CommandClass(builder, stepid, args)
85 where 'builder' is the parent SlaveBuilder object, and 'args' is a
86 dict that is interpreted per-command.
87
88 The setup(args) method is available for setup, and is run from __init__.
89
90 The Command is started with start(). This method must be implemented in a
91 subclass, and it should return a Deferred. When your step is done, you
92 should fire the Deferred (the results are not used). If the command is
93 interrupted, it should fire the Deferred anyway.
94
95 While the command runs. it may send status messages back to the
96 buildmaster by calling self.sendStatus(statusdict). The statusdict is
97 interpreted by the master-side BuildStep however it likes.
98
99 A separate completion message is sent when the deferred fires, which
100 indicates that the Command has finished, but does not carry any status
101 data. If the Command needs to return an exit code of some sort, that
102 should be sent as a regular status message before the deferred is fired .
103 Once builder.commandComplete has been run, no more status messages may be
104 sent.
105
106 If interrupt() is called, the Command should attempt to shut down as
107 quickly as possible. Child processes should be killed, new ones should
108 not be started. The Command should send some kind of error status update,
109 then complete as usual by firing the Deferred.
110
111 .interrupted should be set by interrupt(), and can be tested to avoid
112 sending multiple error status messages.
113
114 If .running is False, the bot is shutting down (or has otherwise lost the
115 connection to the master), and should not send any status messages. This
116 is checked in Command.sendStatus .
117
118 """
119
120
121
122
123
124 debug = False
125 interrupted = False
126 running = False
127
128
129 _reactor = reactor
130
131 - def __init__(self, builder, stepId, args):
137
139 """Override this in a subclass to extract items from the args dict."""
140 pass
141
150 d.addBoth(commandComplete)
151 return d
152
154 """Start the command. This method should return a Deferred that will
155 fire when the command has completed. The Deferred's argument will be
156 ignored.
157
158 This method should be overridden by subclasses."""
159 raise NotImplementedError, "You must implement this in a subclass"
160
169
173
175 """Override this in a subclass to allow commands to be interrupted.
176 May be called multiple times, test and set self.interrupted=True if
177 this matters."""
178 pass
179
180
181
183 if type(rc) is not int:
184 log.msg("weird, _abandonOnFailure was given rc=%s (%s)" % \
185 (rc, type(rc)))
186 assert isinstance(rc, int)
187 if rc != 0:
188 raise AbandonChain(rc)
189 return rc
190
193
195 log.msg("_checkAbandoned", why)
196 why.trap(AbandonChain)
197 log.msg(" abandoning chain", why.value)
198 self.sendStatus({'rc': why.value.args[0]})
199 return None
200
202 """Abstract base class for Version Control System operations (checkout
203 and update). This class extracts the following arguments from the
204 dictionary received from the master:
205
206 - ['workdir']: (required) the subdirectory where the buildable sources
207 should be placed
208
209 - ['mode']: one of update/copy/clobber/export, defaults to 'update'
210
211 - ['revision']: (required) If not None, this is an int or string which indicates
212 which sources (along a time-like axis) should be used.
213 It is the thing you provide as the CVS -r or -D
214 argument.
215
216 - ['patch']: If not None, this is a tuple of (striplevel, patch)
217 which contains a patch that should be applied after the
218 checkout has occurred. Once applied, the tree is no
219 longer eligible for use with mode='update', and it only
220 makes sense to use this in conjunction with a
221 ['revision'] argument. striplevel is an int, and patch
222 is a string in standard unified diff format. The patch
223 will be applied with 'patch -p%d <PATCH', with
224 STRIPLEVEL substituted as %d. The command will fail if
225 the patch process fails (rejected hunks).
226
227 - ['timeout']: seconds of silence tolerated before we kill off the
228 command
229
230 - ['maxTime']: seconds before we kill off the command
231
232 - ['retry']: If not None, this is a tuple of (delay, repeats)
233 which means that any failed VC updates should be
234 reattempted, up to REPEATS times, after a delay of
235 DELAY seconds. This is intended to deal with slaves
236 that experience transient network failures.
237 """
238
239 sourcedata = ""
240
242
243
244
245 self.env = os.environ.copy()
246 self.env['LC_MESSAGES'] = "C"
247
248 self.workdir = args['workdir']
249 self.mode = args.get('mode', "update")
250 self.revision = args.get('revision')
251 self.patch = args.get('patch')
252 self.timeout = args.get('timeout', 120)
253 self.maxTime = args.get('maxTime', None)
254 self.retry = args.get('retry')
255 self.logEnviron = args.get('logEnviron',True)
256 self._commandPaths = {}
257
258
259
261 """Wrapper around utils.getCommand that will output a resonable
262 error message and raise AbandonChain if the command cannot be
263 found"""
264 if name not in self._commandPaths:
265 try:
266 self._commandPaths[name] = utils.getCommand(name)
267 except RuntimeError:
268 self.sendStatus({'stderr' : "could not find '%s'\n" % name})
269 self.sendStatus({'stderr' : "PATH is '%s'\n" % os.environ.get('PATH', '')})
270 raise AbandonChain(-1)
271 return self._commandPaths[name]
272
274 self.sendStatus({'header': "starting " + self.header + "\n"})
275 self.command = None
276
277
278 if self.mode == "copy":
279 self.srcdir = "source"
280 else:
281 self.srcdir = self.workdir
282
283 self.sourcedatafile = os.path.join(self.builder.basedir,
284 ".buildbot-sourcedata-" + b64encode(self.srcdir))
285
286
287 old_sd_path = os.path.join(self.builder.basedir, self.srcdir, ".buildbot-sourcedata")
288 if os.path.exists(old_sd_path) and not os.path.exists(self.sourcedatafile):
289 os.rename(old_sd_path, self.sourcedatafile)
290
291
292
293 old_sd_path = os.path.join(self.builder.basedir, ".buildbot-sourcedata")
294 if os.path.exists(old_sd_path) and not os.path.exists(self.sourcedatafile):
295 os.rename(old_sd_path, self.sourcedatafile)
296
297 d = defer.succeed(None)
298 self.maybeClobber(d)
299 if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()):
300
301
302
303 d.addCallback(self.doClobber, self.srcdir)
304
305 d.addCallback(self.doVC)
306
307 if self.mode == "copy":
308 d.addCallback(self.doCopy)
309 if self.patch:
310 d.addCallback(self.doPatch)
311 d.addCallbacks(self._sendRC, self._checkAbandoned)
312 return d
313
315
316 if self.mode in ("copy", "clobber", "export"):
317 d.addCallback(self.doClobber, self.workdir)
318
323
324 - def doVC(self, res):
337
339 try:
340 olddata = self.readSourcedata()
341 if olddata != self.sourcedata:
342 return False
343 except IOError:
344 return False
345 return True
346
351
353 d = defer.maybeDeferred(self.parseGotRevision)
354 d.addCallback(lambda got_revision:
355 self.sendStatus({'got_revision': got_revision}))
356 return d
357
359 """Override this in a subclass. It should return a string that
360 represents which revision was actually checked out, or a Deferred
361 that will fire with such a string. If, in a future build, you were to
362 pass this 'got_revision' string in as the 'revision' component of a
363 SourceStamp, you should wind up with the same source code as this
364 checkout just obtained.
365
366 It is probably most useful to scan self.command.stdout for a string
367 of some sort. Be sure to set keepStdout=True on the VC command that
368 you run, so that you'll have something available to look at.
369
370 If this information is unavailable, just return None."""
371
372 return None
373
375 """
376 Read the sourcedata file and return its contents
377
378 @returns: source data
379 @raises: IOError if the file does not exist
380 """
381 return open(self.sourcedatafile, "r").read()
382
386
388 """Returns True if the tree can be updated."""
389 raise NotImplementedError("this must be implemented in a subclass")
390
392 """Returns a deferred with the steps to update a checkout."""
393 raise NotImplementedError("this must be implemented in a subclass")
394
396 """Returns a deferred with the steps to do a fresh checkout."""
397 raise NotImplementedError("this must be implemented in a subclass")
398
419
421 msg = "now retrying VC operation"
422 self.sendStatus({'header': msg + "\n"})
423 log.msg(msg)
424 d = self.doVCFull()
425 d.addBoth(self.maybeDoVCRetry)
426 d.addCallback(self._abandonOnFailure)
427 return d
428
430 """Override this in a subclass if you want to detect unrecoverable
431 checkout errors where clobbering the repo wouldn't help, and stop
432 the current VC chain before it clobbers the repo for future builds.
433
434 Use 'raise AbandonChain' to pass up a halt if you do detect such."""
435 pass
436
438 """We get here somewhere after a VC chain has finished. res could
439 be::
440
441 - 0: the operation was successful
442 - nonzero: the operation failed. retry if possible
443 - AbandonChain: the operation failed, someone else noticed. retry.
444 - Failure: some other exception, re-raise
445 """
446
447 if isinstance(res, failure.Failure):
448 if self.interrupted:
449 return res
450 res.trap(AbandonChain)
451 else:
452 if type(res) is int and res == 0:
453 return res
454 if self.interrupted:
455 raise AbandonChain(1)
456
457 if self.retry:
458 delay, repeats = self.retry
459 if repeats >= 0:
460 self.retry = (delay, repeats-1)
461 msg = ("update failed, trying %d more times after %d seconds"
462 % (repeats, delay))
463 self.sendStatus({'header': msg + "\n"})
464 log.msg(msg)
465 d = defer.Deferred()
466
467
468 self.doClobber(d, self.workdir)
469 if self.srcdir:
470 self.doClobber(d, self.srcdir)
471 d.addCallback(lambda res: self.doVCFull())
472 d.addBoth(self.maybeDoVCRetry)
473 self._reactor.callLater(delay, d.callback, None)
474 return d
475 return res
476
477 - def doClobber(self, dummy, dirname, chmodDone=False):
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496 d = os.path.join(self.builder.basedir, dirname)
497 if runtime.platformType != "posix":
498
499
500 utils.rmdirRecursive(d)
501 return defer.succeed(0)
502 command = ["rm", "-rf", d]
503 c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
504 sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
505 logEnviron=self.logEnviron, usePTY=False)
506
507 self.command = c
508
509
510
511 d = c.start()
512
513
514
515 if chmodDone:
516 d.addCallback(self._abandonOnFailure)
517 else:
518 d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname))
519 return d
520
522 assert isinstance(rc, int)
523 if rc == 0:
524 return defer.succeed(0)
525
526
527 command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, dirname)]
528 if sys.platform.startswith('freebsd'):
529
530
531
532 command = ["find", os.path.join(self.builder.basedir, dirname),
533 '-exec', 'chmod', 'u+rwx', '{}', ';' ]
534 c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
535 sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
536 logEnviron=self.logEnviron, usePTY=False)
537
538 self.command = c
539 d = c.start()
540 d.addCallback(self._abandonOnFailure)
541 d.addCallback(lambda dummy: self.doClobber(dummy, dirname, True))
542 return d
543
545
546 fromdir = os.path.join(self.builder.basedir, self.srcdir)
547 todir = os.path.join(self.builder.basedir, self.workdir)
548 if runtime.platformType != "posix":
549 self.sendStatus({'header': "Since we're on a non-POSIX platform, "
550 "we're not going to try to execute cp in a subprocess, but instead "
551 "use shutil.copytree(), which will block until it is complete. "
552 "fromdir: %s, todir: %s\n" % (fromdir, todir)})
553 shutil.copytree(fromdir, todir)
554 return defer.succeed(0)
555
556 if not os.path.exists(os.path.dirname(todir)):
557 os.makedirs(os.path.dirname(todir))
558 if os.path.exists(todir):
559
560 log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir)
561
562 command = ['cp', '-R', '-P', '-p', fromdir, todir]
563 c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
564 sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
565 logEnviron=self.logEnviron, usePTY=False)
566 self.command = c
567 d = c.start()
568 d.addCallback(self._abandonOnFailure)
569 return d
570
572 patchlevel = self.patch[0]
573 diff = self.patch[1]
574 root = None
575 if len(self.patch) >= 3:
576 root = self.patch[2]
577 command = [
578 utils.getCommand("patch"),
579 '-p%d' % patchlevel,
580 '--remove-empty-files',
581 '--force',
582 '--forward',
583 '-i', '.buildbot-diff',
584 ]
585 dir = os.path.join(self.builder.basedir, self.workdir)
586
587
588 open(os.path.join(dir, ".buildbot-patched"), "w").write("patched\n")
589
590
591 open(os.path.join(dir, ".buildbot-diff"), "w").write(diff)
592
593
594
595 if (root and
596 os.path.abspath(os.path.join(dir, root)
597 ).startswith(os.path.abspath(dir))):
598 dir = os.path.join(dir, root)
599
600
601 c = runprocess.RunProcess(self.builder, command, dir,
602 sendRC=False, timeout=self.timeout,
603 maxTime=self.maxTime, logEnviron=self.logEnviron,
604 usePTY=False)
605 self.command = c
606 d = c.start()
607
608
609 def cleanup(x):
610 try:
611 os.unlink(os.path.join(dir, ".buildbot-diff"))
612 except:
613 pass
614 return x
615 d.addBoth(cleanup)
616
617 d.addCallback(self._abandonOnFailure)
618 return d
619
620 - def setFileContents(self, filename, contents):
621 """Put the given C{contents} in C{filename}; this is a bit more
622 succinct than opening, writing, and closing, and has the advantage of
623 being patchable in tests. Note that the enclosing directory is
624 not automatically created, nor is this an "atomic" overwrite."""
625 f = open(filename, 'w')
626 f.write(contents)
627 f.close()
628