buildslave.runprocess

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 """ 17 Support for running 'shell commands' 18 """ 19 20 import sys 21 import os 22 import signal 23 import types 24 import re 25 import subprocess 26 import traceback 27 import stat 28 from collections import deque 29 from tempfile import NamedTemporaryFile 30 31 from twisted.python import runtime, log 32 from twisted.python.win32 import quoteArguments 33 from twisted.internet import reactor, defer, protocol, task, error 34 35 from buildslave import util 36 from buildslave.exceptions import AbandonChain 37 38 if runtime.platformType == 'posix': 39 from twisted.internet.process import Process 40

41 -def shell_quote(cmd_list):

42 # attempt to quote cmd_list such that a shell will properly re-interpret 43 # it. The pipes module is only available on UNIX, and Windows "shell" 44 # quoting is indescribably convoluted - so much so that it's not clear it's 45 # reversible. Also, the quote function is undocumented (although it looks 46 # like it will be documentd soon: http://bugs.python.org/issue9723). 47 # Finally, it has a nasty bug in some versions where an empty string is not 48 # quoted. 49 # 50 # So: 51 # - use pipes.quote on UNIX, handling '' as a special case 52 # - use Python's repr() on Windows, as a best effort 53 if runtime.platformType == 'win32': 54 return " ".join([ `e` for e in cmd_list ]) 55 else: 56 import pipes 57 def quote(e): 58 if not e: 59 return '""' 60 return pipes.quote(e)

61 return " ".join([ quote(e) for e in cmd_list ]) 62

63 -class LogFileWatcher:

64 POLL_INTERVAL = 2 65

66 - def __init__(self, command, name, logfile, follow=False):

67 self.command = command 68 self.name = name 69 self.logfile = logfile 70 71 log.msg("LogFileWatcher created to watch %s" % logfile) 72 # we are created before the ShellCommand starts. If the logfile we're 73 # supposed to be watching already exists, record its size and 74 # ctime/mtime so we can tell when it starts to change. 75 self.old_logfile_stats = self.statFile() 76 self.started = False 77 78 # follow the file, only sending back lines 79 # added since we started watching 80 self.follow = follow 81 82 # every 2 seconds we check on the file again 83 self.poller = task.LoopingCall(self.poll)

84

85 - def start(self):

86 self.poller.start(self.POLL_INTERVAL).addErrback(self._cleanupPoll)

87

88 - def _cleanupPoll(self, err):

89 log.err(err, msg="Polling error") 90 self.poller = None

91

92 - def stop(self):

93 self.poll() 94 if self.poller is not None: 95 self.poller.stop() 96 if self.started: 97 self.f.close()

98

99 - def statFile(self):

100 if os.path.exists(self.logfile): 101 s = os.stat(self.logfile) 102 return (s[stat.ST_CTIME], s[stat.ST_MTIME], s[stat.ST_SIZE]) 103 return None

104

105 - def poll(self):

106 if not self.started: 107 s = self.statFile() 108 if s == self.old_logfile_stats: 109 return # not started yet 110 if not s: 111 # the file was there, but now it's deleted. Forget about the 112 # initial state, clearly the process has deleted the logfile 113 # in preparation for creating a new one. 114 self.old_logfile_stats = None 115 return # no file to work with 116 self.f = open(self.logfile, "rb") 117 # if we only want new lines, seek to 118 # where we stat'd so we only find new 119 # lines 120 if self.follow: 121 self.f.seek(s[2], 0) 122 self.started = True 123 self.f.seek(self.f.tell(), 0) 124 while True: 125 data = self.f.read(10000) 126 if not data: 127 return 128 self.command.addLogfile(self.name, data)

129 130 131 if runtime.platformType == 'posix':

132 - class ProcGroupProcess(Process):

133 """Simple subclass of Process to also make the spawned process a process 134 group leader, so we can kill all members of the process group.""" 135

136 - def _setupChild(self, *args, **kwargs):

137 Process._setupChild(self, *args, **kwargs) 138 139 # this will cause the child to be the leader of its own process group; 140 # it's also spelled setpgrp() on BSD, but this spelling seems to work 141 # everywhere 142 os.setpgid(0, 0)

143 144

145 -class RunProcessPP(protocol.ProcessProtocol):

146 debug = False 147

148 - def __init__(self, command):

149 self.command = command 150 self.pending_stdin = "" 151 self.stdin_finished = False 152 self.killed = False

153

154 - def setStdin(self, data):

155 assert not self.connected 156 self.pending_stdin = data

157

158 - def connectionMade(self):

159 if self.debug: 160 log.msg("RunProcessPP.connectionMade") 161 162 if self.command.useProcGroup: 163 if self.debug: 164 log.msg(" recording pid %d as subprocess pgid" 165 % (self.transport.pid,)) 166 self.transport.pgid = self.transport.pid 167 168 if self.pending_stdin: 169 if self.debug: log.msg(" writing to stdin") 170 self.transport.write(self.pending_stdin) 171 if self.debug: log.msg(" closing stdin") 172 self.transport.closeStdin()

173

174 - def outReceived(self, data):

175 if self.debug: 176 log.msg("RunProcessPP.outReceived") 177 self.command.addStdout(data)

178

179 - def errReceived(self, data):

180 if self.debug: 181 log.msg("RunProcessPP.errReceived") 182 self.command.addStderr(data)

183

184 - def processEnded(self, status_object):

185 if self.debug: 186 log.msg("RunProcessPP.processEnded", status_object) 187 # status_object is a Failure wrapped around an 188 # error.ProcessTerminated or and error.ProcessDone. 189 # requires twisted >= 1.0.4 to overcome a bug in process.py 190 sig = status_object.value.signal 191 rc = status_object.value.exitCode 192 193 # sometimes, even when we kill a process, GetExitCodeProcess will still return 194 # a zero exit status. So we force it. See 195 # http://stackoverflow.com/questions/2061735/42-passed-to-terminateprocess-sometimes-getexitcodeprocess-returns-0 196 if self.killed and rc == 0: 197 log.msg("process was killed, but exited with status 0; faking a failure") 198 # windows returns '1' even for signalled failures, while POSIX returns -1 199 if runtime.platformType == 'win32': 200 rc = 1 201 else: 202 rc = -1 203 self.command.finished(sig, rc)

204 205

206 -class RunProcess:

207 """ 208 This is a helper class, used by slave commands to run programs in a child 209 shell. 210 """ 211 212 notreally = False 213 BACKUP_TIMEOUT = 5 214 interruptSignal = "KILL" 215 CHUNK_LIMIT = 128*1024 216 217 # Don't send any data until at least BUFFER_SIZE bytes have been collected 218 # or BUFFER_TIMEOUT elapsed 219 BUFFER_SIZE = 64*1024 220 BUFFER_TIMEOUT = 5 221 222 # For sending elapsed time: 223 startTime = None 224 elapsedTime = None 225 226 # For scheduling future events 227 _reactor = reactor 228 229 # I wish we had easy access to CLOCK_MONOTONIC in Python: 230 # http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html 231 # Then changes to the system clock during a run wouldn't effect the "elapsed 232 # time" results. 233

234 - def __init__(self, builder, command, 235 workdir, environ=None, 236 sendStdout=True, sendStderr=True, sendRC=True, 237 timeout=None, maxTime=None, initialStdin=None, 238 keepStdout=False, keepStderr=False, 239 logEnviron=True, logfiles={}, usePTY="slave-config", 240 useProcGroup=True):

241 """ 242 243 @param keepStdout: if True, we keep a copy of all the stdout text 244 that we've seen. This copy is available in 245 self.stdout, which can be read after the command 246 has finished. 247 @param keepStderr: same, for stderr 248 249 @param usePTY: "slave-config" -> use the SlaveBuilder's usePTY; 250 otherwise, true to use a PTY, false to not use a PTY. 251 252 @param useProcGroup: (default True) use a process group for non-PTY 253 process invocations 254 """ 255 256 self.builder = builder 257 258 # We need to take unicode commands and arguments and encode them using 259 # the appropriate encoding for the slave. This is mostly platform 260 # specific, but can be overridden in the slave's buildbot.tac file. 261 # 262 # Encoding the command line here ensures that the called executables 263 # receive arguments as bytestrings encoded with an appropriate 264 # platform-specific encoding. It also plays nicely with twisted's 265 # spawnProcess which checks that arguments are regular strings or 266 # unicode strings that can be encoded as ascii (which generates a 267 # warning). 268 def to_str(cmd): 269 if isinstance(cmd, (tuple, list)): 270 for i, a in enumerate(cmd): 271 if isinstance(a, unicode): 272 cmd[i] = a.encode(self.builder.unicode_encoding) 273 elif isinstance(cmd, unicode): 274 cmd = cmd.encode(self.builder.unicode_encoding) 275 return cmd

276 277 self.command = to_str(util.Obfuscated.get_real(command)) 278 self.fake_command = to_str(util.Obfuscated.get_fake(command)) 279 280 self.sendStdout = sendStdout 281 self.sendStderr = sendStderr 282 self.sendRC = sendRC 283 self.logfiles = logfiles 284 self.workdir = workdir 285 self.process = None 286 if not os.path.exists(workdir): 287 os.makedirs(workdir) 288 if environ: 289 for key, v in environ.iteritems(): 290 if isinstance(v, list): 291 # Need to do os.pathsep translation. We could either do that 292 # by replacing all incoming ':'s with os.pathsep, or by 293 # accepting lists. I like lists better. 294 # If it's not a string, treat it as a sequence to be 295 # turned in to a string. 296 environ[key] = os.pathsep.join(environ[key]) 297 298 if environ.has_key('PYTHONPATH'): 299 environ['PYTHONPATH'] += os.pathsep + "${PYTHONPATH}" 300 301 # do substitution on variable values matching pattern: ${name} 302 p = re.compile('\${([0-9a-zA-Z_]*)}') 303 def subst(match): 304 return os.environ.get(match.group(1), "")

305 newenv = {} 306 for key in os.environ.keys(): 307 # setting a key to None will delete it from the slave environment 308 if key not in environ or environ[key] is not None: 309 newenv[key] = os.environ[key] 310 for key, v in environ.iteritems(): 311 if v is not None: 312 if not isinstance(v, basestring): 313 raise RuntimeError("'env' values must be strings or " 314 "lists; key '%s' is incorrect" % (key,)) 315 newenv[key] = p.sub(subst, v) 316 317 self.environ = newenv 318 else: # not environ 319 self.environ = os.environ.copy() 320 self.initialStdin = initialStdin 321 self.logEnviron = logEnviron 322 self.timeout = timeout 323 self.timer = None 324 self.maxTime = maxTime 325 self.maxTimer = None 326 self.keepStdout = keepStdout 327 self.keepStderr = keepStderr 328 329 self.buffered = deque() 330 self.buflen = 0 331 self.buftimer = None 332 333 if usePTY == "slave-config": 334 self.usePTY = self.builder.usePTY 335 else: 336 self.usePTY = usePTY 337 338 # usePTY=True is a convenience for cleaning up all children and 339 # grandchildren of a hung command. Fall back to usePTY=False on systems 340 # and in situations where ptys cause problems. PTYs are posix-only, 341 # and for .closeStdin to matter, we must use a pipe, not a PTY 342 if runtime.platformType != "posix" or initialStdin is not None: 343 if self.usePTY and usePTY != "slave-config": 344 self.sendStatus({'header': "WARNING: disabling usePTY for this command"}) 345 self.usePTY = False 346 347 # use an explicit process group on POSIX, noting that usePTY always implies 348 # a process group. 349 if runtime.platformType != 'posix': 350 useProcGroup = False 351 elif self.usePTY: 352 useProcGroup = True 353 self.useProcGroup = useProcGroup 354 355 self.logFileWatchers = [] 356 for name,filevalue in self.logfiles.items(): 357 filename = filevalue 358 follow = False 359 360 # check for a dictionary of options 361 # filename is required, others are optional 362 if type(filevalue) == dict: 363 filename = filevalue['filename'] 364 follow = filevalue.get('follow', False) 365 366 w = LogFileWatcher(self, name, 367 os.path.join(self.workdir, filename), 368 follow=follow) 369 self.logFileWatchers.append(w) 370

371 - def __repr__(self):

372 return "<%s '%s'>" % (self.__class__.__name__, self.fake_command)

373

374 - def sendStatus(self, status):

375 self.builder.sendUpdate(status)

376

377 - def start(self):

378 # return a Deferred which fires (with the exit code) when the command 379 # completes 380 if self.keepStdout: 381 self.stdout = "" 382 if self.keepStderr: 383 self.stderr = "" 384 self.deferred = defer.Deferred() 385 try: 386 self._startCommand() 387 except: 388 log.msg("error in RunProcess._startCommand") 389 log.err() 390 self._addToBuffers('stderr', "error in RunProcess._startCommand\n") 391 self._addToBuffers('stderr', traceback.format_exc()) 392 self._sendBuffers() 393 # pretend it was a shell error 394 self.deferred.errback(AbandonChain(-1)) 395 return self.deferred

396

397 - def _startCommand(self):

398 # ensure workdir exists 399 if not os.path.isdir(self.workdir): 400 os.makedirs(self.workdir) 401 log.msg("RunProcess._startCommand") 402 if self.notreally: 403 self._addToBuffers('header', "command '%s' in dir %s" % \ 404 (self.fake_command, self.workdir)) 405 self._addToBuffers('header', "(not really)\n") 406 self.finished(None, 0) 407 return 408 409 self.pp = RunProcessPP(self) 410 411 self.using_comspec = False 412 if type(self.command) in types.StringTypes: 413 if runtime.platformType == 'win32': 414 argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args 415 if '/c' not in argv: argv += ['/c'] 416 argv += [self.command] 417 self.using_comspec = True 418 else: 419 # for posix, use /bin/sh. for other non-posix, well, doesn't 420 # hurt to try 421 argv = ['/bin/sh', '-c', self.command] 422 display = self.fake_command 423 else: 424 # On windows, CreateProcess requires an absolute path to the executable. 425 # When we call spawnProcess below, we pass argv[0] as the executable. 426 # So, for .exe's that we have absolute paths to, we can call directly 427 # Otherwise, we should run under COMSPEC (usually cmd.exe) to 428 # handle path searching, etc. 429 if runtime.platformType == 'win32' and not \ 430 (self.command[0].lower().endswith(".exe") and os.path.isabs(self.command[0])): 431 argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args 432 if '/c' not in argv: argv += ['/c'] 433 argv += list(self.command) 434 self.using_comspec = True 435 else: 436 argv = self.command 437 # Attempt to format this for use by a shell, although the process isn't perfect 438 display = shell_quote(self.fake_command) 439 440 # $PWD usually indicates the current directory; spawnProcess may not 441 # update this value, though, so we set it explicitly here. This causes 442 # weird problems (bug #456) on msys, though.. 443 if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys': 444 self.environ['PWD'] = os.path.abspath(self.workdir) 445 446 # self.stdin is handled in RunProcessPP.connectionMade 447 448 log.msg(" " + display) 449 self._addToBuffers('header', display+"\n") 450 451 # then comes the secondary information 452 msg = " in dir %s" % (self.workdir,) 453 if self.timeout: 454 if self.timeout == 1: 455 unit = "sec" 456 else: 457 unit = "secs" 458 msg += " (timeout %d %s)" % (self.timeout, unit) 459 if self.maxTime: 460 if self.maxTime == 1: 461 unit = "sec" 462 else: 463 unit = "secs" 464 msg += " (maxTime %d %s)" % (self.maxTime, unit) 465 log.msg(" " + msg) 466 self._addToBuffers('header', msg+"\n") 467 468 msg = " watching logfiles %s" % (self.logfiles,) 469 log.msg(" " + msg) 470 self._addToBuffers('header', msg+"\n") 471 472 # then the obfuscated command array for resolving unambiguity 473 msg = " argv: %s" % (self.fake_command,) 474 log.msg(" " + msg) 475 self._addToBuffers('header', msg+"\n") 476 477 # then the environment, since it sometimes causes problems 478 if self.logEnviron: 479 msg = " environment:\n" 480 env_names = self.environ.keys() 481 env_names.sort() 482 for name in env_names: 483 msg += " %s=%s\n" % (name, self.environ[name]) 484 log.msg(" environment: %s" % (self.environ,)) 485 self._addToBuffers('header', msg) 486 487 if self.initialStdin: 488 msg = " writing %d bytes to stdin" % len(self.initialStdin) 489 log.msg(" " + msg) 490 self._addToBuffers('header', msg+"\n") 491 492 msg = " using PTY: %s" % bool(self.usePTY) 493 log.msg(" " + msg) 494 self._addToBuffers('header', msg+"\n") 495 496 # put data into stdin and close it, if necessary. This will be 497 # buffered until connectionMade is called 498 if self.initialStdin: 499 self.pp.setStdin(self.initialStdin) 500 501 self.startTime = util.now(self._reactor) 502 503 # start the process 504 505 self.process = self._spawnProcess( 506 self.pp, argv[0], argv, 507 self.environ, 508 self.workdir, 509 usePTY=self.usePTY) 510 511 # set up timeouts 512 513 if self.timeout: 514 self.timer = self._reactor.callLater(self.timeout, self.doTimeout) 515 516 if self.maxTime: 517 self.maxTimer = self._reactor.callLater(self.maxTime, self.doMaxTimeout) 518 519 for w in self.logFileWatchers: 520 w.start()

521

522 - def _spawnProcess(self, processProtocol, executable, args=(), env={}, 523 path=None, uid=None, gid=None, usePTY=False, childFDs=None):

524 """private implementation of reactor.spawnProcess, to allow use of 525 L{ProcGroupProcess}""" 526 527 # use the ProcGroupProcess class, if available 528 if runtime.platformType == 'posix': 529 if self.useProcGroup and not usePTY: 530 return ProcGroupProcess(reactor, executable, args, env, path, 531 processProtocol, uid, gid, childFDs) 532 533 # fall back 534 if self.using_comspec: 535 return self._spawnAsBatch(processProtocol, executable, args, env, 536 path, usePTY=usePTY) 537 else: 538 return reactor.spawnProcess(processProtocol, executable, args, env, 539 path, usePTY=usePTY)

540

541 - def _spawnAsBatch(self, processProtocol, executable, args, env, 542 path, usePTY):

543 """A cheat that routes around the impedance mismatch between 544 twisted and cmd.exe with respect to escaping quotes""" 545 546 tf = NamedTemporaryFile(dir='.',suffix=".bat",delete=False) 547 #echo off hides this cheat from the log files. 548 tf.write( "@echo off\n" ) 549 if type(self.command) in types.StringTypes: 550 tf.write( self.command ) 551 else: 552 def maybe_escape_pipes(arg): 553 if arg != '|': 554 return arg.replace('|','^|') 555 else: 556 return '|'

557 cmd = [maybe_escape_pipes(arg) for arg in self.command] 558 tf.write( quoteArguments(cmd) ) 559 tf.close() 560 561 argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args 562 if '/c' not in argv: argv += ['/c'] 563 argv += [tf.name] 564 565 def unlink_temp(result): 566 os.unlink(tf.name) 567 return result 568 self.deferred.addBoth(unlink_temp) 569 570 return reactor.spawnProcess(processProtocol, executable, argv, env, 571 path, usePTY=usePTY) 572

573 - def _chunkForSend(self, data):

574 """ 575 limit the chunks that we send over PB to 128k, since it has a hardwired 576 string-size limit of 640k. 577 """ 578 LIMIT = self.CHUNK_LIMIT 579 for i in range(0, len(data), LIMIT): 580 yield data[i:i+LIMIT]

581

582 - def _collapseMsg(self, msg):

583 """ 584 Take msg, which is a dictionary of lists of output chunks, and 585 concatentate all the chunks into a single string 586 """ 587 retval = {} 588 for log in msg: 589 data = "".join(msg[log]) 590 if isinstance(log, tuple) and log[0] == 'log': 591 retval['log'] = (log[1], data) 592 else: 593 retval[log] = data 594 return retval

595

596 - def _sendMessage(self, msg):

597 """ 598 Collapse and send msg to the master 599 """ 600 if not msg: 601 return 602 msg = self._collapseMsg(msg) 603 self.sendStatus(msg)

604

605 - def _bufferTimeout(self):

606 self.buftimer = None 607 self._sendBuffers()

608

609 - def _sendBuffers(self):

610 """ 611 Send all the content in our buffers. 612 """ 613 msg = {} 614 msg_size = 0 615 lastlog = None 616 logdata = [] 617 while self.buffered: 618 # Grab the next bits from the buffer 619 logname, data = self.buffered.popleft() 620 621 # If this log is different than the last one, then we have to send 622 # out the message so far. This is because the message is 623 # transferred as a dictionary, which makes the ordering of keys 624 # unspecified, and makes it impossible to interleave data from 625 # different logs. A future enhancement could be to change the 626 # master to support a list of (logname, data) tuples instead of a 627 # dictionary. 628 # On our first pass through this loop lastlog is None 629 if lastlog is None: 630 lastlog = logname 631 elif logname != lastlog: 632 self._sendMessage(msg) 633 msg = {} 634 msg_size = 0 635 lastlog = logname 636 637 logdata = msg.setdefault(logname, []) 638 639 # Chunkify the log data to make sure we're not sending more than 640 # CHUNK_LIMIT at a time 641 for chunk in self._chunkForSend(data): 642 if len(chunk) == 0: continue 643 logdata.append(chunk) 644 msg_size += len(chunk) 645 if msg_size >= self.CHUNK_LIMIT: 646 # We've gone beyond the chunk limit, so send out our 647 # message. At worst this results in a message slightly 648 # larger than (2*CHUNK_LIMIT)-1 649 self._sendMessage(msg) 650 msg = {} 651 logdata = msg.setdefault(logname, []) 652 msg_size = 0 653 self.buflen = 0 654 if logdata: 655 self._sendMessage(msg) 656 if self.buftimer: 657 if self.buftimer.active(): 658 self.buftimer.cancel() 659 self.buftimer = None

660

661 - def _addToBuffers(self, logname, data):

662 """ 663 Add data to the buffer for logname 664 Start a timer to send the buffers if BUFFER_TIMEOUT elapses. 665 If adding data causes the buffer size to grow beyond BUFFER_SIZE, then 666 the buffers will be sent. 667 """ 668 n = len(data) 669 670 self.buflen += n 671 self.buffered.append((logname, data)) 672 if self.buflen > self.BUFFER_SIZE: 673 self._sendBuffers() 674 elif not self.buftimer: 675 self.buftimer = self._reactor.callLater(self.BUFFER_TIMEOUT, self._bufferTimeout)

676

677 - def addStdout(self, data):

678 if self.sendStdout: 679 self._addToBuffers('stdout', data) 680 681 if self.keepStdout: 682 self.stdout += data 683 if self.timer: 684 self.timer.reset(self.timeout)

685

686 - def addStderr(self, data):

687 if self.sendStderr: 688 self._addToBuffers('stderr', data) 689 690 if self.keepStderr: 691 self.stderr += data 692 if self.timer: 693 self.timer.reset(self.timeout)

694

695 - def addLogfile(self, name, data):

696 self._addToBuffers( ('log', name), data) 697 698 if self.timer: 699 self.timer.reset(self.timeout)

700

701 - def finished(self, sig, rc):

702 self.elapsedTime = util.now(self._reactor) - self.startTime 703 log.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (sig,rc,self.elapsedTime)) 704 for w in self.logFileWatchers: 705 # this will send the final updates 706 w.stop() 707 self._sendBuffers() 708 if sig is not None: 709 rc = -1 710 if self.sendRC: 711 if sig is not None: 712 self.sendStatus( 713 {'header': "process killed by signal %d\n" % sig}) 714 self.sendStatus({'rc': rc}) 715 self.sendStatus({'header': "elapsedTime=%0.6f\n" % self.elapsedTime}) 716 if self.timer: 717 self.timer.cancel() 718 self.timer = None 719 if self.maxTimer: 720 self.maxTimer.cancel() 721 self.maxTimer = None 722 if self.buftimer: 723 self.buftimer.cancel() 724 self.buftimer = None 725 d = self.deferred 726 self.deferred = None 727 if d: 728 d.callback(rc) 729 else: 730 log.msg("Hey, command %s finished twice" % self)

731

732 - def failed(self, why):

733 self._sendBuffers() 734 log.msg("RunProcess.failed: command failed: %s" % (why,)) 735 if self.timer: 736 self.timer.cancel() 737 self.timer = None 738 if self.maxTimer: 739 self.maxTimer.cancel() 740 self.maxTimer = None 741 if self.buftimer: 742 self.buftimer.cancel() 743 self.buftimer = None 744 d = self.deferred 745 self.deferred = None 746 if d: 747 d.errback(why) 748 else: 749 log.msg("Hey, command %s finished twice" % self)

750

751 - def doTimeout(self):

752 self.timer = None 753 msg = "command timed out: %d seconds without output" % self.timeout 754 self.kill(msg)

755

756 - def doMaxTimeout(self):

757 self.maxTimer = None 758 msg = "command timed out: %d seconds elapsed" % self.maxTime 759 self.kill(msg)

760

761 - def kill(self, msg):

762 # This may be called by the timeout, or when the user has decided to 763 # abort this build. 764 self._sendBuffers() 765 if self.timer: 766 self.timer.cancel() 767 self.timer = None 768 if self.maxTimer: 769 self.maxTimer.cancel() 770 self.maxTimer = None 771 if self.buftimer: 772 self.buftimer.cancel() 773 self.buftimer = None 774 msg += ", attempting to kill" 775 log.msg(msg) 776 self.sendStatus({'header': "\n" + msg + "\n"}) 777 778 # let the PP know that we are killing it, so that it can ensure that 779 # the exit status comes out right 780 self.pp.killed = True 781 782 # keep track of whether we believe we've successfully killed something 783 hit = 0 784 785 # try signalling the process group 786 if not hit and self.useProcGroup and runtime.platformType == "posix": 787 sig = getattr(signal, "SIG"+ self.interruptSignal, None) 788 789 if sig is None: 790 log.msg("signal module is missing SIG%s" % self.interruptSignal) 791 elif not hasattr(os, "kill"): 792 log.msg("os module is missing the 'kill' function") 793 elif self.process.pgid is None: 794 log.msg("self.process has no pgid") 795 else: 796 log.msg("trying to kill process group %d" % 797 (self.process.pgid,)) 798 try: 799 os.kill(-self.process.pgid, sig) 800 log.msg(" signal %s sent successfully" % sig) 801 self.process.pgid = None 802 hit = 1 803 except OSError: 804 log.msg('failed to kill process group (ignored): %s' % 805 (sys.exc_info()[1],)) 806 # probably no-such-process, maybe because there is no process 807 # group 808 pass 809 810 elif runtime.platformType == "win32": 811 if self.interruptSignal == None: 812 log.msg("self.interruptSignal==None, only pretending to kill child") 813 elif self.process.pid is not None: 814 log.msg("using TASKKILL /F PID /T to kill pid %s" % self.process.pid) 815 subprocess.check_call("TASKKILL /F /PID %s /T" % self.process.pid) 816 log.msg("taskkill'd pid %s" % self.process.pid) 817 hit = 1 818 819 # try signalling the process itself (works on Windows too, sorta) 820 if not hit: 821 try: 822 log.msg("trying process.signalProcess('%s')" % (self.interruptSignal,)) 823 self.process.signalProcess(self.interruptSignal) 824 log.msg(" signal %s sent successfully" % (self.interruptSignal,)) 825 hit = 1 826 except OSError: 827 log.err("from process.signalProcess:") 828 # could be no-such-process, because they finished very recently 829 pass 830 except error.ProcessExitedAlready: 831 log.msg("Process exited already - can't kill") 832 # the process has already exited, and likely finished() has 833 # been called already or will be called shortly 834 pass 835 836 if not hit: 837 log.msg("signalProcess/os.kill failed both times") 838 839 if runtime.platformType == "posix": 840 # we only do this under posix because the win32eventreactor 841 # blocks here until the process has terminated, while closing 842 # stderr. This is weird. 843 self.pp.transport.loseConnection() 844 845 if self.deferred: 846 # finished ought to be called momentarily. Just in case it doesn't, 847 # set a timer which will abandon the command. 848 self.timer = self._reactor.callLater(self.BACKUP_TIMEOUT, 849 self.doBackupTimeout)

850

851 - def doBackupTimeout(self):

852 log.msg("we tried to kill the process, and it wouldn't die.." 853 " finish anyway") 854 self.timer = None 855 self.sendStatus({'header': "SIGKILL failed to kill process\n"}) 856 if self.sendRC: 857 self.sendStatus({'header': "using fake rc=-1\n"}) 858 self.sendStatus({'rc': -1}) 859 self.failed(RuntimeError("SIGKILL failed to kill process"))

860

Source Code for Module buildslave.runprocess