buildbot.steps.transfer

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import os.path, tarfile, tempfile 18 try: 19 from cStringIO import StringIO 20 assert StringIO 21 except ImportError: 22 from StringIO import StringIO 23 from twisted.internet import reactor 24 from twisted.spread import pb 25 from twisted.python import log 26 from buildbot.process.buildstep import RemoteCommand, BuildStep 27 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 28 from buildbot.interfaces import BuildSlaveTooOldError 29 from buildbot.util import json 30 31

32 -class _FileWriter(pb.Referenceable):

33 """ 34 Helper class that acts as a file-object with write access 35 """ 36

37 - def __init__(self, destfile, maxsize, mode):

38 # Create missing directories. 39 destfile = os.path.abspath(destfile) 40 dirname = os.path.dirname(destfile) 41 if not os.path.exists(dirname): 42 os.makedirs(dirname) 43 44 self.destfile = destfile 45 self.mode = mode 46 fd, self.tmpname = tempfile.mkstemp(dir=dirname) 47 self.fp = os.fdopen(fd, 'wb') 48 self.remaining = maxsize

49

50 - def remote_write(self, data):

51 """ 52 Called from remote slave to write L{data} to L{fp} within boundaries 53 of L{maxsize} 54 55 @type data: C{string} 56 @param data: String of data to write 57 """ 58 if self.remaining is not None: 59 if len(data) > self.remaining: 60 data = data[:self.remaining] 61 self.fp.write(data) 62 self.remaining = self.remaining - len(data) 63 else: 64 self.fp.write(data)

65

66 - def remote_utime(self, accessed_modified):

67 os.utime(self.destfile,accessed_modified)

68

69 - def remote_close(self):

70 """ 71 Called by remote slave to state that no more data will be transfered 72 """ 73 self.fp.close() 74 self.fp = None 75 # on windows, os.rename does not automatically unlink, so do it manually 76 if os.path.exists(self.destfile): 77 os.unlink(self.destfile) 78 os.rename(self.tmpname, self.destfile) 79 self.tmpname = None 80 if self.mode is not None: 81 os.chmod(self.destfile, self.mode)

82

83 - def __del__(self):

84 # unclean shutdown, the file is probably truncated, so delete it 85 # altogether rather than deliver a corrupted file 86 fp = getattr(self, "fp", None) 87 if fp: 88 fp.close() 89 os.unlink(self.destfile) 90 if self.tmpname and os.path.exists(self.tmpname): 91 os.unlink(self.tmpname)

92 93

94 -def _extractall(self, path=".", members=None):

95 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 96 97 import copy 98 99 directories = [] 100 101 if members is None: 102 members = self 103 104 for tarinfo in members: 105 if tarinfo.isdir(): 106 # Extract directories with a safe mode. 107 directories.append(tarinfo) 108 tarinfo = copy.copy(tarinfo) 109 tarinfo.mode = 0700 110 self.extract(tarinfo, path) 111 112 # Reverse sort directories. 113 directories.sort(lambda a, b: cmp(a.name, b.name)) 114 directories.reverse() 115 116 # Set correct owner, mtime and filemode on directories. 117 for tarinfo in directories: 118 dirpath = os.path.join(path, tarinfo.name) 119 try: 120 self.chown(tarinfo, dirpath) 121 self.utime(tarinfo, dirpath) 122 self.chmod(tarinfo, dirpath) 123 except tarfile.ExtractError, e: 124 if self.errorlevel > 1: 125 raise 126 else: 127 self._dbg(1, "tarfile: %s" % e)

128

129 -class _DirectoryWriter(_FileWriter):

130 """ 131 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 132 step to unpack the archive, once the transfer has completed. 133 """ 134

135 - def __init__(self, destroot, maxsize, compress, mode):

136 self.destroot = destroot 137 self.compress = compress 138 139 self.fd, self.tarname = tempfile.mkstemp() 140 os.close(self.fd) 141 142 _FileWriter.__init__(self, self.tarname, maxsize, mode)

143

144 - def remote_unpack(self):

145 """ 146 Called by remote slave to state that no more data will be transfered 147 """ 148 # Make sure remote_close is called, otherwise atomic rename wont happen 149 self.remote_close() 150 151 # Map configured compression to a TarFile setting 152 if self.compress == 'bz2': 153 mode='r|bz2' 154 elif self.compress == 'gz': 155 mode='r|gz' 156 else: 157 mode = 'r' 158 159 # Support old python 160 if not hasattr(tarfile.TarFile, 'extractall'): 161 tarfile.TarFile.extractall = _extractall 162 163 # Unpack archive and clean up after self 164 archive = tarfile.open(name=self.tarname, mode=mode) 165 archive.extractall(path=self.destroot) 166 archive.close() 167 os.remove(self.tarname)

168 169

170 -class StatusRemoteCommand(RemoteCommand):

171 - def __init__(self, remote_command, args):

172 RemoteCommand.__init__(self, remote_command, args) 173 174 self.rc = None 175 self.stderr = ''

176

177 - def remoteUpdate(self, update):

178 #log.msg('StatusRemoteCommand: update=%r' % update) 179 if 'rc' in update: 180 self.rc = update['rc'] 181 if 'stderr' in update: 182 self.stderr = self.stderr + update['stderr'] + '\n'

183

184 -class _TransferBuildStep(BuildStep):

185 """ 186 Base class for FileUpload and FileDownload to factor out common 187 functionality. 188 """ 189 DEFAULT_WORKDIR = "build" # is this redundant? 190 191 renderables = [ 'workdir' ] 192 193 haltOnFailure = True 194 flunkOnFailure = True 195

196 - def setDefaultWorkdir(self, workdir):

197 if self.workdir is None: 198 self.workdir = workdir

199

200 - def _getWorkdir(self):

201 if self.workdir is None: 202 workdir = self.DEFAULT_WORKDIR 203 else: 204 workdir = self.workdir 205 return workdir

206

207 - def interrupt(self, reason):

208 self.addCompleteLog('interrupt', str(reason)) 209 if self.cmd: 210 d = self.cmd.interrupt(reason) 211 return d

212

213 - def finished(self, result):

214 # Subclasses may choose to skip a transfer. In those cases, self.cmd 215 # will be None, and we should just let BuildStep.finished() handle 216 # the rest 217 if result == SKIPPED: 218 return BuildStep.finished(self, SKIPPED) 219 if self.cmd.stderr != '': 220 self.addCompleteLog('stderr', self.cmd.stderr) 221 222 if self.cmd.rc is None or self.cmd.rc == 0: 223 return BuildStep.finished(self, SUCCESS) 224 return BuildStep.finished(self, FAILURE)

225 226

227 -class FileUpload(_TransferBuildStep):

228 """ 229 Build step to transfer a file from the slave to the master. 230 231 arguments: 232 233 - ['slavesrc'] filename of source file at slave, relative to workdir 234 - ['masterdest'] filename of destination file at master 235 - ['workdir'] string with slave working directory relative to builder 236 base dir, default 'build' 237 - ['maxsize'] maximum size of the file, default None (=unlimited) 238 - ['blocksize'] maximum size of each block being transfered 239 - ['mode'] file access mode for the resulting master-side file. 240 The default (=None) is to leave it up to the umask of 241 the buildmaster process. 242 - ['keepstamp'] whether to preserve file modified and accessed times 243 244 """ 245 246 name = 'upload' 247 248 renderables = [ 'slavesrc', 'masterdest' ] 249

250 - def __init__(self, slavesrc, masterdest, 251 workdir=None, maxsize=None, blocksize=16*1024, mode=None, keepstamp=False, 252 **buildstep_kwargs):

253 BuildStep.__init__(self, **buildstep_kwargs) 254 self.addFactoryArguments(slavesrc=slavesrc, 255 masterdest=masterdest, 256 workdir=workdir, 257 maxsize=maxsize, 258 blocksize=blocksize, 259 mode=mode, 260 keepstamp=keepstamp, 261 ) 262 263 self.slavesrc = slavesrc 264 self.masterdest = masterdest 265 self.workdir = workdir 266 self.maxsize = maxsize 267 self.blocksize = blocksize 268 assert isinstance(mode, (int, type(None))) 269 self.mode = mode 270 self.keepstamp = keepstamp

271

272 - def start(self):

273 version = self.slaveVersion("uploadFile") 274 275 if not version: 276 m = "slave is too old, does not know about uploadFile" 277 raise BuildSlaveTooOldError(m) 278 279 source = self.slavesrc 280 masterdest = self.masterdest 281 # we rely upon the fact that the buildmaster runs chdir'ed into its 282 # basedir to make sure that relative paths in masterdest are expanded 283 # properly. TODO: maybe pass the master's basedir all the way down 284 # into the BuildStep so we can do this better. 285 masterdest = os.path.expanduser(masterdest) 286 log.msg("FileUpload started, from slave %r to master %r" 287 % (source, masterdest)) 288 289 self.step_status.setText(['uploading', os.path.basename(source)]) 290 291 # we use maxsize to limit the amount of data on both sides 292 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 293 294 if self.keepstamp and self.slaveVersionIsOlderThan("uploadFile","2.13"): 295 m = ("This buildslave (%s) does not support preserving timestamps. " 296 "Please upgrade the buildslave." % self.build.slavename ) 297 raise BuildSlaveTooOldError(m) 298 299 # default arguments 300 args = { 301 'slavesrc': source, 302 'workdir': self._getWorkdir(), 303 'writer': fileWriter, 304 'maxsize': self.maxsize, 305 'blocksize': self.blocksize, 306 'keepstamp': self.keepstamp, 307 } 308 309 self.cmd = StatusRemoteCommand('uploadFile', args) 310 d = self.runCommand(self.cmd) 311 d.addCallback(self.finished).addErrback(self.failed)

312 313

314 -class DirectoryUpload(BuildStep):

315 """ 316 Build step to transfer a directory from the slave to the master. 317 318 arguments: 319 320 - ['slavesrc'] name of source directory at slave, relative to workdir 321 - ['masterdest'] name of destination directory at master 322 - ['workdir'] string with slave working directory relative to builder 323 base dir, default 'build' 324 - ['maxsize'] maximum size of the compressed tarfile containing the 325 whole directory 326 - ['blocksize'] maximum size of each block being transfered 327 - ['compress'] compression type to use: one of [None, 'gz', 'bz2'] 328 329 """ 330 331 name = 'upload' 332 333 renderables = [ 'slavesrc', 'masterdest' ] 334

335 - def __init__(self, slavesrc, masterdest, 336 workdir="build", maxsize=None, blocksize=16*1024, 337 compress=None, **buildstep_kwargs):

338 BuildStep.__init__(self, **buildstep_kwargs) 339 self.addFactoryArguments(slavesrc=slavesrc, 340 masterdest=masterdest, 341 workdir=workdir, 342 maxsize=maxsize, 343 blocksize=blocksize, 344 compress=compress, 345 ) 346 347 self.slavesrc = slavesrc 348 self.masterdest = masterdest 349 self.workdir = workdir 350 self.maxsize = maxsize 351 self.blocksize = blocksize 352 assert compress in (None, 'gz', 'bz2') 353 self.compress = compress

354

355 - def start(self):

356 version = self.slaveVersion("uploadDirectory") 357 358 if not version: 359 m = "slave is too old, does not know about uploadDirectory" 360 raise BuildSlaveTooOldError(m) 361 362 source = self.slavesrc 363 masterdest = self.masterdest 364 # we rely upon the fact that the buildmaster runs chdir'ed into its 365 # basedir to make sure that relative paths in masterdest are expanded 366 # properly. TODO: maybe pass the master's basedir all the way down 367 # into the BuildStep so we can do this better. 368 masterdest = os.path.expanduser(masterdest) 369 log.msg("DirectoryUpload started, from slave %r to master %r" 370 % (source, masterdest)) 371 372 self.step_status.setText(['uploading', os.path.basename(source)]) 373 374 # we use maxsize to limit the amount of data on both sides 375 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600) 376 377 # default arguments 378 args = { 379 'slavesrc': source, 380 'workdir': self.workdir, 381 'writer': dirWriter, 382 'maxsize': self.maxsize, 383 'blocksize': self.blocksize, 384 'compress': self.compress 385 } 386 387 self.cmd = StatusRemoteCommand('uploadDirectory', args) 388 d = self.runCommand(self.cmd) 389 d.addCallback(self.finished).addErrback(self.failed)

390

391 - def finished(self, result):

392 # Subclasses may choose to skip a transfer. In those cases, self.cmd 393 # will be None, and we should just let BuildStep.finished() handle 394 # the rest 395 if result == SKIPPED: 396 return BuildStep.finished(self, SKIPPED) 397 if self.cmd.stderr != '': 398 self.addCompleteLog('stderr', self.cmd.stderr) 399 400 if self.cmd.rc is None or self.cmd.rc == 0: 401 return BuildStep.finished(self, SUCCESS) 402 return BuildStep.finished(self, FAILURE)

403 404 405 406

407 -class _FileReader(pb.Referenceable):

408 """ 409 Helper class that acts as a file-object with read access 410 """ 411

412 - def __init__(self, fp):

413 self.fp = fp

414

415 - def remote_read(self, maxlength):

416 """ 417 Called from remote slave to read at most L{maxlength} bytes of data 418 419 @type maxlength: C{integer} 420 @param maxlength: Maximum number of data bytes that can be returned 421 422 @return: Data read from L{fp} 423 @rtype: C{string} of bytes read from file 424 """ 425 if self.fp is None: 426 return '' 427 428 data = self.fp.read(maxlength) 429 return data

430

431 - def remote_close(self):

432 """ 433 Called by remote slave to state that no more data will be transfered 434 """ 435 if self.fp is not None: 436 self.fp.close() 437 self.fp = None

438 439

440 -class FileDownload(_TransferBuildStep):

441 """ 442 Download the first 'maxsize' bytes of a file, from the buildmaster to the 443 buildslave. Set the mode of the file 444 445 Arguments:: 446 447 ['mastersrc'] filename of source file at master 448 ['slavedest'] filename of destination file at slave 449 ['workdir'] string with slave working directory relative to builder 450 base dir, default 'build' 451 ['maxsize'] maximum size of the file, default None (=unlimited) 452 ['blocksize'] maximum size of each block being transfered 453 ['mode'] use this to set the access permissions of the resulting 454 buildslave-side file. This is traditionally an octal 455 integer, like 0644 to be world-readable (but not 456 world-writable), or 0600 to only be readable by 457 the buildslave account, or 0755 to be world-executable. 458 The default (=None) is to leave it up to the umask of 459 the buildslave process. 460 461 """ 462 name = 'download' 463 464 renderables = [ 'mastersrc', 'slavedest' ] 465

466 - def __init__(self, mastersrc, slavedest, 467 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 468 **buildstep_kwargs):

469 BuildStep.__init__(self, **buildstep_kwargs) 470 self.addFactoryArguments(mastersrc=mastersrc, 471 slavedest=slavedest, 472 workdir=workdir, 473 maxsize=maxsize, 474 blocksize=blocksize, 475 mode=mode, 476 ) 477 478 self.mastersrc = mastersrc 479 self.slavedest = slavedest 480 self.workdir = workdir 481 self.maxsize = maxsize 482 self.blocksize = blocksize 483 assert isinstance(mode, (int, type(None))) 484 self.mode = mode

485

486 - def start(self):

487 version = self.slaveVersion("downloadFile") 488 if not version: 489 m = "slave is too old, does not know about downloadFile" 490 raise BuildSlaveTooOldError(m) 491 492 # we are currently in the buildmaster's basedir, so any non-absolute 493 # paths will be interpreted relative to that 494 source = os.path.expanduser(self.mastersrc) 495 slavedest = self.slavedest 496 log.msg("FileDownload started, from master %r to slave %r" % 497 (source, slavedest)) 498 499 self.step_status.setText(['downloading', "to", 500 os.path.basename(slavedest)]) 501 502 # setup structures for reading the file 503 try: 504 fp = open(source, 'rb') 505 except IOError: 506 # if file does not exist, bail out with an error 507 self.addCompleteLog('stderr', 508 'File %r not available at master' % source) 509 # TODO: once BuildStep.start() gets rewritten to use 510 # maybeDeferred, just re-raise the exception here. 511 reactor.callLater(0, BuildStep.finished, self, FAILURE) 512 return 513 fileReader = _FileReader(fp) 514 515 # default arguments 516 args = { 517 'slavedest': slavedest, 518 'maxsize': self.maxsize, 519 'reader': fileReader, 520 'blocksize': self.blocksize, 521 'workdir': self._getWorkdir(), 522 'mode': self.mode, 523 } 524 525 self.cmd = StatusRemoteCommand('downloadFile', args) 526 d = self.runCommand(self.cmd) 527 d.addCallback(self.finished).addErrback(self.failed)

528

529 -class StringDownload(_TransferBuildStep):

530 """ 531 Download the first 'maxsize' bytes of a string, from the buildmaster to the 532 buildslave. Set the mode of the file 533 534 Arguments:: 535 536 ['s'] string to transfer 537 ['slavedest'] filename of destination file at slave 538 ['workdir'] string with slave working directory relative to builder 539 base dir, default 'build' 540 ['maxsize'] maximum size of the file, default None (=unlimited) 541 ['blocksize'] maximum size of each block being transfered 542 ['mode'] use this to set the access permissions of the resulting 543 buildslave-side file. This is traditionally an octal 544 integer, like 0644 to be world-readable (but not 545 world-writable), or 0600 to only be readable by 546 the buildslave account, or 0755 to be world-executable. 547 The default (=None) is to leave it up to the umask of 548 the buildslave process. 549 """ 550 name = 'string_download' 551 552 renderables = [ 'slavedest', 's' ] 553

554 - def __init__(self, s, slavedest, 555 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 556 **buildstep_kwargs):

557 BuildStep.__init__(self, **buildstep_kwargs) 558 self.addFactoryArguments(s=s, 559 slavedest=slavedest, 560 workdir=workdir, 561 maxsize=maxsize, 562 blocksize=blocksize, 563 mode=mode, 564 ) 565 566 self.s = s 567 self.slavedest = slavedest 568 self.workdir = workdir 569 self.maxsize = maxsize 570 self.blocksize = blocksize 571 assert isinstance(mode, (int, type(None))) 572 self.mode = mode

573

574 - def start(self):

575 version = self.slaveVersion("downloadFile") 576 if not version: 577 m = "slave is too old, does not know about downloadFile" 578 raise BuildSlaveTooOldError(m) 579 580 # we are currently in the buildmaster's basedir, so any non-absolute 581 # paths will be interpreted relative to that 582 slavedest = self.slavedest 583 log.msg("StringDownload started, from master to slave %r" % slavedest) 584 585 self.step_status.setText(['downloading', "to", 586 os.path.basename(slavedest)]) 587 588 # setup structures for reading the file 589 fp = StringIO(self.s) 590 fileReader = _FileReader(fp) 591 592 # default arguments 593 args = { 594 'slavedest': slavedest, 595 'maxsize': self.maxsize, 596 'reader': fileReader, 597 'blocksize': self.blocksize, 598 'workdir': self._getWorkdir(), 599 'mode': self.mode, 600 } 601 602 self.cmd = StatusRemoteCommand('downloadFile', args) 603 d = self.runCommand(self.cmd) 604 d.addCallback(self.finished).addErrback(self.failed)

605

606 -class JSONStringDownload(StringDownload):

607 """ 608 Encode object o as a json string and save it on the buildslave 609 610 Arguments:: 611 612 ['o'] object to encode and transfer 613 """ 614 name = "json_download"

615 - def __init__(self, o, slavedest, **buildstep_kwargs):

616 if 's' in buildstep_kwargs: 617 del buildstep_kwargs['s'] 618 s = json.dumps(o) 619 StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs) 620 self.addFactoryArguments(o=o)

621

622 -class JSONPropertiesDownload(StringDownload):

623 """ 624 Download the current build properties as a json string and save it on the 625 buildslave 626 """ 627 name = "json_properties_download"

628 - def __init__(self, slavedest, **buildstep_kwargs):

629 self.super_class = StringDownload 630 if 's' in buildstep_kwargs: 631 del buildstep_kwargs['s'] 632 StringDownload.__init__(self, s=None, slavedest=slavedest, **buildstep_kwargs)

633

634 - def start(self):

635 properties = self.build.getProperties() 636 props = {} 637 for key, value, source in properties.asList(): 638 props[key] = value 639 640 self.s = json.dumps(dict( 641 properties=props, 642 sourcestamp=self.build.getSourceStamp().asDict(), 643 ), 644 ) 645 return self.super_class.start(self)

646

Source Code for Module buildbot.steps.transfer