buildbot.steps.transfer

1 # -*- test-case-name: buildbot.test.test_transfer -*- 2 3 import os.path, tarfile, tempfile 4 try: 5 from cStringIO import StringIO 6 except ImportError: 7 from StringIO import StringIO 8 from twisted.internet import reactor 9 from twisted.spread import pb 10 from twisted.python import log 11 from buildbot.process.buildstep import RemoteCommand, BuildStep 12 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 13 from buildbot.interfaces import BuildSlaveTooOldError 14 from buildbot.util import json 15 16

17 -class _FileWriter(pb.Referenceable):

18 """ 19 Helper class that acts as a file-object with write access 20 """ 21

22 - def __init__(self, destfile, maxsize, mode):

23 # Create missing directories. 24 destfile = os.path.abspath(destfile) 25 dirname = os.path.dirname(destfile) 26 if not os.path.exists(dirname): 27 os.makedirs(dirname) 28 29 self.destfile = destfile 30 fd, self.tmpname = tempfile.mkstemp(dir=dirname) 31 self.fp = os.fdopen(fd, 'wb') 32 if mode is not None: 33 os.chmod(destfile, mode) 34 self.remaining = maxsize

35

36 - def remote_write(self, data):

37 """ 38 Called from remote slave to write L{data} to L{fp} within boundaries 39 of L{maxsize} 40 41 @type data: C{string} 42 @param data: String of data to write 43 """ 44 if self.remaining is not None: 45 if len(data) > self.remaining: 46 data = data[:self.remaining] 47 self.fp.write(data) 48 self.remaining = self.remaining - len(data) 49 else: 50 self.fp.write(data)

51

52 - def remote_close(self):

53 """ 54 Called by remote slave to state that no more data will be transfered 55 """ 56 self.fp.close() 57 self.fp = None 58 os.rename(self.tmpname, self.destfile) 59 self.tmpname = None

60

61 - def __del__(self):

62 # unclean shutdown, the file is probably truncated, so delete it 63 # altogether rather than deliver a corrupted file 64 fp = getattr(self, "fp", None) 65 if fp: 66 fp.close() 67 os.unlink(self.destfile) 68 if self.tmpname and os.path.exists(self.tmpname): 69 os.unlink(self.tmpname)

70 71

72 -def _extractall(self, path=".", members=None):

73 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 74 75 import copy 76 77 directories = [] 78 79 if members is None: 80 members = self 81 82 for tarinfo in members: 83 if tarinfo.isdir(): 84 # Extract directories with a safe mode. 85 directories.append(tarinfo) 86 tarinfo = copy.copy(tarinfo) 87 tarinfo.mode = 0700 88 self.extract(tarinfo, path) 89 90 # Reverse sort directories. 91 directories.sort(lambda a, b: cmp(a.name, b.name)) 92 directories.reverse() 93 94 # Set correct owner, mtime and filemode on directories. 95 for tarinfo in directories: 96 dirpath = os.path.join(path, tarinfo.name) 97 try: 98 self.chown(tarinfo, dirpath) 99 self.utime(tarinfo, dirpath) 100 self.chmod(tarinfo, dirpath) 101 except tarfile.ExtractError, e: 102 if self.errorlevel > 1: 103 raise 104 else: 105 self._dbg(1, "tarfile: %s" % e)

106

107 -class _DirectoryWriter(_FileWriter):

108 """ 109 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 110 step to unpack the archive, once the transfer has completed. 111 """ 112

113 - def __init__(self, destroot, maxsize, compress, mode):

114 self.destroot = destroot 115 self.compress = compress 116 117 self.fd, self.tarname = tempfile.mkstemp() 118 os.close(self.fd) 119 120 _FileWriter.__init__(self, self.tarname, maxsize, mode)

121

122 - def remote_unpack(self):

123 """ 124 Called by remote slave to state that no more data will be transfered 125 """ 126 # Make sure remote_close is called, otherwise atomic rename wont happen 127 self.remote_close() 128 129 # Map configured compression to a TarFile setting 130 if self.compress == 'bz2': 131 mode='r|bz2' 132 elif self.compress == 'gz': 133 mode='r|gz' 134 else: 135 mode = 'r' 136 137 # Support old python 138 if not hasattr(tarfile.TarFile, 'extractall'): 139 tarfile.TarFile.extractall = _extractall 140 141 # Unpack archive and clean up after self 142 archive = tarfile.open(name=self.tarname, mode=mode) 143 archive.extractall(path=self.destroot) 144 archive.close() 145 os.remove(self.tarname)

146 147

148 -class StatusRemoteCommand(RemoteCommand):

149 - def __init__(self, remote_command, args):

150 RemoteCommand.__init__(self, remote_command, args) 151 152 self.rc = None 153 self.stderr = ''

154

155 - def remoteUpdate(self, update):

156 #log.msg('StatusRemoteCommand: update=%r' % update) 157 if 'rc' in update: 158 self.rc = update['rc'] 159 if 'stderr' in update: 160 self.stderr = self.stderr + update['stderr'] + '\n'

161

162 -class _TransferBuildStep(BuildStep):

163 """ 164 Base class for FileUpload and FileDownload to factor out common 165 functionality. 166 """ 167 DEFAULT_WORKDIR = "build" # is this redundant? 168

169 - def setDefaultWorkdir(self, workdir):

170 if self.workdir is None: 171 self.workdir = workdir

172

173 - def _getWorkdir(self):

174 properties = self.build.getProperties() 175 if self.workdir is None: 176 workdir = self.DEFAULT_WORKDIR 177 else: 178 workdir = self.workdir 179 return properties.render(workdir)

180

181 - def finished(self, result):

182 # Subclasses may choose to skip a transfer. In those cases, self.cmd 183 # will be None, and we should just let BuildStep.finished() handle 184 # the rest 185 if result == SKIPPED: 186 return BuildStep.finished(self, SKIPPED) 187 if self.cmd.stderr != '': 188 self.addCompleteLog('stderr', self.cmd.stderr) 189 190 if self.cmd.rc is None or self.cmd.rc == 0: 191 return BuildStep.finished(self, SUCCESS) 192 return BuildStep.finished(self, FAILURE)

193 194

195 -class FileUpload(_TransferBuildStep):

196 """ 197 Build step to transfer a file from the slave to the master. 198 199 arguments: 200 201 - ['slavesrc'] filename of source file at slave, relative to workdir 202 - ['masterdest'] filename of destination file at master 203 - ['workdir'] string with slave working directory relative to builder 204 base dir, default 'build' 205 - ['maxsize'] maximum size of the file, default None (=unlimited) 206 - ['blocksize'] maximum size of each block being transfered 207 - ['mode'] file access mode for the resulting master-side file. 208 The default (=None) is to leave it up to the umask of 209 the buildmaster process. 210 211 """ 212 213 name = 'upload' 214

215 - def __init__(self, slavesrc, masterdest, 216 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 217 **buildstep_kwargs):

218 BuildStep.__init__(self, **buildstep_kwargs) 219 self.addFactoryArguments(slavesrc=slavesrc, 220 masterdest=masterdest, 221 workdir=workdir, 222 maxsize=maxsize, 223 blocksize=blocksize, 224 mode=mode, 225 ) 226 227 self.slavesrc = slavesrc 228 self.masterdest = masterdest 229 self.workdir = workdir 230 self.maxsize = maxsize 231 self.blocksize = blocksize 232 assert isinstance(mode, (int, type(None))) 233 self.mode = mode

234

235 - def start(self):

236 version = self.slaveVersion("uploadFile") 237 properties = self.build.getProperties() 238 239 if not version: 240 m = "slave is too old, does not know about uploadFile" 241 raise BuildSlaveTooOldError(m) 242 243 source = properties.render(self.slavesrc) 244 masterdest = properties.render(self.masterdest) 245 # we rely upon the fact that the buildmaster runs chdir'ed into its 246 # basedir to make sure that relative paths in masterdest are expanded 247 # properly. TODO: maybe pass the master's basedir all the way down 248 # into the BuildStep so we can do this better. 249 masterdest = os.path.expanduser(masterdest) 250 log.msg("FileUpload started, from slave %r to master %r" 251 % (source, masterdest)) 252 253 self.step_status.setText(['uploading', os.path.basename(source)]) 254 255 # we use maxsize to limit the amount of data on both sides 256 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 257 258 # default arguments 259 args = { 260 'slavesrc': source, 261 'workdir': self._getWorkdir(), 262 'writer': fileWriter, 263 'maxsize': self.maxsize, 264 'blocksize': self.blocksize, 265 } 266 267 self.cmd = StatusRemoteCommand('uploadFile', args) 268 d = self.runCommand(self.cmd) 269 d.addCallback(self.finished).addErrback(self.failed)

270 271

272 -class DirectoryUpload(BuildStep):

273 """ 274 Build step to transfer a directory from the slave to the master. 275 276 arguments: 277 278 - ['slavesrc'] name of source directory at slave, relative to workdir 279 - ['masterdest'] name of destination directory at master 280 - ['workdir'] string with slave working directory relative to builder 281 base dir, default 'build' 282 - ['maxsize'] maximum size of the compressed tarfile containing the 283 whole directory 284 - ['blocksize'] maximum size of each block being transfered 285 - ['compress'] compression type to use: one of [None, 'gz', 'bz2'] 286 287 """ 288 289 name = 'upload' 290

291 - def __init__(self, slavesrc, masterdest, 292 workdir="build", maxsize=None, blocksize=16*1024, 293 compress=None, **buildstep_kwargs):

294 BuildStep.__init__(self, **buildstep_kwargs) 295 self.addFactoryArguments(slavesrc=slavesrc, 296 masterdest=masterdest, 297 workdir=workdir, 298 maxsize=maxsize, 299 blocksize=blocksize, 300 compress=compress, 301 ) 302 303 self.slavesrc = slavesrc 304 self.masterdest = masterdest 305 self.workdir = workdir 306 self.maxsize = maxsize 307 self.blocksize = blocksize 308 assert compress in (None, 'gz', 'bz2') 309 self.compress = compress

310

311 - def start(self):

312 version = self.slaveVersion("uploadDirectory") 313 properties = self.build.getProperties() 314 315 if not version: 316 m = "slave is too old, does not know about uploadDirectory" 317 raise BuildSlaveTooOldError(m) 318 319 source = properties.render(self.slavesrc) 320 masterdest = properties.render(self.masterdest) 321 # we rely upon the fact that the buildmaster runs chdir'ed into its 322 # basedir to make sure that relative paths in masterdest are expanded 323 # properly. TODO: maybe pass the master's basedir all the way down 324 # into the BuildStep so we can do this better. 325 masterdest = os.path.expanduser(masterdest) 326 log.msg("DirectoryUpload started, from slave %r to master %r" 327 % (source, masterdest)) 328 329 self.step_status.setText(['uploading', os.path.basename(source)]) 330 331 # we use maxsize to limit the amount of data on both sides 332 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600) 333 334 # default arguments 335 args = { 336 'slavesrc': source, 337 'workdir': self.workdir, 338 'writer': dirWriter, 339 'maxsize': self.maxsize, 340 'blocksize': self.blocksize, 341 'compress': self.compress 342 } 343 344 self.cmd = StatusRemoteCommand('uploadDirectory', args) 345 d = self.runCommand(self.cmd) 346 d.addCallback(self.finished).addErrback(self.failed)

347

348 - def finished(self, result):

349 # Subclasses may choose to skip a transfer. In those cases, self.cmd 350 # will be None, and we should just let BuildStep.finished() handle 351 # the rest 352 if result == SKIPPED: 353 return BuildStep.finished(self, SKIPPED) 354 if self.cmd.stderr != '': 355 self.addCompleteLog('stderr', self.cmd.stderr) 356 357 if self.cmd.rc is None or self.cmd.rc == 0: 358 return BuildStep.finished(self, SUCCESS) 359 return BuildStep.finished(self, FAILURE)

360 361 362 363

364 -class _FileReader(pb.Referenceable):

365 """ 366 Helper class that acts as a file-object with read access 367 """ 368

369 - def __init__(self, fp):

370 self.fp = fp

371

372 - def remote_read(self, maxlength):

373 """ 374 Called from remote slave to read at most L{maxlength} bytes of data 375 376 @type maxlength: C{integer} 377 @param maxlength: Maximum number of data bytes that can be returned 378 379 @return: Data read from L{fp} 380 @rtype: C{string} of bytes read from file 381 """ 382 if self.fp is None: 383 return '' 384 385 data = self.fp.read(maxlength) 386 return data

387

388 - def remote_close(self):

389 """ 390 Called by remote slave to state that no more data will be transfered 391 """ 392 if self.fp is not None: 393 self.fp.close() 394 self.fp = None

395 396

397 -class FileDownload(_TransferBuildStep):

398 """ 399 Download the first 'maxsize' bytes of a file, from the buildmaster to the 400 buildslave. Set the mode of the file 401 402 Arguments:: 403 404 ['mastersrc'] filename of source file at master 405 ['slavedest'] filename of destination file at slave 406 ['workdir'] string with slave working directory relative to builder 407 base dir, default 'build' 408 ['maxsize'] maximum size of the file, default None (=unlimited) 409 ['blocksize'] maximum size of each block being transfered 410 ['mode'] use this to set the access permissions of the resulting 411 buildslave-side file. This is traditionally an octal 412 integer, like 0644 to be world-readable (but not 413 world-writable), or 0600 to only be readable by 414 the buildslave account, or 0755 to be world-executable. 415 The default (=None) is to leave it up to the umask of 416 the buildslave process. 417 418 """ 419 name = 'download' 420

421 - def __init__(self, mastersrc, slavedest, 422 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 423 **buildstep_kwargs):

424 BuildStep.__init__(self, **buildstep_kwargs) 425 self.addFactoryArguments(mastersrc=mastersrc, 426 slavedest=slavedest, 427 workdir=workdir, 428 maxsize=maxsize, 429 blocksize=blocksize, 430 mode=mode, 431 ) 432 433 self.mastersrc = mastersrc 434 self.slavedest = slavedest 435 self.workdir = workdir 436 self.maxsize = maxsize 437 self.blocksize = blocksize 438 assert isinstance(mode, (int, type(None))) 439 self.mode = mode

440

441 - def start(self):

442 properties = self.build.getProperties() 443 444 version = self.slaveVersion("downloadFile") 445 if not version: 446 m = "slave is too old, does not know about downloadFile" 447 raise BuildSlaveTooOldError(m) 448 449 # we are currently in the buildmaster's basedir, so any non-absolute 450 # paths will be interpreted relative to that 451 source = os.path.expanduser(properties.render(self.mastersrc)) 452 slavedest = properties.render(self.slavedest) 453 log.msg("FileDownload started, from master %r to slave %r" % 454 (source, slavedest)) 455 456 self.step_status.setText(['downloading', "to", 457 os.path.basename(slavedest)]) 458 459 # setup structures for reading the file 460 try: 461 fp = open(source, 'rb') 462 except IOError: 463 # if file does not exist, bail out with an error 464 self.addCompleteLog('stderr', 465 'File %r not available at master' % source) 466 # TODO: once BuildStep.start() gets rewritten to use 467 # maybeDeferred, just re-raise the exception here. 468 reactor.callLater(0, BuildStep.finished, self, FAILURE) 469 return 470 fileReader = _FileReader(fp) 471 472 # default arguments 473 args = { 474 'slavedest': slavedest, 475 'maxsize': self.maxsize, 476 'reader': fileReader, 477 'blocksize': self.blocksize, 478 'workdir': self._getWorkdir(), 479 'mode': self.mode, 480 } 481 482 self.cmd = StatusRemoteCommand('downloadFile', args) 483 d = self.runCommand(self.cmd) 484 d.addCallback(self.finished).addErrback(self.failed)

485

486 -class StringDownload(_TransferBuildStep):

487 """ 488 Download the first 'maxsize' bytes of a string, from the buildmaster to the 489 buildslave. Set the mode of the file 490 491 Arguments:: 492 493 ['s'] string to transfer 494 ['slavedest'] filename of destination file at slave 495 ['workdir'] string with slave working directory relative to builder 496 base dir, default 'build' 497 ['maxsize'] maximum size of the file, default None (=unlimited) 498 ['blocksize'] maximum size of each block being transfered 499 ['mode'] use this to set the access permissions of the resulting 500 buildslave-side file. This is traditionally an octal 501 integer, like 0644 to be world-readable (but not 502 world-writable), or 0600 to only be readable by 503 the buildslave account, or 0755 to be world-executable. 504 The default (=None) is to leave it up to the umask of 505 the buildslave process. 506 """ 507 name = 'string_download' 508

509 - def __init__(self, s, slavedest, 510 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 511 **buildstep_kwargs):

512 BuildStep.__init__(self, **buildstep_kwargs) 513 self.addFactoryArguments(s=s, 514 slavedest=slavedest, 515 workdir=workdir, 516 maxsize=maxsize, 517 blocksize=blocksize, 518 mode=mode, 519 ) 520 521 self.s = s 522 self.slavedest = slavedest 523 self.workdir = workdir 524 self.maxsize = maxsize 525 self.blocksize = blocksize 526 assert isinstance(mode, (int, type(None))) 527 self.mode = mode

528

529 - def start(self):

530 properties = self.build.getProperties() 531 532 version = self.slaveVersion("downloadFile") 533 if not version: 534 m = "slave is too old, does not know about downloadFile" 535 raise BuildSlaveTooOldError(m) 536 537 # we are currently in the buildmaster's basedir, so any non-absolute 538 # paths will be interpreted relative to that 539 slavedest = properties.render(self.slavedest) 540 log.msg("StringDownload started, from master to slave %r" % slavedest) 541 542 self.step_status.setText(['downloading', "to", 543 os.path.basename(slavedest)]) 544 545 # setup structures for reading the file 546 fp = StringIO(self.s) 547 fileReader = _FileReader(fp) 548 549 # default arguments 550 args = { 551 'slavedest': slavedest, 552 'maxsize': self.maxsize, 553 'reader': fileReader, 554 'blocksize': self.blocksize, 555 'workdir': self._getWorkdir(), 556 'mode': self.mode, 557 } 558 559 self.cmd = StatusRemoteCommand('downloadFile', args) 560 d = self.runCommand(self.cmd) 561 d.addCallback(self.finished).addErrback(self.failed)

562

563 -class JSONStringDownload(StringDownload):

564 """ 565 Encode object o as a json string and save it on the buildslave 566 567 Arguments:: 568 569 ['o'] object to encode and transfer 570 """ 571 name = "json_download"

572 - def __init__(self, o, slavedest, **buildstep_kwargs):

573 if 's' in buildstep_kwargs: 574 del buildstep_kwargs['s'] 575 s = json.dumps(o) 576 StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs) 577 self.addFactoryArguments(o=o)

578

579 -class JSONPropertiesDownload(StringDownload):

580 """ 581 Download the current build properties as a json string and save it on the 582 buildslave 583 """ 584 name = "json_properties_download"

585 - def __init__(self, slavedest, **buildstep_kwargs):

586 self.super_class = StringDownload 587 if 's' in buildstep_kwargs: 588 del buildstep_kwargs['s'] 589 StringDownload.__init__(self, s=None, slavedest=slavedest, **buildstep_kwargs)

590

591 - def start(self):

592 properties = self.build.getProperties() 593 props = {} 594 for key, value, source in properties.asList(): 595 props[key] = value 596 597 self.s = json.dumps(dict( 598 properties=props, 599 sourcestamp=self.build.getSourceStamp().asDict(), 600 ), 601 ) 602 return self.super_class.start(self)

603

Source Code for Module buildbot.steps.transfer