buildbot.steps.transfer

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import os.path, tarfile, tempfile 18 try: 19 from cStringIO import StringIO 20 assert StringIO 21 except ImportError: 22 from StringIO import StringIO 23 from twisted.internet import reactor 24 from twisted.spread import pb 25 from twisted.python import log 26 from buildbot.process.buildstep import RemoteCommand, BuildStep 27 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 28 from buildbot.interfaces import BuildSlaveTooOldError 29 from buildbot.util import json 30 31

32 -class _FileWriter(pb.Referenceable):

33 """ 34 Helper class that acts as a file-object with write access 35 """ 36

37 - def __init__(self, destfile, maxsize, mode):

38 # Create missing directories. 39 destfile = os.path.abspath(destfile) 40 dirname = os.path.dirname(destfile) 41 if not os.path.exists(dirname): 42 os.makedirs(dirname) 43 44 self.destfile = destfile 45 self.mode = mode 46 fd, self.tmpname = tempfile.mkstemp(dir=dirname) 47 self.fp = os.fdopen(fd, 'wb') 48 self.remaining = maxsize

49

50 - def remote_write(self, data):

51 """ 52 Called from remote slave to write L{data} to L{fp} within boundaries 53 of L{maxsize} 54 55 @type data: C{string} 56 @param data: String of data to write 57 """ 58 if self.remaining is not None: 59 if len(data) > self.remaining: 60 data = data[:self.remaining] 61 self.fp.write(data) 62 self.remaining = self.remaining - len(data) 63 else: 64 self.fp.write(data)

65

66 - def remote_close(self):

67 """ 68 Called by remote slave to state that no more data will be transfered 69 """ 70 self.fp.close() 71 self.fp = None 72 # on windows, os.rename does not automatically unlink, so do it manually 73 if os.path.exists(self.destfile): 74 os.unlink(self.destfile) 75 os.rename(self.tmpname, self.destfile) 76 self.tmpname = None 77 if self.mode is not None: 78 os.chmod(self.destfile, self.mode)

79

80 - def __del__(self):

81 # unclean shutdown, the file is probably truncated, so delete it 82 # altogether rather than deliver a corrupted file 83 fp = getattr(self, "fp", None) 84 if fp: 85 fp.close() 86 os.unlink(self.destfile) 87 if self.tmpname and os.path.exists(self.tmpname): 88 os.unlink(self.tmpname)

89 90

91 -def _extractall(self, path=".", members=None):

92 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 93 94 import copy 95 96 directories = [] 97 98 if members is None: 99 members = self 100 101 for tarinfo in members: 102 if tarinfo.isdir(): 103 # Extract directories with a safe mode. 104 directories.append(tarinfo) 105 tarinfo = copy.copy(tarinfo) 106 tarinfo.mode = 0700 107 self.extract(tarinfo, path) 108 109 # Reverse sort directories. 110 directories.sort(lambda a, b: cmp(a.name, b.name)) 111 directories.reverse() 112 113 # Set correct owner, mtime and filemode on directories. 114 for tarinfo in directories: 115 dirpath = os.path.join(path, tarinfo.name) 116 try: 117 self.chown(tarinfo, dirpath) 118 self.utime(tarinfo, dirpath) 119 self.chmod(tarinfo, dirpath) 120 except tarfile.ExtractError, e: 121 if self.errorlevel > 1: 122 raise 123 else: 124 self._dbg(1, "tarfile: %s" % e)

125

126 -class _DirectoryWriter(_FileWriter):

127 """ 128 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 129 step to unpack the archive, once the transfer has completed. 130 """ 131

132 - def __init__(self, destroot, maxsize, compress, mode):

133 self.destroot = destroot 134 self.compress = compress 135 136 self.fd, self.tarname = tempfile.mkstemp() 137 os.close(self.fd) 138 139 _FileWriter.__init__(self, self.tarname, maxsize, mode)

140

141 - def remote_unpack(self):

142 """ 143 Called by remote slave to state that no more data will be transfered 144 """ 145 # Make sure remote_close is called, otherwise atomic rename wont happen 146 self.remote_close() 147 148 # Map configured compression to a TarFile setting 149 if self.compress == 'bz2': 150 mode='r|bz2' 151 elif self.compress == 'gz': 152 mode='r|gz' 153 else: 154 mode = 'r' 155 156 # Support old python 157 if not hasattr(tarfile.TarFile, 'extractall'): 158 tarfile.TarFile.extractall = _extractall 159 160 # Unpack archive and clean up after self 161 archive = tarfile.open(name=self.tarname, mode=mode) 162 archive.extractall(path=self.destroot) 163 archive.close() 164 os.remove(self.tarname)

165 166

167 -class StatusRemoteCommand(RemoteCommand):

168 - def __init__(self, remote_command, args):

169 RemoteCommand.__init__(self, remote_command, args) 170 171 self.rc = None 172 self.stderr = ''

173

174 - def remoteUpdate(self, update):

175 #log.msg('StatusRemoteCommand: update=%r' % update) 176 if 'rc' in update: 177 self.rc = update['rc'] 178 if 'stderr' in update: 179 self.stderr = self.stderr + update['stderr'] + '\n'

180

181 -class _TransferBuildStep(BuildStep):

182 """ 183 Base class for FileUpload and FileDownload to factor out common 184 functionality. 185 """ 186 DEFAULT_WORKDIR = "build" # is this redundant? 187

188 - def setDefaultWorkdir(self, workdir):

189 if self.workdir is None: 190 self.workdir = workdir

191

192 - def _getWorkdir(self):

193 properties = self.build.getProperties() 194 if self.workdir is None: 195 workdir = self.DEFAULT_WORKDIR 196 else: 197 workdir = self.workdir 198 return properties.render(workdir)

199

200 - def finished(self, result):

201 # Subclasses may choose to skip a transfer. In those cases, self.cmd 202 # will be None, and we should just let BuildStep.finished() handle 203 # the rest 204 if result == SKIPPED: 205 return BuildStep.finished(self, SKIPPED) 206 if self.cmd.stderr != '': 207 self.addCompleteLog('stderr', self.cmd.stderr) 208 209 if self.cmd.rc is None or self.cmd.rc == 0: 210 return BuildStep.finished(self, SUCCESS) 211 return BuildStep.finished(self, FAILURE)

212 213

214 -class FileUpload(_TransferBuildStep):

215 """ 216 Build step to transfer a file from the slave to the master. 217 218 arguments: 219 220 - ['slavesrc'] filename of source file at slave, relative to workdir 221 - ['masterdest'] filename of destination file at master 222 - ['workdir'] string with slave working directory relative to builder 223 base dir, default 'build' 224 - ['maxsize'] maximum size of the file, default None (=unlimited) 225 - ['blocksize'] maximum size of each block being transfered 226 - ['mode'] file access mode for the resulting master-side file. 227 The default (=None) is to leave it up to the umask of 228 the buildmaster process. 229 230 """ 231 232 name = 'upload' 233

234 - def __init__(self, slavesrc, masterdest, 235 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 236 **buildstep_kwargs):

237 BuildStep.__init__(self, **buildstep_kwargs) 238 self.addFactoryArguments(slavesrc=slavesrc, 239 masterdest=masterdest, 240 workdir=workdir, 241 maxsize=maxsize, 242 blocksize=blocksize, 243 mode=mode, 244 ) 245 246 self.slavesrc = slavesrc 247 self.masterdest = masterdest 248 self.workdir = workdir 249 self.maxsize = maxsize 250 self.blocksize = blocksize 251 assert isinstance(mode, (int, type(None))) 252 self.mode = mode

253

254 - def start(self):

255 version = self.slaveVersion("uploadFile") 256 properties = self.build.getProperties() 257 258 if not version: 259 m = "slave is too old, does not know about uploadFile" 260 raise BuildSlaveTooOldError(m) 261 262 source = properties.render(self.slavesrc) 263 masterdest = properties.render(self.masterdest) 264 # we rely upon the fact that the buildmaster runs chdir'ed into its 265 # basedir to make sure that relative paths in masterdest are expanded 266 # properly. TODO: maybe pass the master's basedir all the way down 267 # into the BuildStep so we can do this better. 268 masterdest = os.path.expanduser(masterdest) 269 log.msg("FileUpload started, from slave %r to master %r" 270 % (source, masterdest)) 271 272 self.step_status.setText(['uploading', os.path.basename(source)]) 273 274 # we use maxsize to limit the amount of data on both sides 275 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 276 277 # default arguments 278 args = { 279 'slavesrc': source, 280 'workdir': self._getWorkdir(), 281 'writer': fileWriter, 282 'maxsize': self.maxsize, 283 'blocksize': self.blocksize, 284 } 285 286 self.cmd = StatusRemoteCommand('uploadFile', args) 287 d = self.runCommand(self.cmd) 288 d.addCallback(self.finished).addErrback(self.failed)

289 290

291 -class DirectoryUpload(BuildStep):

292 """ 293 Build step to transfer a directory from the slave to the master. 294 295 arguments: 296 297 - ['slavesrc'] name of source directory at slave, relative to workdir 298 - ['masterdest'] name of destination directory at master 299 - ['workdir'] string with slave working directory relative to builder 300 base dir, default 'build' 301 - ['maxsize'] maximum size of the compressed tarfile containing the 302 whole directory 303 - ['blocksize'] maximum size of each block being transfered 304 - ['compress'] compression type to use: one of [None, 'gz', 'bz2'] 305 306 """ 307 308 name = 'upload' 309

310 - def __init__(self, slavesrc, masterdest, 311 workdir="build", maxsize=None, blocksize=16*1024, 312 compress=None, **buildstep_kwargs):

313 BuildStep.__init__(self, **buildstep_kwargs) 314 self.addFactoryArguments(slavesrc=slavesrc, 315 masterdest=masterdest, 316 workdir=workdir, 317 maxsize=maxsize, 318 blocksize=blocksize, 319 compress=compress, 320 ) 321 322 self.slavesrc = slavesrc 323 self.masterdest = masterdest 324 self.workdir = workdir 325 self.maxsize = maxsize 326 self.blocksize = blocksize 327 assert compress in (None, 'gz', 'bz2') 328 self.compress = compress

329

330 - def start(self):

331 version = self.slaveVersion("uploadDirectory") 332 properties = self.build.getProperties() 333 334 if not version: 335 m = "slave is too old, does not know about uploadDirectory" 336 raise BuildSlaveTooOldError(m) 337 338 source = properties.render(self.slavesrc) 339 masterdest = properties.render(self.masterdest) 340 # we rely upon the fact that the buildmaster runs chdir'ed into its 341 # basedir to make sure that relative paths in masterdest are expanded 342 # properly. TODO: maybe pass the master's basedir all the way down 343 # into the BuildStep so we can do this better. 344 masterdest = os.path.expanduser(masterdest) 345 log.msg("DirectoryUpload started, from slave %r to master %r" 346 % (source, masterdest)) 347 348 self.step_status.setText(['uploading', os.path.basename(source)]) 349 350 # we use maxsize to limit the amount of data on both sides 351 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600) 352 353 # default arguments 354 args = { 355 'slavesrc': source, 356 'workdir': self.workdir, 357 'writer': dirWriter, 358 'maxsize': self.maxsize, 359 'blocksize': self.blocksize, 360 'compress': self.compress 361 } 362 363 self.cmd = StatusRemoteCommand('uploadDirectory', args) 364 d = self.runCommand(self.cmd) 365 d.addCallback(self.finished).addErrback(self.failed)

366

367 - def finished(self, result):

368 # Subclasses may choose to skip a transfer. In those cases, self.cmd 369 # will be None, and we should just let BuildStep.finished() handle 370 # the rest 371 if result == SKIPPED: 372 return BuildStep.finished(self, SKIPPED) 373 if self.cmd.stderr != '': 374 self.addCompleteLog('stderr', self.cmd.stderr) 375 376 if self.cmd.rc is None or self.cmd.rc == 0: 377 return BuildStep.finished(self, SUCCESS) 378 return BuildStep.finished(self, FAILURE)

379 380 381 382

383 -class _FileReader(pb.Referenceable):

384 """ 385 Helper class that acts as a file-object with read access 386 """ 387

388 - def __init__(self, fp):

389 self.fp = fp

390

391 - def remote_read(self, maxlength):

392 """ 393 Called from remote slave to read at most L{maxlength} bytes of data 394 395 @type maxlength: C{integer} 396 @param maxlength: Maximum number of data bytes that can be returned 397 398 @return: Data read from L{fp} 399 @rtype: C{string} of bytes read from file 400 """ 401 if self.fp is None: 402 return '' 403 404 data = self.fp.read(maxlength) 405 return data

406

407 - def remote_close(self):

408 """ 409 Called by remote slave to state that no more data will be transfered 410 """ 411 if self.fp is not None: 412 self.fp.close() 413 self.fp = None

414 415

416 -class FileDownload(_TransferBuildStep):

417 """ 418 Download the first 'maxsize' bytes of a file, from the buildmaster to the 419 buildslave. Set the mode of the file 420 421 Arguments:: 422 423 ['mastersrc'] filename of source file at master 424 ['slavedest'] filename of destination file at slave 425 ['workdir'] string with slave working directory relative to builder 426 base dir, default 'build' 427 ['maxsize'] maximum size of the file, default None (=unlimited) 428 ['blocksize'] maximum size of each block being transfered 429 ['mode'] use this to set the access permissions of the resulting 430 buildslave-side file. This is traditionally an octal 431 integer, like 0644 to be world-readable (but not 432 world-writable), or 0600 to only be readable by 433 the buildslave account, or 0755 to be world-executable. 434 The default (=None) is to leave it up to the umask of 435 the buildslave process. 436 437 """ 438 name = 'download' 439

440 - def __init__(self, mastersrc, slavedest, 441 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 442 **buildstep_kwargs):

443 BuildStep.__init__(self, **buildstep_kwargs) 444 self.addFactoryArguments(mastersrc=mastersrc, 445 slavedest=slavedest, 446 workdir=workdir, 447 maxsize=maxsize, 448 blocksize=blocksize, 449 mode=mode, 450 ) 451 452 self.mastersrc = mastersrc 453 self.slavedest = slavedest 454 self.workdir = workdir 455 self.maxsize = maxsize 456 self.blocksize = blocksize 457 assert isinstance(mode, (int, type(None))) 458 self.mode = mode

459

460 - def start(self):

461 properties = self.build.getProperties() 462 463 version = self.slaveVersion("downloadFile") 464 if not version: 465 m = "slave is too old, does not know about downloadFile" 466 raise BuildSlaveTooOldError(m) 467 468 # we are currently in the buildmaster's basedir, so any non-absolute 469 # paths will be interpreted relative to that 470 source = os.path.expanduser(properties.render(self.mastersrc)) 471 slavedest = properties.render(self.slavedest) 472 log.msg("FileDownload started, from master %r to slave %r" % 473 (source, slavedest)) 474 475 self.step_status.setText(['downloading', "to", 476 os.path.basename(slavedest)]) 477 478 # setup structures for reading the file 479 try: 480 fp = open(source, 'rb') 481 except IOError: 482 # if file does not exist, bail out with an error 483 self.addCompleteLog('stderr', 484 'File %r not available at master' % source) 485 # TODO: once BuildStep.start() gets rewritten to use 486 # maybeDeferred, just re-raise the exception here. 487 reactor.callLater(0, BuildStep.finished, self, FAILURE) 488 return 489 fileReader = _FileReader(fp) 490 491 # default arguments 492 args = { 493 'slavedest': slavedest, 494 'maxsize': self.maxsize, 495 'reader': fileReader, 496 'blocksize': self.blocksize, 497 'workdir': self._getWorkdir(), 498 'mode': self.mode, 499 } 500 501 self.cmd = StatusRemoteCommand('downloadFile', args) 502 d = self.runCommand(self.cmd) 503 d.addCallback(self.finished).addErrback(self.failed)

504

505 -class StringDownload(_TransferBuildStep):

506 """ 507 Download the first 'maxsize' bytes of a string, from the buildmaster to the 508 buildslave. Set the mode of the file 509 510 Arguments:: 511 512 ['s'] string to transfer 513 ['slavedest'] filename of destination file at slave 514 ['workdir'] string with slave working directory relative to builder 515 base dir, default 'build' 516 ['maxsize'] maximum size of the file, default None (=unlimited) 517 ['blocksize'] maximum size of each block being transfered 518 ['mode'] use this to set the access permissions of the resulting 519 buildslave-side file. This is traditionally an octal 520 integer, like 0644 to be world-readable (but not 521 world-writable), or 0600 to only be readable by 522 the buildslave account, or 0755 to be world-executable. 523 The default (=None) is to leave it up to the umask of 524 the buildslave process. 525 """ 526 name = 'string_download' 527

528 - def __init__(self, s, slavedest, 529 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 530 **buildstep_kwargs):

531 BuildStep.__init__(self, **buildstep_kwargs) 532 self.addFactoryArguments(s=s, 533 slavedest=slavedest, 534 workdir=workdir, 535 maxsize=maxsize, 536 blocksize=blocksize, 537 mode=mode, 538 ) 539 540 self.s = s 541 self.slavedest = slavedest 542 self.workdir = workdir 543 self.maxsize = maxsize 544 self.blocksize = blocksize 545 assert isinstance(mode, (int, type(None))) 546 self.mode = mode

547

548 - def start(self):

549 properties = self.build.getProperties() 550 551 version = self.slaveVersion("downloadFile") 552 if not version: 553 m = "slave is too old, does not know about downloadFile" 554 raise BuildSlaveTooOldError(m) 555 556 # we are currently in the buildmaster's basedir, so any non-absolute 557 # paths will be interpreted relative to that 558 slavedest = properties.render(self.slavedest) 559 log.msg("StringDownload started, from master to slave %r" % slavedest) 560 561 self.step_status.setText(['downloading', "to", 562 os.path.basename(slavedest)]) 563 564 # setup structures for reading the file 565 fp = StringIO(properties.render(self.s)) 566 fileReader = _FileReader(fp) 567 568 # default arguments 569 args = { 570 'slavedest': slavedest, 571 'maxsize': self.maxsize, 572 'reader': fileReader, 573 'blocksize': self.blocksize, 574 'workdir': self._getWorkdir(), 575 'mode': self.mode, 576 } 577 578 self.cmd = StatusRemoteCommand('downloadFile', args) 579 d = self.runCommand(self.cmd) 580 d.addCallback(self.finished).addErrback(self.failed)

581

582 -class JSONStringDownload(StringDownload):

583 """ 584 Encode object o as a json string and save it on the buildslave 585 586 Arguments:: 587 588 ['o'] object to encode and transfer 589 """ 590 name = "json_download"

591 - def __init__(self, o, slavedest, **buildstep_kwargs):

592 if 's' in buildstep_kwargs: 593 del buildstep_kwargs['s'] 594 s = json.dumps(o) 595 StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs) 596 self.addFactoryArguments(o=o)

597

598 -class JSONPropertiesDownload(StringDownload):

599 """ 600 Download the current build properties as a json string and save it on the 601 buildslave 602 """ 603 name = "json_properties_download"

604 - def __init__(self, slavedest, **buildstep_kwargs):

605 self.super_class = StringDownload 606 if 's' in buildstep_kwargs: 607 del buildstep_kwargs['s'] 608 StringDownload.__init__(self, s=None, slavedest=slavedest, **buildstep_kwargs)

609

610 - def start(self):

611 properties = self.build.getProperties() 612 props = {} 613 for key, value, source in properties.asList(): 614 props[key] = value 615 616 self.s = json.dumps(dict( 617 properties=props, 618 sourcestamp=self.build.getSourceStamp().asDict(), 619 ), 620 ) 621 return self.super_class.start(self)

622

Source Code for Module buildbot.steps.transfer