buildbot.steps.transfer

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import os.path, tarfile, tempfile 18 try: 19 from cStringIO import StringIO 20 assert StringIO 21 except ImportError: 22 from StringIO import StringIO 23 from twisted.internet import reactor 24 from twisted.spread import pb 25 from twisted.python import log 26 from buildbot.process.buildstep import RemoteCommand, BuildStep 27 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 28 from buildbot.interfaces import BuildSlaveTooOldError 29 from buildbot.util import json 30 from buildbot import config

31 32 33 -class _FileWriter(pb.Referenceable):

34 """ 35 Helper class that acts as a file-object with write access 36 """ 37

38 - def __init__(self, destfile, maxsize, mode):

39 # Create missing directories. 40 destfile = os.path.abspath(destfile) 41 dirname = os.path.dirname(destfile) 42 if not os.path.exists(dirname): 43 os.makedirs(dirname) 44 45 self.destfile = destfile 46 self.mode = mode 47 fd, self.tmpname = tempfile.mkstemp(dir=dirname) 48 self.fp = os.fdopen(fd, 'wb') 49 self.remaining = maxsize

50

51 - def remote_write(self, data):

52 """ 53 Called from remote slave to write L{data} to L{fp} within boundaries 54 of L{maxsize} 55 56 @type data: C{string} 57 @param data: String of data to write 58 """ 59 if self.remaining is not None: 60 if len(data) > self.remaining: 61 data = data[:self.remaining] 62 self.fp.write(data) 63 self.remaining = self.remaining - len(data) 64 else: 65 self.fp.write(data)

66

67 - def remote_utime(self, accessed_modified):

68 os.utime(self.destfile,accessed_modified)

69

70 - def remote_close(self):

71 """ 72 Called by remote slave to state that no more data will be transfered 73 """ 74 self.fp.close() 75 self.fp = None 76 # on windows, os.rename does not automatically unlink, so do it manually 77 if os.path.exists(self.destfile): 78 os.unlink(self.destfile) 79 os.rename(self.tmpname, self.destfile) 80 self.tmpname = None 81 if self.mode is not None: 82 os.chmod(self.destfile, self.mode)

83

84 - def cancel(self):

85 # unclean shutdown, the file is probably truncated, so delete it 86 # altogether rather than deliver a corrupted file 87 fp = getattr(self, "fp", None) 88 if fp: 89 fp.close() 90 os.unlink(self.destfile) 91 if self.tmpname and os.path.exists(self.tmpname): 92 os.unlink(self.tmpname)

93

94 95 -def _extractall(self, path=".", members=None):

96 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 97 98 import copy 99 100 directories = [] 101 102 if members is None: 103 members = self 104 105 for tarinfo in members: 106 if tarinfo.isdir(): 107 # Extract directories with a safe mode. 108 directories.append(tarinfo) 109 tarinfo = copy.copy(tarinfo) 110 tarinfo.mode = 0700 111 self.extract(tarinfo, path) 112 113 # Reverse sort directories. 114 directories.sort(lambda a, b: cmp(a.name, b.name)) 115 directories.reverse() 116 117 # Set correct owner, mtime and filemode on directories. 118 for tarinfo in directories: 119 dirpath = os.path.join(path, tarinfo.name) 120 try: 121 self.chown(tarinfo, dirpath) 122 self.utime(tarinfo, dirpath) 123 self.chmod(tarinfo, dirpath) 124 except tarfile.ExtractError, e: 125 if self.errorlevel > 1: 126 raise 127 else: 128 self._dbg(1, "tarfile: %s" % e)

129

130 -class _DirectoryWriter(_FileWriter):

131 """ 132 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 133 step to unpack the archive, once the transfer has completed. 134 """ 135

136 - def __init__(self, destroot, maxsize, compress, mode):

137 self.destroot = destroot 138 self.compress = compress 139 140 self.fd, self.tarname = tempfile.mkstemp() 141 os.close(self.fd) 142 143 _FileWriter.__init__(self, self.tarname, maxsize, mode)

144

145 - def remote_unpack(self):

146 """ 147 Called by remote slave to state that no more data will be transfered 148 """ 149 # Make sure remote_close is called, otherwise atomic rename wont happen 150 self.remote_close() 151 152 # Map configured compression to a TarFile setting 153 if self.compress == 'bz2': 154 mode='r|bz2' 155 elif self.compress == 'gz': 156 mode='r|gz' 157 else: 158 mode = 'r' 159 160 # Support old python 161 if not hasattr(tarfile.TarFile, 'extractall'): 162 tarfile.TarFile.extractall = _extractall 163 164 # Unpack archive and clean up after self 165 archive = tarfile.open(name=self.tarname, mode=mode) 166 archive.extractall(path=self.destroot) 167 archive.close() 168 os.remove(self.tarname)

169

170 171 -class StatusRemoteCommand(RemoteCommand):

172 - def __init__(self, step, remote_command, args):

173 RemoteCommand.__init__(self, remote_command, args) 174 callback = lambda arg: step.step_status.addLog('stdio') 175 self.useLogDelayed('stdio', callback, True)

176

177 -class _TransferBuildStep(BuildStep):

178 """ 179 Base class for FileUpload and FileDownload to factor out common 180 functionality. 181 """ 182 DEFAULT_WORKDIR = "build" # is this redundant? 183 184 renderables = [ 'workdir' ] 185 186 haltOnFailure = True 187 flunkOnFailure = True 188

189 - def setDefaultWorkdir(self, workdir):

190 if self.workdir is None: 191 self.workdir = workdir

192

193 - def _getWorkdir(self):

194 if self.workdir is None: 195 workdir = self.DEFAULT_WORKDIR 196 else: 197 workdir = self.workdir 198 return workdir

199

200 - def interrupt(self, reason):

201 self.addCompleteLog('interrupt', str(reason)) 202 if self.cmd: 203 d = self.cmd.interrupt(reason) 204 return d

205

206 - def finished(self, result):

207 # Subclasses may choose to skip a transfer. In those cases, self.cmd 208 # will be None, and we should just let BuildStep.finished() handle 209 # the rest 210 if result == SKIPPED: 211 return BuildStep.finished(self, SKIPPED) 212 213 if self.cmd.rc is None or self.cmd.rc == 0: 214 return BuildStep.finished(self, SUCCESS) 215 return BuildStep.finished(self, FAILURE)

216

217 218 -class FileUpload(_TransferBuildStep):

219 220 name = 'upload' 221 222 renderables = [ 'slavesrc', 'masterdest', 'url' ] 223

224 - def __init__(self, slavesrc, masterdest, 225 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 226 keepstamp=False, url=None, 227 **buildstep_kwargs):

228 BuildStep.__init__(self, **buildstep_kwargs) 229 self.addFactoryArguments(slavesrc=slavesrc, 230 masterdest=masterdest, 231 workdir=workdir, 232 maxsize=maxsize, 233 blocksize=blocksize, 234 mode=mode, 235 keepstamp=keepstamp, 236 url=url, 237 ) 238 239 self.slavesrc = slavesrc 240 self.masterdest = masterdest 241 self.workdir = workdir 242 self.maxsize = maxsize 243 self.blocksize = blocksize 244 if not isinstance(mode, (int, type(None))): 245 config.error( 246 'mode must be an integer or None') 247 self.mode = mode 248 self.keepstamp = keepstamp 249 self.url = url

250

251 - def start(self):

252 version = self.slaveVersion("uploadFile") 253 254 if not version: 255 m = "slave is too old, does not know about uploadFile" 256 raise BuildSlaveTooOldError(m) 257 258 source = self.slavesrc 259 masterdest = self.masterdest 260 # we rely upon the fact that the buildmaster runs chdir'ed into its 261 # basedir to make sure that relative paths in masterdest are expanded 262 # properly. TODO: maybe pass the master's basedir all the way down 263 # into the BuildStep so we can do this better. 264 masterdest = os.path.expanduser(masterdest) 265 log.msg("FileUpload started, from slave %r to master %r" 266 % (source, masterdest)) 267 268 self.step_status.setText(['uploading', os.path.basename(source)]) 269 if self.url is not None: 270 self.addURL(os.path.basename(masterdest), self.url) 271 272 # we use maxsize to limit the amount of data on both sides 273 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 274 275 if self.keepstamp and self.slaveVersionIsOlderThan("uploadFile","2.13"): 276 m = ("This buildslave (%s) does not support preserving timestamps. " 277 "Please upgrade the buildslave." % self.build.slavename ) 278 raise BuildSlaveTooOldError(m) 279 280 # default arguments 281 args = { 282 'slavesrc': source, 283 'workdir': self._getWorkdir(), 284 'writer': fileWriter, 285 'maxsize': self.maxsize, 286 'blocksize': self.blocksize, 287 'keepstamp': self.keepstamp, 288 } 289 290 self.cmd = StatusRemoteCommand(self, 'uploadFile', args) 291 d = self.runCommand(self.cmd) 292 @d.addErrback 293 def cancel(res): 294 fileWriter.cancel() 295 return res

296 d.addCallback(self.finished).addErrback(self.failed)

297

298 299 -class DirectoryUpload(_TransferBuildStep):

300 301 name = 'upload' 302 303 renderables = [ 'slavesrc', 'masterdest', 'url' ] 304

305 - def __init__(self, slavesrc, masterdest, 306 workdir=None, maxsize=None, blocksize=16*1024, 307 compress=None, url=None, **buildstep_kwargs):

308 BuildStep.__init__(self, **buildstep_kwargs) 309 self.addFactoryArguments(slavesrc=slavesrc, 310 masterdest=masterdest, 311 workdir=workdir, 312 maxsize=maxsize, 313 blocksize=blocksize, 314 compress=compress, 315 url=url, 316 ) 317 318 self.slavesrc = slavesrc 319 self.masterdest = masterdest 320 self.workdir = workdir 321 self.maxsize = maxsize 322 self.blocksize = blocksize 323 if compress not in (None, 'gz', 'bz2'): 324 config.error( 325 "'compress' must be one of None, 'gz', or 'bz2'") 326 self.compress = compress 327 self.url = url

328

329 - def start(self):

330 version = self.slaveVersion("uploadDirectory") 331 332 if not version: 333 m = "slave is too old, does not know about uploadDirectory" 334 raise BuildSlaveTooOldError(m) 335 336 source = self.slavesrc 337 masterdest = self.masterdest 338 # we rely upon the fact that the buildmaster runs chdir'ed into its 339 # basedir to make sure that relative paths in masterdest are expanded 340 # properly. TODO: maybe pass the master's basedir all the way down 341 # into the BuildStep so we can do this better. 342 masterdest = os.path.expanduser(masterdest) 343 log.msg("DirectoryUpload started, from slave %r to master %r" 344 % (source, masterdest)) 345 346 self.step_status.setText(['uploading', os.path.basename(source)]) 347 if self.url is not None: 348 self.addURL(os.path.basename(masterdest), self.url) 349 350 # we use maxsize to limit the amount of data on both sides 351 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600) 352 353 # default arguments 354 args = { 355 'slavesrc': source, 356 'workdir': self._getWorkdir(), 357 'writer': dirWriter, 358 'maxsize': self.maxsize, 359 'blocksize': self.blocksize, 360 'compress': self.compress 361 } 362 363 self.cmd = StatusRemoteCommand(self, 'uploadDirectory', args) 364 d = self.runCommand(self.cmd) 365 @d.addErrback 366 def cancel(res): 367 dirWriter.cancel() 368 return res

369 d.addCallback(self.finished).addErrback(self.failed)

370

371 - def finished(self, result):

372 # Subclasses may choose to skip a transfer. In those cases, self.cmd 373 # will be None, and we should just let BuildStep.finished() handle 374 # the rest 375 if result == SKIPPED: 376 return BuildStep.finished(self, SKIPPED) 377 378 if self.cmd.rc is None or self.cmd.rc == 0: 379 return BuildStep.finished(self, SUCCESS) 380 return BuildStep.finished(self, FAILURE)

381

382 383 384 385 -class _FileReader(pb.Referenceable):

386 """ 387 Helper class that acts as a file-object with read access 388 """ 389

390 - def __init__(self, fp):

391 self.fp = fp

392

393 - def remote_read(self, maxlength):

394 """ 395 Called from remote slave to read at most L{maxlength} bytes of data 396 397 @type maxlength: C{integer} 398 @param maxlength: Maximum number of data bytes that can be returned 399 400 @return: Data read from L{fp} 401 @rtype: C{string} of bytes read from file 402 """ 403 if self.fp is None: 404 return '' 405 406 data = self.fp.read(maxlength) 407 return data

408

409 - def remote_close(self):

410 """ 411 Called by remote slave to state that no more data will be transfered 412 """ 413 if self.fp is not None: 414 self.fp.close() 415 self.fp = None

416

417 418 -class FileDownload(_TransferBuildStep):

419 420 name = 'download' 421 422 renderables = [ 'mastersrc', 'slavedest' ] 423

424 - def __init__(self, mastersrc, slavedest, 425 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 426 **buildstep_kwargs):

427 BuildStep.__init__(self, **buildstep_kwargs) 428 self.addFactoryArguments(mastersrc=mastersrc, 429 slavedest=slavedest, 430 workdir=workdir, 431 maxsize=maxsize, 432 blocksize=blocksize, 433 mode=mode, 434 ) 435 436 self.mastersrc = mastersrc 437 self.slavedest = slavedest 438 self.workdir = workdir 439 self.maxsize = maxsize 440 self.blocksize = blocksize 441 if not isinstance(mode, (int, type(None))): 442 config.error( 443 'mode must be an integer or None') 444 self.mode = mode

445

446 - def start(self):

447 version = self.slaveVersion("downloadFile") 448 if not version: 449 m = "slave is too old, does not know about downloadFile" 450 raise BuildSlaveTooOldError(m) 451 452 # we are currently in the buildmaster's basedir, so any non-absolute 453 # paths will be interpreted relative to that 454 source = os.path.expanduser(self.mastersrc) 455 slavedest = self.slavedest 456 log.msg("FileDownload started, from master %r to slave %r" % 457 (source, slavedest)) 458 459 self.step_status.setText(['downloading', "to", 460 os.path.basename(slavedest)]) 461 462 # setup structures for reading the file 463 try: 464 fp = open(source, 'rb') 465 except IOError: 466 # if file does not exist, bail out with an error 467 self.addCompleteLog('stderr', 468 'File %r not available at master' % source) 469 # TODO: once BuildStep.start() gets rewritten to use 470 # maybeDeferred, just re-raise the exception here. 471 reactor.callLater(0, BuildStep.finished, self, FAILURE) 472 return 473 fileReader = _FileReader(fp) 474 475 # default arguments 476 args = { 477 'slavedest': slavedest, 478 'maxsize': self.maxsize, 479 'reader': fileReader, 480 'blocksize': self.blocksize, 481 'workdir': self._getWorkdir(), 482 'mode': self.mode, 483 } 484 485 self.cmd = StatusRemoteCommand(self, 'downloadFile', args) 486 d = self.runCommand(self.cmd) 487 d.addCallback(self.finished).addErrback(self.failed)

488

489 -class StringDownload(_TransferBuildStep):

490 491 name = 'string_download' 492 493 renderables = [ 'slavedest', 's' ] 494

495 - def __init__(self, s, slavedest, 496 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 497 **buildstep_kwargs):

498 BuildStep.__init__(self, **buildstep_kwargs) 499 self.addFactoryArguments(s=s, 500 slavedest=slavedest, 501 workdir=workdir, 502 maxsize=maxsize, 503 blocksize=blocksize, 504 mode=mode, 505 ) 506 507 self.s = s 508 self.slavedest = slavedest 509 self.workdir = workdir 510 self.maxsize = maxsize 511 self.blocksize = blocksize 512 if not isinstance(mode, (int, type(None))): 513 config.error( 514 'mode must be an integer or None') 515 self.mode = mode

516

517 - def start(self):

518 version = self.slaveVersion("downloadFile") 519 if not version: 520 m = "slave is too old, does not know about downloadFile" 521 raise BuildSlaveTooOldError(m) 522 523 # we are currently in the buildmaster's basedir, so any non-absolute 524 # paths will be interpreted relative to that 525 slavedest = self.slavedest 526 log.msg("StringDownload started, from master to slave %r" % slavedest) 527 528 self.step_status.setText(['downloading', "to", 529 os.path.basename(slavedest)]) 530 531 # setup structures for reading the file 532 fp = StringIO(self.s) 533 fileReader = _FileReader(fp) 534 535 # default arguments 536 args = { 537 'slavedest': slavedest, 538 'maxsize': self.maxsize, 539 'reader': fileReader, 540 'blocksize': self.blocksize, 541 'workdir': self._getWorkdir(), 542 'mode': self.mode, 543 } 544 545 self.cmd = StatusRemoteCommand(self, 'downloadFile', args) 546 d = self.runCommand(self.cmd) 547 d.addCallback(self.finished).addErrback(self.failed)

548

549 -class JSONStringDownload(StringDownload):

550 551 name = "json_download" 552

553 - def __init__(self, o, slavedest, **buildstep_kwargs):

554 if 's' in buildstep_kwargs: 555 del buildstep_kwargs['s'] 556 s = json.dumps(o) 557 StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs) 558 self.addFactoryArguments(o=o)

559

560 -class JSONPropertiesDownload(StringDownload):

561 562 name = "json_properties_download" 563

564 - def __init__(self, slavedest, **buildstep_kwargs):

565 self.super_class = StringDownload 566 if 's' in buildstep_kwargs: 567 del buildstep_kwargs['s'] 568 StringDownload.__init__(self, s=None, slavedest=slavedest, **buildstep_kwargs)

569

570 - def start(self):

571 properties = self.build.getProperties() 572 props = {} 573 for key, value, source in properties.asList(): 574 props[key] = value 575 576 self.s = json.dumps(dict( 577 properties=props, 578 sourcestamp=self.build.getSourceStamp().asDict(), 579 ), 580 ) 581 return self.super_class.start(self)

582

Source Code for Module buildbot.steps.transfer