buildbot.steps.transfer

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from __future__ import with_statement 17 18 19 import os.path, tarfile, tempfile 20 try: 21 from cStringIO import StringIO 22 assert StringIO 23 except ImportError: 24 from StringIO import StringIO 25 from twisted.internet import reactor 26 from twisted.spread import pb 27 from twisted.python import log 28 from buildbot.process import buildstep 29 from buildbot.process.buildstep import BuildStep 30 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 31 from buildbot.interfaces import BuildSlaveTooOldError 32 from buildbot.util import json 33 from buildbot import config

34 35 36 -class _FileWriter(pb.Referenceable):

37 """ 38 Helper class that acts as a file-object with write access 39 """ 40

41 - def __init__(self, destfile, maxsize, mode):

42 # Create missing directories. 43 destfile = os.path.abspath(destfile) 44 dirname = os.path.dirname(destfile) 45 if not os.path.exists(dirname): 46 os.makedirs(dirname) 47 48 self.destfile = destfile 49 self.mode = mode 50 fd, self.tmpname = tempfile.mkstemp(dir=dirname) 51 self.fp = os.fdopen(fd, 'wb') 52 self.remaining = maxsize

53

54 - def remote_write(self, data):

55 """ 56 Called from remote slave to write L{data} to L{fp} within boundaries 57 of L{maxsize} 58 59 @type data: C{string} 60 @param data: String of data to write 61 """ 62 if self.remaining is not None: 63 if len(data) > self.remaining: 64 data = data[:self.remaining] 65 self.fp.write(data) 66 self.remaining = self.remaining - len(data) 67 else: 68 self.fp.write(data)

69

70 - def remote_utime(self, accessed_modified):

71 os.utime(self.destfile,accessed_modified)

72

73 - def remote_close(self):

74 """ 75 Called by remote slave to state that no more data will be transfered 76 """ 77 self.fp.close() 78 self.fp = None 79 # on windows, os.rename does not automatically unlink, so do it manually 80 if os.path.exists(self.destfile): 81 os.unlink(self.destfile) 82 os.rename(self.tmpname, self.destfile) 83 self.tmpname = None 84 if self.mode is not None: 85 os.chmod(self.destfile, self.mode)

86

87 - def cancel(self):

88 # unclean shutdown, the file is probably truncated, so delete it 89 # altogether rather than deliver a corrupted file 90 fp = getattr(self, "fp", None) 91 if fp: 92 fp.close() 93 os.unlink(self.destfile) 94 if self.tmpname and os.path.exists(self.tmpname): 95 os.unlink(self.tmpname)

96

97 98 -def _extractall(self, path=".", members=None):

99 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 100 101 import copy 102 103 directories = [] 104 105 if members is None: 106 members = self 107 108 for tarinfo in members: 109 if tarinfo.isdir(): 110 # Extract directories with a safe mode. 111 directories.append(tarinfo) 112 tarinfo = copy.copy(tarinfo) 113 tarinfo.mode = 0700 114 self.extract(tarinfo, path) 115 116 # Reverse sort directories. 117 directories.sort(lambda a, b: cmp(a.name, b.name)) 118 directories.reverse() 119 120 # Set correct owner, mtime and filemode on directories. 121 for tarinfo in directories: 122 dirpath = os.path.join(path, tarinfo.name) 123 try: 124 self.chown(tarinfo, dirpath) 125 self.utime(tarinfo, dirpath) 126 self.chmod(tarinfo, dirpath) 127 except tarfile.ExtractError, e: 128 if self.errorlevel > 1: 129 raise 130 else: 131 self._dbg(1, "tarfile: %s" % e)

132

133 -class _DirectoryWriter(_FileWriter):

134 """ 135 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 136 step to unpack the archive, once the transfer has completed. 137 """ 138

139 - def __init__(self, destroot, maxsize, compress, mode):

140 self.destroot = destroot 141 self.compress = compress 142 143 self.fd, self.tarname = tempfile.mkstemp() 144 os.close(self.fd) 145 146 _FileWriter.__init__(self, self.tarname, maxsize, mode)

147

148 - def remote_unpack(self):

149 """ 150 Called by remote slave to state that no more data will be transfered 151 """ 152 # Make sure remote_close is called, otherwise atomic rename wont happen 153 self.remote_close() 154 155 # Map configured compression to a TarFile setting 156 if self.compress == 'bz2': 157 mode='r|bz2' 158 elif self.compress == 'gz': 159 mode='r|gz' 160 else: 161 mode = 'r' 162 163 # Support old python 164 if not hasattr(tarfile.TarFile, 'extractall'): 165 tarfile.TarFile.extractall = _extractall 166 167 # Unpack archive and clean up after self 168 archive = tarfile.open(name=self.tarname, mode=mode) 169 archive.extractall(path=self.destroot) 170 archive.close() 171 os.remove(self.tarname)

172

173 174 -def makeStatusRemoteCommand(step, remote_command, args):

175 self = buildstep.RemoteCommand(remote_command, args, decodeRC={None:SUCCESS, 0:SUCCESS}) 176 callback = lambda arg: step.step_status.addLog('stdio') 177 self.useLogDelayed('stdio', callback, True) 178 return self

179

180 -class _TransferBuildStep(BuildStep):

181 """ 182 Base class for FileUpload and FileDownload to factor out common 183 functionality. 184 """ 185 DEFAULT_WORKDIR = "build" # is this redundant? 186 187 renderables = [ 'workdir' ] 188 189 haltOnFailure = True 190 flunkOnFailure = True 191

192 - def setDefaultWorkdir(self, workdir):

193 if self.workdir is None: 194 self.workdir = workdir

195

196 - def _getWorkdir(self):

197 if self.workdir is None: 198 workdir = self.DEFAULT_WORKDIR 199 else: 200 workdir = self.workdir 201 return workdir

202

203 - def interrupt(self, reason):

204 self.addCompleteLog('interrupt', str(reason)) 205 if self.cmd: 206 d = self.cmd.interrupt(reason) 207 return d

208

209 - def finished(self, result):

210 # Subclasses may choose to skip a transfer. In those cases, self.cmd 211 # will be None, and we should just let BuildStep.finished() handle 212 # the rest 213 if result == SKIPPED: 214 return BuildStep.finished(self, SKIPPED) 215 216 if self.cmd.didFail(): 217 return BuildStep.finished(self, FAILURE) 218 return BuildStep.finished(self, SUCCESS)

219

220 221 -class FileUpload(_TransferBuildStep):

222 223 name = 'upload' 224 225 renderables = [ 'slavesrc', 'masterdest', 'url' ] 226

227 - def __init__(self, slavesrc, masterdest, 228 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 229 keepstamp=False, url=None, 230 **buildstep_kwargs):

231 BuildStep.__init__(self, **buildstep_kwargs) 232 233 self.slavesrc = slavesrc 234 self.masterdest = masterdest 235 self.workdir = workdir 236 self.maxsize = maxsize 237 self.blocksize = blocksize 238 if not isinstance(mode, (int, type(None))): 239 config.error( 240 'mode must be an integer or None') 241 self.mode = mode 242 self.keepstamp = keepstamp 243 self.url = url

244

245 - def start(self):

246 version = self.slaveVersion("uploadFile") 247 248 if not version: 249 m = "slave is too old, does not know about uploadFile" 250 raise BuildSlaveTooOldError(m) 251 252 source = self.slavesrc 253 masterdest = self.masterdest 254 # we rely upon the fact that the buildmaster runs chdir'ed into its 255 # basedir to make sure that relative paths in masterdest are expanded 256 # properly. TODO: maybe pass the master's basedir all the way down 257 # into the BuildStep so we can do this better. 258 masterdest = os.path.expanduser(masterdest) 259 log.msg("FileUpload started, from slave %r to master %r" 260 % (source, masterdest)) 261 262 self.step_status.setText(['uploading', os.path.basename(source)]) 263 if self.url is not None: 264 self.addURL(os.path.basename(masterdest), self.url) 265 266 # we use maxsize to limit the amount of data on both sides 267 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 268 269 if self.keepstamp and self.slaveVersionIsOlderThan("uploadFile","2.13"): 270 m = ("This buildslave (%s) does not support preserving timestamps. " 271 "Please upgrade the buildslave." % self.build.slavename ) 272 raise BuildSlaveTooOldError(m) 273 274 # default arguments 275 args = { 276 'slavesrc': source, 277 'workdir': self._getWorkdir(), 278 'writer': fileWriter, 279 'maxsize': self.maxsize, 280 'blocksize': self.blocksize, 281 'keepstamp': self.keepstamp, 282 } 283 284 self.cmd = makeStatusRemoteCommand(self, 'uploadFile', args) 285 d = self.runCommand(self.cmd) 286 @d.addErrback 287 def cancel(res): 288 fileWriter.cancel() 289 return res

290 d.addCallback(self.finished).addErrback(self.failed)

291

292 293 -class DirectoryUpload(_TransferBuildStep):

294 295 name = 'upload' 296 297 renderables = [ 'slavesrc', 'masterdest', 'url' ] 298

299 - def __init__(self, slavesrc, masterdest, 300 workdir=None, maxsize=None, blocksize=16*1024, 301 compress=None, url=None, **buildstep_kwargs):

302 BuildStep.__init__(self, **buildstep_kwargs) 303 304 self.slavesrc = slavesrc 305 self.masterdest = masterdest 306 self.workdir = workdir 307 self.maxsize = maxsize 308 self.blocksize = blocksize 309 if compress not in (None, 'gz', 'bz2'): 310 config.error( 311 "'compress' must be one of None, 'gz', or 'bz2'") 312 self.compress = compress 313 self.url = url

314

315 - def start(self):

316 version = self.slaveVersion("uploadDirectory") 317 318 if not version: 319 m = "slave is too old, does not know about uploadDirectory" 320 raise BuildSlaveTooOldError(m) 321 322 source = self.slavesrc 323 masterdest = self.masterdest 324 # we rely upon the fact that the buildmaster runs chdir'ed into its 325 # basedir to make sure that relative paths in masterdest are expanded 326 # properly. TODO: maybe pass the master's basedir all the way down 327 # into the BuildStep so we can do this better. 328 masterdest = os.path.expanduser(masterdest) 329 log.msg("DirectoryUpload started, from slave %r to master %r" 330 % (source, masterdest)) 331 332 self.step_status.setText(['uploading', os.path.basename(source)]) 333 if self.url is not None: 334 self.addURL(os.path.basename(masterdest), self.url) 335 336 # we use maxsize to limit the amount of data on both sides 337 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600) 338 339 # default arguments 340 args = { 341 'slavesrc': source, 342 'workdir': self._getWorkdir(), 343 'writer': dirWriter, 344 'maxsize': self.maxsize, 345 'blocksize': self.blocksize, 346 'compress': self.compress 347 } 348 349 self.cmd = makeStatusRemoteCommand(self, 'uploadDirectory', args) 350 d = self.runCommand(self.cmd) 351 @d.addErrback 352 def cancel(res): 353 dirWriter.cancel() 354 return res

355 d.addCallback(self.finished).addErrback(self.failed)

356

357 - def finished(self, result):

358 # Subclasses may choose to skip a transfer. In those cases, self.cmd 359 # will be None, and we should just let BuildStep.finished() handle 360 # the rest 361 if result == SKIPPED: 362 return BuildStep.finished(self, SKIPPED) 363 364 if self.cmd.didFail(): 365 return BuildStep.finished(self, FAILURE) 366 return BuildStep.finished(self, SUCCESS)

367

368 369 -class _FileReader(pb.Referenceable):

370 """ 371 Helper class that acts as a file-object with read access 372 """ 373

374 - def __init__(self, fp):

375 self.fp = fp

376

377 - def remote_read(self, maxlength):

378 """ 379 Called from remote slave to read at most L{maxlength} bytes of data 380 381 @type maxlength: C{integer} 382 @param maxlength: Maximum number of data bytes that can be returned 383 384 @return: Data read from L{fp} 385 @rtype: C{string} of bytes read from file 386 """ 387 if self.fp is None: 388 return '' 389 390 data = self.fp.read(maxlength) 391 return data

392

393 - def remote_close(self):

394 """ 395 Called by remote slave to state that no more data will be transfered 396 """ 397 if self.fp is not None: 398 self.fp.close() 399 self.fp = None

400

401 402 -class FileDownload(_TransferBuildStep):

403 404 name = 'download' 405 406 renderables = [ 'mastersrc', 'slavedest' ] 407

408 - def __init__(self, mastersrc, slavedest, 409 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 410 **buildstep_kwargs):

411 BuildStep.__init__(self, **buildstep_kwargs) 412 413 self.mastersrc = mastersrc 414 self.slavedest = slavedest 415 self.workdir = workdir 416 self.maxsize = maxsize 417 self.blocksize = blocksize 418 if not isinstance(mode, (int, type(None))): 419 config.error( 420 'mode must be an integer or None') 421 self.mode = mode

422

423 - def start(self):

424 version = self.slaveVersion("downloadFile") 425 if not version: 426 m = "slave is too old, does not know about downloadFile" 427 raise BuildSlaveTooOldError(m) 428 429 # we are currently in the buildmaster's basedir, so any non-absolute 430 # paths will be interpreted relative to that 431 source = os.path.expanduser(self.mastersrc) 432 slavedest = self.slavedest 433 log.msg("FileDownload started, from master %r to slave %r" % 434 (source, slavedest)) 435 436 self.step_status.setText(['downloading', "to", 437 os.path.basename(slavedest)]) 438 439 # setup structures for reading the file 440 try: 441 fp = open(source, 'rb') 442 except IOError: 443 # if file does not exist, bail out with an error 444 self.addCompleteLog('stderr', 445 'File %r not available at master' % source) 446 # TODO: once BuildStep.start() gets rewritten to use 447 # maybeDeferred, just re-raise the exception here. 448 reactor.callLater(0, BuildStep.finished, self, FAILURE) 449 return 450 fileReader = _FileReader(fp) 451 452 # default arguments 453 args = { 454 'slavedest': slavedest, 455 'maxsize': self.maxsize, 456 'reader': fileReader, 457 'blocksize': self.blocksize, 458 'workdir': self._getWorkdir(), 459 'mode': self.mode, 460 } 461 462 self.cmd = makeStatusRemoteCommand(self, 'downloadFile', args) 463 d = self.runCommand(self.cmd) 464 d.addCallback(self.finished).addErrback(self.failed)

465

466 -class StringDownload(_TransferBuildStep):

467 468 name = 'string_download' 469 470 renderables = [ 'slavedest', 's' ] 471

472 - def __init__(self, s, slavedest, 473 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 474 **buildstep_kwargs):

475 BuildStep.__init__(self, **buildstep_kwargs) 476 477 self.s = s 478 self.slavedest = slavedest 479 self.workdir = workdir 480 self.maxsize = maxsize 481 self.blocksize = blocksize 482 if not isinstance(mode, (int, type(None))): 483 config.error( 484 'mode must be an integer or None') 485 self.mode = mode

486

487 - def start(self):

488 version = self.slaveVersion("downloadFile") 489 if not version: 490 m = "slave is too old, does not know about downloadFile" 491 raise BuildSlaveTooOldError(m) 492 493 # we are currently in the buildmaster's basedir, so any non-absolute 494 # paths will be interpreted relative to that 495 slavedest = self.slavedest 496 log.msg("StringDownload started, from master to slave %r" % slavedest) 497 498 self.step_status.setText(['downloading', "to", 499 os.path.basename(slavedest)]) 500 501 # setup structures for reading the file 502 fp = StringIO(self.s) 503 fileReader = _FileReader(fp) 504 505 # default arguments 506 args = { 507 'slavedest': slavedest, 508 'maxsize': self.maxsize, 509 'reader': fileReader, 510 'blocksize': self.blocksize, 511 'workdir': self._getWorkdir(), 512 'mode': self.mode, 513 } 514 515 self.cmd = makeStatusRemoteCommand(self, 'downloadFile', args) 516 d = self.runCommand(self.cmd) 517 d.addCallback(self.finished).addErrback(self.failed)

518

519 -class JSONStringDownload(StringDownload):

520 521 name = "json_download" 522

523 - def __init__(self, o, slavedest, **buildstep_kwargs):

524 if 's' in buildstep_kwargs: 525 del buildstep_kwargs['s'] 526 s = json.dumps(o) 527 StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs)

528

529 -class JSONPropertiesDownload(StringDownload):

530 531 name = "json_properties_download" 532

533 - def __init__(self, slavedest, **buildstep_kwargs):

534 self.super_class = StringDownload 535 if 's' in buildstep_kwargs: 536 del buildstep_kwargs['s'] 537 StringDownload.__init__(self, s=None, slavedest=slavedest, **buildstep_kwargs)

538

539 - def start(self):

540 properties = self.build.getProperties() 541 props = {} 542 for key, value, source in properties.asList(): 543 props[key] = value 544 545 self.s = json.dumps(dict( 546 properties=props, 547 sourcestamp=self.build.getSourceStamp().asDict(), 548 ), 549 ) 550 return self.super_class.start(self)

551

Source Code for Module buildbot.steps.transfer