buildbot.steps.transfer

1 # -*- test-case-name: buildbot.test.test_transfer -*- 2 3 import os.path, tarfile, tempfile 4 from twisted.internet import reactor 5 from twisted.spread import pb 6 from twisted.python import log 7 from buildbot.process.buildstep import RemoteCommand, BuildStep 8 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 9 from buildbot.interfaces import BuildSlaveTooOldError 10 11

12 -class _FileWriter(pb.Referenceable):

13 """ 14 Helper class that acts as a file-object with write access 15 """ 16

17 - def __init__(self, destfile, maxsize, mode):

18 # Create missing directories. 19 destfile = os.path.abspath(destfile) 20 dirname = os.path.dirname(destfile) 21 if not os.path.exists(dirname): 22 os.makedirs(dirname) 23 24 self.destfile = destfile 25 self.fp = open(destfile, "wb") 26 if mode is not None: 27 os.chmod(destfile, mode) 28 self.remaining = maxsize

29

30 - def remote_write(self, data):

31 """ 32 Called from remote slave to write L{data} to L{fp} within boundaries 33 of L{maxsize} 34 35 @type data: C{string} 36 @param data: String of data to write 37 """ 38 if self.remaining is not None: 39 if len(data) > self.remaining: 40 data = data[:self.remaining] 41 self.fp.write(data) 42 self.remaining = self.remaining - len(data) 43 else: 44 self.fp.write(data)

45

46 - def remote_close(self):

47 """ 48 Called by remote slave to state that no more data will be transfered 49 """ 50 self.fp.close() 51 self.fp = None

52

53 - def __del__(self):

54 # unclean shutdown, the file is probably truncated, so delete it 55 # altogether rather than deliver a corrupted file 56 fp = getattr(self, "fp", None) 57 if fp: 58 fp.close() 59 os.unlink(self.destfile)

60 61

62 -def _extractall(self, path=".", members=None):

63 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 64 65 import copy 66 import operator 67 68 directories = [] 69 70 if members is None: 71 members = self 72 73 for tarinfo in members: 74 if tarinfo.isdir(): 75 # Extract directories with a safe mode. 76 directories.append(tarinfo) 77 tarinfo = copy.copy(tarinfo) 78 tarinfo.mode = 0700 79 self.extract(tarinfo, path) 80 81 # Reverse sort directories. 82 directories.sort(lambda a, b: cmp(a.name, b.name)) 83 directories.reverse() 84 85 # Set correct owner, mtime and filemode on directories. 86 for tarinfo in directories: 87 dirpath = os.path.join(path, tarinfo.name) 88 try: 89 self.chown(tarinfo, dirpath) 90 self.utime(tarinfo, dirpath) 91 self.chmod(tarinfo, dirpath) 92 except tarfile.ExtractError, e: 93 if self.errorlevel > 1: 94 raise 95 else: 96 self._dbg(1, "tarfile: %s" % e)

97

98 -class _DirectoryWriter(_FileWriter):

99 """ 100 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 101 step to unpack the archive, once the transfer has completed. 102 """ 103

104 - def __init__(self, destroot, maxsize, compress, mode):

105 self.destroot = destroot 106 107 self.fd, self.tarname = tempfile.mkstemp() 108 self.compress = compress 109 _FileWriter.__init__(self, self.tarname, maxsize, mode)

110

111 - def remote_unpack(self):

112 """ 113 Called by remote slave to state that no more data will be transfered 114 """ 115 if self.fp: 116 self.fp.close() 117 self.fp = None 118 fileobj = os.fdopen(self.fd, 'r') 119 if self.compress == 'bz2': 120 mode='r|bz2' 121 elif self.compress == 'gz': 122 mode='r|gz' 123 else: 124 mode = 'r' 125 if not hasattr(tarfile.TarFile, 'extractall'): 126 tarfile.TarFile.extractall = _extractall 127 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) 128 archive.extractall(path=self.destroot) 129 os.remove(self.tarname)

130 131

132 -class StatusRemoteCommand(RemoteCommand):

133 - def __init__(self, remote_command, args):

134 RemoteCommand.__init__(self, remote_command, args) 135 136 self.rc = None 137 self.stderr = ''

138

139 - def remoteUpdate(self, update):

140 #log.msg('StatusRemoteCommand: update=%r' % update) 141 if 'rc' in update: 142 self.rc = update['rc'] 143 if 'stderr' in update: 144 self.stderr = self.stderr + update['stderr'] + '\n'

145

146 -class _TransferBuildStep(BuildStep):

147 """ 148 Base class for FileUpload and FileDownload to factor out common 149 functionality. 150 """ 151 DEFAULT_WORKDIR = "build" # is this redundant? 152

153 - def setDefaultWorkdir(self, workdir):

154 if self.workdir is None: 155 self.workdir = workdir

156

157 - def _getWorkdir(self):

158 properties = self.build.getProperties() 159 if self.workdir is None: 160 workdir = self.DEFAULT_WORKDIR 161 else: 162 workdir = self.workdir 163 return properties.render(workdir)

164

165 - def finished(self, result):

166 # Subclasses may choose to skip a transfer. In those cases, self.cmd 167 # will be None, and we should just let BuildStep.finished() handle 168 # the rest 169 if result == SKIPPED: 170 return BuildStep.finished(self, SKIPPED) 171 if self.cmd.stderr != '': 172 self.addCompleteLog('stderr', self.cmd.stderr) 173 174 if self.cmd.rc is None or self.cmd.rc == 0: 175 return BuildStep.finished(self, SUCCESS) 176 return BuildStep.finished(self, FAILURE)

177 178

179 -class FileUpload(_TransferBuildStep):

180 """ 181 Build step to transfer a file from the slave to the master. 182 183 arguments: 184 185 - ['slavesrc'] filename of source file at slave, relative to workdir 186 - ['masterdest'] filename of destination file at master 187 - ['workdir'] string with slave working directory relative to builder 188 base dir, default 'build' 189 - ['maxsize'] maximum size of the file, default None (=unlimited) 190 - ['blocksize'] maximum size of each block being transfered 191 - ['mode'] file access mode for the resulting master-side file. 192 The default (=None) is to leave it up to the umask of 193 the buildmaster process. 194 195 """ 196 197 name = 'upload' 198

199 - def __init__(self, slavesrc, masterdest, 200 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 201 **buildstep_kwargs):

202 BuildStep.__init__(self, **buildstep_kwargs) 203 self.addFactoryArguments(slavesrc=slavesrc, 204 masterdest=masterdest, 205 workdir=workdir, 206 maxsize=maxsize, 207 blocksize=blocksize, 208 mode=mode, 209 ) 210 211 self.slavesrc = slavesrc 212 self.masterdest = masterdest 213 self.workdir = workdir 214 self.maxsize = maxsize 215 self.blocksize = blocksize 216 assert isinstance(mode, (int, type(None))) 217 self.mode = mode

218

219 - def start(self):

220 version = self.slaveVersion("uploadFile") 221 properties = self.build.getProperties() 222 223 if not version: 224 m = "slave is too old, does not know about uploadFile" 225 raise BuildSlaveTooOldError(m) 226 227 source = properties.render(self.slavesrc) 228 masterdest = properties.render(self.masterdest) 229 # we rely upon the fact that the buildmaster runs chdir'ed into its 230 # basedir to make sure that relative paths in masterdest are expanded 231 # properly. TODO: maybe pass the master's basedir all the way down 232 # into the BuildStep so we can do this better. 233 masterdest = os.path.expanduser(masterdest) 234 log.msg("FileUpload started, from slave %r to master %r" 235 % (source, masterdest)) 236 237 self.step_status.setText(['uploading', os.path.basename(source)]) 238 239 # we use maxsize to limit the amount of data on both sides 240 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 241 242 # default arguments 243 args = { 244 'slavesrc': source, 245 'workdir': self._getWorkdir(), 246 'writer': fileWriter, 247 'maxsize': self.maxsize, 248 'blocksize': self.blocksize, 249 } 250 251 self.cmd = StatusRemoteCommand('uploadFile', args) 252 d = self.runCommand(self.cmd) 253 d.addCallback(self.finished).addErrback(self.failed)

254 255

256 -class DirectoryUpload(BuildStep):

257 """ 258 Build step to transfer a directory from the slave to the master. 259 260 arguments: 261 262 - ['slavesrc'] name of source directory at slave, relative to workdir 263 - ['masterdest'] name of destination directory at master 264 - ['workdir'] string with slave working directory relative to builder 265 base dir, default 'build' 266 - ['maxsize'] maximum size of the compressed tarfile containing the 267 whole directory 268 - ['blocksize'] maximum size of each block being transfered 269 - ['compress'] compression type to use: one of [None, 'gz', 'bz2'] 270 271 """ 272 273 name = 'upload' 274

275 - def __init__(self, slavesrc, masterdest, 276 workdir="build", maxsize=None, blocksize=16*1024, 277 compress=None, **buildstep_kwargs):

278 BuildStep.__init__(self, **buildstep_kwargs) 279 self.addFactoryArguments(slavesrc=slavesrc, 280 masterdest=masterdest, 281 workdir=workdir, 282 maxsize=maxsize, 283 blocksize=blocksize, 284 compress=compress, 285 ) 286 287 self.slavesrc = slavesrc 288 self.masterdest = masterdest 289 self.workdir = workdir 290 self.maxsize = maxsize 291 self.blocksize = blocksize 292 assert compress in (None, 'gz', 'bz2') 293 self.compress = compress

294

295 - def start(self):

296 version = self.slaveVersion("uploadDirectory") 297 properties = self.build.getProperties() 298 299 if not version: 300 m = "slave is too old, does not know about uploadDirectory" 301 raise BuildSlaveTooOldError(m) 302 303 source = properties.render(self.slavesrc) 304 masterdest = properties.render(self.masterdest) 305 # we rely upon the fact that the buildmaster runs chdir'ed into its 306 # basedir to make sure that relative paths in masterdest are expanded 307 # properly. TODO: maybe pass the master's basedir all the way down 308 # into the BuildStep so we can do this better. 309 masterdest = os.path.expanduser(masterdest) 310 log.msg("DirectoryUpload started, from slave %r to master %r" 311 % (source, masterdest)) 312 313 self.step_status.setText(['uploading', os.path.basename(source)]) 314 315 # we use maxsize to limit the amount of data on both sides 316 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600) 317 318 # default arguments 319 args = { 320 'slavesrc': source, 321 'workdir': self.workdir, 322 'writer': dirWriter, 323 'maxsize': self.maxsize, 324 'blocksize': self.blocksize, 325 'compress': self.compress 326 } 327 328 self.cmd = StatusRemoteCommand('uploadDirectory', args) 329 d = self.runCommand(self.cmd) 330 d.addCallback(self.finished).addErrback(self.failed)

331

332 - def finished(self, result):

333 # Subclasses may choose to skip a transfer. In those cases, self.cmd 334 # will be None, and we should just let BuildStep.finished() handle 335 # the rest 336 if result == SKIPPED: 337 return BuildStep.finished(self, SKIPPED) 338 if self.cmd.stderr != '': 339 self.addCompleteLog('stderr', self.cmd.stderr) 340 341 if self.cmd.rc is None or self.cmd.rc == 0: 342 return BuildStep.finished(self, SUCCESS) 343 return BuildStep.finished(self, FAILURE)

344 345 346 347

348 -class _FileReader(pb.Referenceable):

349 """ 350 Helper class that acts as a file-object with read access 351 """ 352

353 - def __init__(self, fp):

354 self.fp = fp

355

356 - def remote_read(self, maxlength):

357 """ 358 Called from remote slave to read at most L{maxlength} bytes of data 359 360 @type maxlength: C{integer} 361 @param maxlength: Maximum number of data bytes that can be returned 362 363 @return: Data read from L{fp} 364 @rtype: C{string} of bytes read from file 365 """ 366 if self.fp is None: 367 return '' 368 369 data = self.fp.read(maxlength) 370 return data

371

372 - def remote_close(self):

373 """ 374 Called by remote slave to state that no more data will be transfered 375 """ 376 if self.fp is not None: 377 self.fp.close() 378 self.fp = None

379 380

381 -class FileDownload(_TransferBuildStep):

382 """ 383 Download the first 'maxsize' bytes of a file, from the buildmaster to the 384 buildslave. Set the mode of the file 385 386 Arguments:: 387 388 ['mastersrc'] filename of source file at master 389 ['slavedest'] filename of destination file at slave 390 ['workdir'] string with slave working directory relative to builder 391 base dir, default 'build' 392 ['maxsize'] maximum size of the file, default None (=unlimited) 393 ['blocksize'] maximum size of each block being transfered 394 ['mode'] use this to set the access permissions of the resulting 395 buildslave-side file. This is traditionally an octal 396 integer, like 0644 to be world-readable (but not 397 world-writable), or 0600 to only be readable by 398 the buildslave account, or 0755 to be world-executable. 399 The default (=None) is to leave it up to the umask of 400 the buildslave process. 401 402 """ 403 name = 'download' 404

405 - def __init__(self, mastersrc, slavedest, 406 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 407 **buildstep_kwargs):

408 BuildStep.__init__(self, **buildstep_kwargs) 409 self.addFactoryArguments(mastersrc=mastersrc, 410 slavedest=slavedest, 411 workdir=workdir, 412 maxsize=maxsize, 413 blocksize=blocksize, 414 mode=mode, 415 ) 416 417 self.mastersrc = mastersrc 418 self.slavedest = slavedest 419 self.workdir = workdir 420 self.maxsize = maxsize 421 self.blocksize = blocksize 422 assert isinstance(mode, (int, type(None))) 423 self.mode = mode

424

425 - def start(self):

426 properties = self.build.getProperties() 427 428 version = self.slaveVersion("downloadFile") 429 if not version: 430 m = "slave is too old, does not know about downloadFile" 431 raise BuildSlaveTooOldError(m) 432 433 # we are currently in the buildmaster's basedir, so any non-absolute 434 # paths will be interpreted relative to that 435 source = os.path.expanduser(properties.render(self.mastersrc)) 436 slavedest = properties.render(self.slavedest) 437 log.msg("FileDownload started, from master %r to slave %r" % 438 (source, slavedest)) 439 440 self.step_status.setText(['downloading', "to", 441 os.path.basename(slavedest)]) 442 443 # setup structures for reading the file 444 try: 445 fp = open(source, 'rb') 446 except IOError: 447 # if file does not exist, bail out with an error 448 self.addCompleteLog('stderr', 449 'File %r not available at master' % source) 450 # TODO: once BuildStep.start() gets rewritten to use 451 # maybeDeferred, just re-raise the exception here. 452 reactor.callLater(0, BuildStep.finished, self, FAILURE) 453 return 454 fileReader = _FileReader(fp) 455 456 # default arguments 457 args = { 458 'slavedest': slavedest, 459 'maxsize': self.maxsize, 460 'reader': fileReader, 461 'blocksize': self.blocksize, 462 'workdir': self._getWorkdir(), 463 'mode': self.mode, 464 } 465 466 self.cmd = StatusRemoteCommand('downloadFile', args) 467 d = self.runCommand(self.cmd) 468 d.addCallback(self.finished).addErrback(self.failed)

469

Source Code for Module buildbot.steps.transfer