buildbot.steps.transfer

1 # -*- test-case-name: buildbot.test.test_transfer -*- 2 3 import os.path, tarfile, tempfile 4 from twisted.internet import reactor 5 from twisted.spread import pb 6 from twisted.python import log 7 from buildbot.process.buildstep import RemoteCommand, BuildStep 8 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 9 from buildbot.interfaces import BuildSlaveTooOldError 10 11

12 -class _FileWriter(pb.Referenceable):

13 """ 14 Helper class that acts as a file-object with write access 15 """ 16

17 - def __init__(self, destfile, maxsize, mode):

18 # Create missing directories. 19 destfile = os.path.abspath(destfile) 20 dirname = os.path.dirname(destfile) 21 if not os.path.exists(dirname): 22 os.makedirs(dirname) 23 24 self.destfile = destfile 25 self.fp = open(destfile, "wb") 26 if mode is not None: 27 os.chmod(destfile, mode) 28 self.remaining = maxsize

29

30 - def remote_write(self, data):

31 """ 32 Called from remote slave to write L{data} to L{fp} within boundaries 33 of L{maxsize} 34 35 @type data: C{string} 36 @param data: String of data to write 37 """ 38 if self.remaining is not None: 39 if len(data) > self.remaining: 40 data = data[:self.remaining] 41 self.fp.write(data) 42 self.remaining = self.remaining - len(data) 43 else: 44 self.fp.write(data)

45

46 - def remote_close(self):

47 """ 48 Called by remote slave to state that no more data will be transfered 49 """ 50 self.fp.close() 51 self.fp = None

52

53 - def __del__(self):

54 # unclean shutdown, the file is probably truncated, so delete it 55 # altogether rather than deliver a corrupted file 56 fp = getattr(self, "fp", None) 57 if fp: 58 fp.close() 59 os.unlink(self.destfile)

60 61

62 -def _extractall(self, path=".", members=None):

63 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 64 65 import copy 66 67 directories = [] 68 69 if members is None: 70 members = self 71 72 for tarinfo in members: 73 if tarinfo.isdir(): 74 # Extract directories with a safe mode. 75 directories.append(tarinfo) 76 tarinfo = copy.copy(tarinfo) 77 tarinfo.mode = 0700 78 self.extract(tarinfo, path) 79 80 # Reverse sort directories. 81 directories.sort(lambda a, b: cmp(a.name, b.name)) 82 directories.reverse() 83 84 # Set correct owner, mtime and filemode on directories. 85 for tarinfo in directories: 86 dirpath = os.path.join(path, tarinfo.name) 87 try: 88 self.chown(tarinfo, dirpath) 89 self.utime(tarinfo, dirpath) 90 self.chmod(tarinfo, dirpath) 91 except tarfile.ExtractError, e: 92 if self.errorlevel > 1: 93 raise 94 else: 95 self._dbg(1, "tarfile: %s" % e)

96

97 -class _DirectoryWriter(_FileWriter):

98 """ 99 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 100 step to unpack the archive, once the transfer has completed. 101 """ 102

103 - def __init__(self, destroot, maxsize, compress, mode):

104 self.destroot = destroot 105 106 self.fd, self.tarname = tempfile.mkstemp() 107 self.compress = compress 108 _FileWriter.__init__(self, self.tarname, maxsize, mode)

109

110 - def remote_unpack(self):

111 """ 112 Called by remote slave to state that no more data will be transfered 113 """ 114 if self.fp: 115 self.fp.close() 116 self.fp = None 117 fileobj = os.fdopen(self.fd, 'rb') 118 if self.compress == 'bz2': 119 mode='r|bz2' 120 elif self.compress == 'gz': 121 mode='r|gz' 122 else: 123 mode = 'r' 124 if not hasattr(tarfile.TarFile, 'extractall'): 125 tarfile.TarFile.extractall = _extractall 126 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) 127 archive.extractall(path=self.destroot) 128 archive.close() 129 fileobj.close() 130 os.remove(self.tarname)

131 132

133 -class StatusRemoteCommand(RemoteCommand):

134 - def __init__(self, remote_command, args):

135 RemoteCommand.__init__(self, remote_command, args) 136 137 self.rc = None 138 self.stderr = ''

139

140 - def remoteUpdate(self, update):

141 #log.msg('StatusRemoteCommand: update=%r' % update) 142 if 'rc' in update: 143 self.rc = update['rc'] 144 if 'stderr' in update: 145 self.stderr = self.stderr + update['stderr'] + '\n'

146

147 -class _TransferBuildStep(BuildStep):

148 """ 149 Base class for FileUpload and FileDownload to factor out common 150 functionality. 151 """ 152 DEFAULT_WORKDIR = "build" # is this redundant? 153

154 - def setDefaultWorkdir(self, workdir):

155 if self.workdir is None: 156 self.workdir = workdir

157

158 - def _getWorkdir(self):

159 properties = self.build.getProperties() 160 if self.workdir is None: 161 workdir = self.DEFAULT_WORKDIR 162 else: 163 workdir = self.workdir 164 return properties.render(workdir)

165

166 - def finished(self, result):

167 # Subclasses may choose to skip a transfer. In those cases, self.cmd 168 # will be None, and we should just let BuildStep.finished() handle 169 # the rest 170 if result == SKIPPED: 171 return BuildStep.finished(self, SKIPPED) 172 if self.cmd.stderr != '': 173 self.addCompleteLog('stderr', self.cmd.stderr) 174 175 if self.cmd.rc is None or self.cmd.rc == 0: 176 return BuildStep.finished(self, SUCCESS) 177 return BuildStep.finished(self, FAILURE)

178 179

180 -class FileUpload(_TransferBuildStep):

181 """ 182 Build step to transfer a file from the slave to the master. 183 184 arguments: 185 186 - ['slavesrc'] filename of source file at slave, relative to workdir 187 - ['masterdest'] filename of destination file at master 188 - ['workdir'] string with slave working directory relative to builder 189 base dir, default 'build' 190 - ['maxsize'] maximum size of the file, default None (=unlimited) 191 - ['blocksize'] maximum size of each block being transfered 192 - ['mode'] file access mode for the resulting master-side file. 193 The default (=None) is to leave it up to the umask of 194 the buildmaster process. 195 196 """ 197 198 name = 'upload' 199

200 - def __init__(self, slavesrc, masterdest, 201 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 202 **buildstep_kwargs):

203 BuildStep.__init__(self, **buildstep_kwargs) 204 self.addFactoryArguments(slavesrc=slavesrc, 205 masterdest=masterdest, 206 workdir=workdir, 207 maxsize=maxsize, 208 blocksize=blocksize, 209 mode=mode, 210 ) 211 212 self.slavesrc = slavesrc 213 self.masterdest = masterdest 214 self.workdir = workdir 215 self.maxsize = maxsize 216 self.blocksize = blocksize 217 assert isinstance(mode, (int, type(None))) 218 self.mode = mode

219

220 - def start(self):

221 version = self.slaveVersion("uploadFile") 222 properties = self.build.getProperties() 223 224 if not version: 225 m = "slave is too old, does not know about uploadFile" 226 raise BuildSlaveTooOldError(m) 227 228 source = properties.render(self.slavesrc) 229 masterdest = properties.render(self.masterdest) 230 # we rely upon the fact that the buildmaster runs chdir'ed into its 231 # basedir to make sure that relative paths in masterdest are expanded 232 # properly. TODO: maybe pass the master's basedir all the way down 233 # into the BuildStep so we can do this better. 234 masterdest = os.path.expanduser(masterdest) 235 log.msg("FileUpload started, from slave %r to master %r" 236 % (source, masterdest)) 237 238 self.step_status.setText(['uploading', os.path.basename(source)]) 239 240 # we use maxsize to limit the amount of data on both sides 241 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 242 243 # default arguments 244 args = { 245 'slavesrc': source, 246 'workdir': self._getWorkdir(), 247 'writer': fileWriter, 248 'maxsize': self.maxsize, 249 'blocksize': self.blocksize, 250 } 251 252 self.cmd = StatusRemoteCommand('uploadFile', args) 253 d = self.runCommand(self.cmd) 254 d.addCallback(self.finished).addErrback(self.failed)

255 256

257 -class DirectoryUpload(BuildStep):

258 """ 259 Build step to transfer a directory from the slave to the master. 260 261 arguments: 262 263 - ['slavesrc'] name of source directory at slave, relative to workdir 264 - ['masterdest'] name of destination directory at master 265 - ['workdir'] string with slave working directory relative to builder 266 base dir, default 'build' 267 - ['maxsize'] maximum size of the compressed tarfile containing the 268 whole directory 269 - ['blocksize'] maximum size of each block being transfered 270 - ['compress'] compression type to use: one of [None, 'gz', 'bz2'] 271 272 """ 273 274 name = 'upload' 275

276 - def __init__(self, slavesrc, masterdest, 277 workdir="build", maxsize=None, blocksize=16*1024, 278 compress=None, **buildstep_kwargs):

279 BuildStep.__init__(self, **buildstep_kwargs) 280 self.addFactoryArguments(slavesrc=slavesrc, 281 masterdest=masterdest, 282 workdir=workdir, 283 maxsize=maxsize, 284 blocksize=blocksize, 285 compress=compress, 286 ) 287 288 self.slavesrc = slavesrc 289 self.masterdest = masterdest 290 self.workdir = workdir 291 self.maxsize = maxsize 292 self.blocksize = blocksize 293 assert compress in (None, 'gz', 'bz2') 294 self.compress = compress

295

296 - def start(self):

297 version = self.slaveVersion("uploadDirectory") 298 properties = self.build.getProperties() 299 300 if not version: 301 m = "slave is too old, does not know about uploadDirectory" 302 raise BuildSlaveTooOldError(m) 303 304 source = properties.render(self.slavesrc) 305 masterdest = properties.render(self.masterdest) 306 # we rely upon the fact that the buildmaster runs chdir'ed into its 307 # basedir to make sure that relative paths in masterdest are expanded 308 # properly. TODO: maybe pass the master's basedir all the way down 309 # into the BuildStep so we can do this better. 310 masterdest = os.path.expanduser(masterdest) 311 log.msg("DirectoryUpload started, from slave %r to master %r" 312 % (source, masterdest)) 313 314 self.step_status.setText(['uploading', os.path.basename(source)]) 315 316 # we use maxsize to limit the amount of data on both sides 317 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600) 318 319 # default arguments 320 args = { 321 'slavesrc': source, 322 'workdir': self.workdir, 323 'writer': dirWriter, 324 'maxsize': self.maxsize, 325 'blocksize': self.blocksize, 326 'compress': self.compress 327 } 328 329 self.cmd = StatusRemoteCommand('uploadDirectory', args) 330 d = self.runCommand(self.cmd) 331 d.addCallback(self.finished).addErrback(self.failed)

332

333 - def finished(self, result):

334 # Subclasses may choose to skip a transfer. In those cases, self.cmd 335 # will be None, and we should just let BuildStep.finished() handle 336 # the rest 337 if result == SKIPPED: 338 return BuildStep.finished(self, SKIPPED) 339 if self.cmd.stderr != '': 340 self.addCompleteLog('stderr', self.cmd.stderr) 341 342 if self.cmd.rc is None or self.cmd.rc == 0: 343 return BuildStep.finished(self, SUCCESS) 344 return BuildStep.finished(self, FAILURE)

345 346 347 348

349 -class _FileReader(pb.Referenceable):

350 """ 351 Helper class that acts as a file-object with read access 352 """ 353

354 - def __init__(self, fp):

355 self.fp = fp

356

357 - def remote_read(self, maxlength):

358 """ 359 Called from remote slave to read at most L{maxlength} bytes of data 360 361 @type maxlength: C{integer} 362 @param maxlength: Maximum number of data bytes that can be returned 363 364 @return: Data read from L{fp} 365 @rtype: C{string} of bytes read from file 366 """ 367 if self.fp is None: 368 return '' 369 370 data = self.fp.read(maxlength) 371 return data

372

373 - def remote_close(self):

374 """ 375 Called by remote slave to state that no more data will be transfered 376 """ 377 if self.fp is not None: 378 self.fp.close() 379 self.fp = None

380 381

382 -class FileDownload(_TransferBuildStep):

383 """ 384 Download the first 'maxsize' bytes of a file, from the buildmaster to the 385 buildslave. Set the mode of the file 386 387 Arguments:: 388 389 ['mastersrc'] filename of source file at master 390 ['slavedest'] filename of destination file at slave 391 ['workdir'] string with slave working directory relative to builder 392 base dir, default 'build' 393 ['maxsize'] maximum size of the file, default None (=unlimited) 394 ['blocksize'] maximum size of each block being transfered 395 ['mode'] use this to set the access permissions of the resulting 396 buildslave-side file. This is traditionally an octal 397 integer, like 0644 to be world-readable (but not 398 world-writable), or 0600 to only be readable by 399 the buildslave account, or 0755 to be world-executable. 400 The default (=None) is to leave it up to the umask of 401 the buildslave process. 402 403 """ 404 name = 'download' 405

406 - def __init__(self, mastersrc, slavedest, 407 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 408 **buildstep_kwargs):

409 BuildStep.__init__(self, **buildstep_kwargs) 410 self.addFactoryArguments(mastersrc=mastersrc, 411 slavedest=slavedest, 412 workdir=workdir, 413 maxsize=maxsize, 414 blocksize=blocksize, 415 mode=mode, 416 ) 417 418 self.mastersrc = mastersrc 419 self.slavedest = slavedest 420 self.workdir = workdir 421 self.maxsize = maxsize 422 self.blocksize = blocksize 423 assert isinstance(mode, (int, type(None))) 424 self.mode = mode

425

426 - def start(self):

427 properties = self.build.getProperties() 428 429 version = self.slaveVersion("downloadFile") 430 if not version: 431 m = "slave is too old, does not know about downloadFile" 432 raise BuildSlaveTooOldError(m) 433 434 # we are currently in the buildmaster's basedir, so any non-absolute 435 # paths will be interpreted relative to that 436 source = os.path.expanduser(properties.render(self.mastersrc)) 437 slavedest = properties.render(self.slavedest) 438 log.msg("FileDownload started, from master %r to slave %r" % 439 (source, slavedest)) 440 441 self.step_status.setText(['downloading', "to", 442 os.path.basename(slavedest)]) 443 444 # setup structures for reading the file 445 try: 446 fp = open(source, 'rb') 447 except IOError: 448 # if file does not exist, bail out with an error 449 self.addCompleteLog('stderr', 450 'File %r not available at master' % source) 451 # TODO: once BuildStep.start() gets rewritten to use 452 # maybeDeferred, just re-raise the exception here. 453 reactor.callLater(0, BuildStep.finished, self, FAILURE) 454 return 455 fileReader = _FileReader(fp) 456 457 # default arguments 458 args = { 459 'slavedest': slavedest, 460 'maxsize': self.maxsize, 461 'reader': fileReader, 462 'blocksize': self.blocksize, 463 'workdir': self._getWorkdir(), 464 'mode': self.mode, 465 } 466 467 self.cmd = StatusRemoteCommand('downloadFile', args) 468 d = self.runCommand(self.cmd) 469 d.addCallback(self.finished).addErrback(self.failed)

470

Source Code for Module buildbot.steps.transfer