1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 import os.path, tarfile, tempfile
18 try:
19 from cStringIO import StringIO
20 assert StringIO
21 except ImportError:
22 from StringIO import StringIO
23 from twisted.internet import reactor
24 from twisted.spread import pb
25 from twisted.python import log
26 from buildbot.process.buildstep import RemoteCommand, BuildStep
27 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED
28 from buildbot.interfaces import BuildSlaveTooOldError
29 from buildbot.util import json
30
31
33 """
34 Helper class that acts as a file-object with write access
35 """
36
37 - def __init__(self, destfile, maxsize, mode):
38
39 destfile = os.path.abspath(destfile)
40 dirname = os.path.dirname(destfile)
41 if not os.path.exists(dirname):
42 os.makedirs(dirname)
43
44 self.destfile = destfile
45 self.mode = mode
46 fd, self.tmpname = tempfile.mkstemp(dir=dirname)
47 self.fp = os.fdopen(fd, 'wb')
48 self.remaining = maxsize
49
51 """
52 Called from remote slave to write L{data} to L{fp} within boundaries
53 of L{maxsize}
54
55 @type data: C{string}
56 @param data: String of data to write
57 """
58 if self.remaining is not None:
59 if len(data) > self.remaining:
60 data = data[:self.remaining]
61 self.fp.write(data)
62 self.remaining = self.remaining - len(data)
63 else:
64 self.fp.write(data)
65
67 os.utime(self.destfile,accessed_modified)
68
70 """
71 Called by remote slave to state that no more data will be transfered
72 """
73 self.fp.close()
74 self.fp = None
75
76 if os.path.exists(self.destfile):
77 os.unlink(self.destfile)
78 os.rename(self.tmpname, self.destfile)
79 self.tmpname = None
80 if self.mode is not None:
81 os.chmod(self.destfile, self.mode)
82
84
85
86 fp = getattr(self, "fp", None)
87 if fp:
88 fp.close()
89 os.unlink(self.destfile)
90 if self.tmpname and os.path.exists(self.tmpname):
91 os.unlink(self.tmpname)
92
93
95 """Fallback extractall method for TarFile, in case it doesn't have its own."""
96
97 import copy
98
99 directories = []
100
101 if members is None:
102 members = self
103
104 for tarinfo in members:
105 if tarinfo.isdir():
106
107 directories.append(tarinfo)
108 tarinfo = copy.copy(tarinfo)
109 tarinfo.mode = 0700
110 self.extract(tarinfo, path)
111
112
113 directories.sort(lambda a, b: cmp(a.name, b.name))
114 directories.reverse()
115
116
117 for tarinfo in directories:
118 dirpath = os.path.join(path, tarinfo.name)
119 try:
120 self.chown(tarinfo, dirpath)
121 self.utime(tarinfo, dirpath)
122 self.chmod(tarinfo, dirpath)
123 except tarfile.ExtractError, e:
124 if self.errorlevel > 1:
125 raise
126 else:
127 self._dbg(1, "tarfile: %s" % e)
128
130 """
131 A DirectoryWriter is implemented as a FileWriter, with an added post-processing
132 step to unpack the archive, once the transfer has completed.
133 """
134
135 - def __init__(self, destroot, maxsize, compress, mode):
136 self.destroot = destroot
137 self.compress = compress
138
139 self.fd, self.tarname = tempfile.mkstemp()
140 os.close(self.fd)
141
142 _FileWriter.__init__(self, self.tarname, maxsize, mode)
143
145 """
146 Called by remote slave to state that no more data will be transfered
147 """
148
149 self.remote_close()
150
151
152 if self.compress == 'bz2':
153 mode='r|bz2'
154 elif self.compress == 'gz':
155 mode='r|gz'
156 else:
157 mode = 'r'
158
159
160 if not hasattr(tarfile.TarFile, 'extractall'):
161 tarfile.TarFile.extractall = _extractall
162
163
164 archive = tarfile.open(name=self.tarname, mode=mode)
165 archive.extractall(path=self.destroot)
166 archive.close()
167 os.remove(self.tarname)
168
169
171 - def __init__(self, remote_command, args):
176
178
179 if 'rc' in update:
180 self.rc = update['rc']
181 if 'stderr' in update:
182 self.stderr = self.stderr + update['stderr'] + '\n'
183
185 """
186 Base class for FileUpload and FileDownload to factor out common
187 functionality.
188 """
189 DEFAULT_WORKDIR = "build"
190
191 renderables = [ 'workdir' ]
192
193 haltOnFailure = True
194 flunkOnFailure = True
195
199
206
212
225
226
228 """
229 Build step to transfer a file from the slave to the master.
230
231 arguments:
232
233 - ['slavesrc'] filename of source file at slave, relative to workdir
234 - ['masterdest'] filename of destination file at master
235 - ['workdir'] string with slave working directory relative to builder
236 base dir, default 'build'
237 - ['maxsize'] maximum size of the file, default None (=unlimited)
238 - ['blocksize'] maximum size of each block being transfered
239 - ['mode'] file access mode for the resulting master-side file.
240 The default (=None) is to leave it up to the umask of
241 the buildmaster process.
242 - ['keepstamp'] whether to preserve file modified and accessed times
243
244 """
245
246 name = 'upload'
247
248 renderables = [ 'slavesrc', 'masterdest' ]
249
250 - def __init__(self, slavesrc, masterdest,
251 workdir=None, maxsize=None, blocksize=16*1024, mode=None, keepstamp=False,
252 **buildstep_kwargs):
253 BuildStep.__init__(self, **buildstep_kwargs)
254 self.addFactoryArguments(slavesrc=slavesrc,
255 masterdest=masterdest,
256 workdir=workdir,
257 maxsize=maxsize,
258 blocksize=blocksize,
259 mode=mode,
260 keepstamp=keepstamp,
261 )
262
263 self.slavesrc = slavesrc
264 self.masterdest = masterdest
265 self.workdir = workdir
266 self.maxsize = maxsize
267 self.blocksize = blocksize
268 assert isinstance(mode, (int, type(None)))
269 self.mode = mode
270 self.keepstamp = keepstamp
271
273 version = self.slaveVersion("uploadFile")
274
275 if not version:
276 m = "slave is too old, does not know about uploadFile"
277 raise BuildSlaveTooOldError(m)
278
279 source = self.slavesrc
280 masterdest = self.masterdest
281
282
283
284
285 masterdest = os.path.expanduser(masterdest)
286 log.msg("FileUpload started, from slave %r to master %r"
287 % (source, masterdest))
288
289 self.step_status.setText(['uploading', os.path.basename(source)])
290
291
292 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode)
293
294 if self.keepstamp and self.slaveVersionIsOlderThan("uploadFile","2.13"):
295 m = ("This buildslave (%s) does not support preserving timestamps. "
296 "Please upgrade the buildslave." % self.build.slavename )
297 raise BuildSlaveTooOldError(m)
298
299
300 args = {
301 'slavesrc': source,
302 'workdir': self._getWorkdir(),
303 'writer': fileWriter,
304 'maxsize': self.maxsize,
305 'blocksize': self.blocksize,
306 'keepstamp': self.keepstamp,
307 }
308
309 self.cmd = StatusRemoteCommand('uploadFile', args)
310 d = self.runCommand(self.cmd)
311 d.addCallback(self.finished).addErrback(self.failed)
312
313
315 """
316 Build step to transfer a directory from the slave to the master.
317
318 arguments:
319
320 - ['slavesrc'] name of source directory at slave, relative to workdir
321 - ['masterdest'] name of destination directory at master
322 - ['workdir'] string with slave working directory relative to builder
323 base dir, default 'build'
324 - ['maxsize'] maximum size of the compressed tarfile containing the
325 whole directory
326 - ['blocksize'] maximum size of each block being transfered
327 - ['compress'] compression type to use: one of [None, 'gz', 'bz2']
328
329 """
330
331 name = 'upload'
332
333 renderables = [ 'slavesrc', 'masterdest' ]
334
335 - def __init__(self, slavesrc, masterdest,
336 workdir="build", maxsize=None, blocksize=16*1024,
337 compress=None, **buildstep_kwargs):
338 BuildStep.__init__(self, **buildstep_kwargs)
339 self.addFactoryArguments(slavesrc=slavesrc,
340 masterdest=masterdest,
341 workdir=workdir,
342 maxsize=maxsize,
343 blocksize=blocksize,
344 compress=compress,
345 )
346
347 self.slavesrc = slavesrc
348 self.masterdest = masterdest
349 self.workdir = workdir
350 self.maxsize = maxsize
351 self.blocksize = blocksize
352 assert compress in (None, 'gz', 'bz2')
353 self.compress = compress
354
356 version = self.slaveVersion("uploadDirectory")
357
358 if not version:
359 m = "slave is too old, does not know about uploadDirectory"
360 raise BuildSlaveTooOldError(m)
361
362 source = self.slavesrc
363 masterdest = self.masterdest
364
365
366
367
368 masterdest = os.path.expanduser(masterdest)
369 log.msg("DirectoryUpload started, from slave %r to master %r"
370 % (source, masterdest))
371
372 self.step_status.setText(['uploading', os.path.basename(source)])
373
374
375 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 0600)
376
377
378 args = {
379 'slavesrc': source,
380 'workdir': self.workdir,
381 'writer': dirWriter,
382 'maxsize': self.maxsize,
383 'blocksize': self.blocksize,
384 'compress': self.compress
385 }
386
387 self.cmd = StatusRemoteCommand('uploadDirectory', args)
388 d = self.runCommand(self.cmd)
389 d.addCallback(self.finished).addErrback(self.failed)
390
403
404
405
406
408 """
409 Helper class that acts as a file-object with read access
410 """
411
414
416 """
417 Called from remote slave to read at most L{maxlength} bytes of data
418
419 @type maxlength: C{integer}
420 @param maxlength: Maximum number of data bytes that can be returned
421
422 @return: Data read from L{fp}
423 @rtype: C{string} of bytes read from file
424 """
425 if self.fp is None:
426 return ''
427
428 data = self.fp.read(maxlength)
429 return data
430
432 """
433 Called by remote slave to state that no more data will be transfered
434 """
435 if self.fp is not None:
436 self.fp.close()
437 self.fp = None
438
439
441 """
442 Download the first 'maxsize' bytes of a file, from the buildmaster to the
443 buildslave. Set the mode of the file
444
445 Arguments::
446
447 ['mastersrc'] filename of source file at master
448 ['slavedest'] filename of destination file at slave
449 ['workdir'] string with slave working directory relative to builder
450 base dir, default 'build'
451 ['maxsize'] maximum size of the file, default None (=unlimited)
452 ['blocksize'] maximum size of each block being transfered
453 ['mode'] use this to set the access permissions of the resulting
454 buildslave-side file. This is traditionally an octal
455 integer, like 0644 to be world-readable (but not
456 world-writable), or 0600 to only be readable by
457 the buildslave account, or 0755 to be world-executable.
458 The default (=None) is to leave it up to the umask of
459 the buildslave process.
460
461 """
462 name = 'download'
463
464 renderables = [ 'mastersrc', 'slavedest' ]
465
466 - def __init__(self, mastersrc, slavedest,
467 workdir=None, maxsize=None, blocksize=16*1024, mode=None,
468 **buildstep_kwargs):
469 BuildStep.__init__(self, **buildstep_kwargs)
470 self.addFactoryArguments(mastersrc=mastersrc,
471 slavedest=slavedest,
472 workdir=workdir,
473 maxsize=maxsize,
474 blocksize=blocksize,
475 mode=mode,
476 )
477
478 self.mastersrc = mastersrc
479 self.slavedest = slavedest
480 self.workdir = workdir
481 self.maxsize = maxsize
482 self.blocksize = blocksize
483 assert isinstance(mode, (int, type(None)))
484 self.mode = mode
485
487 version = self.slaveVersion("downloadFile")
488 if not version:
489 m = "slave is too old, does not know about downloadFile"
490 raise BuildSlaveTooOldError(m)
491
492
493
494 source = os.path.expanduser(self.mastersrc)
495 slavedest = self.slavedest
496 log.msg("FileDownload started, from master %r to slave %r" %
497 (source, slavedest))
498
499 self.step_status.setText(['downloading', "to",
500 os.path.basename(slavedest)])
501
502
503 try:
504 fp = open(source, 'rb')
505 except IOError:
506
507 self.addCompleteLog('stderr',
508 'File %r not available at master' % source)
509
510
511 reactor.callLater(0, BuildStep.finished, self, FAILURE)
512 return
513 fileReader = _FileReader(fp)
514
515
516 args = {
517 'slavedest': slavedest,
518 'maxsize': self.maxsize,
519 'reader': fileReader,
520 'blocksize': self.blocksize,
521 'workdir': self._getWorkdir(),
522 'mode': self.mode,
523 }
524
525 self.cmd = StatusRemoteCommand('downloadFile', args)
526 d = self.runCommand(self.cmd)
527 d.addCallback(self.finished).addErrback(self.failed)
528
530 """
531 Download the first 'maxsize' bytes of a string, from the buildmaster to the
532 buildslave. Set the mode of the file
533
534 Arguments::
535
536 ['s'] string to transfer
537 ['slavedest'] filename of destination file at slave
538 ['workdir'] string with slave working directory relative to builder
539 base dir, default 'build'
540 ['maxsize'] maximum size of the file, default None (=unlimited)
541 ['blocksize'] maximum size of each block being transfered
542 ['mode'] use this to set the access permissions of the resulting
543 buildslave-side file. This is traditionally an octal
544 integer, like 0644 to be world-readable (but not
545 world-writable), or 0600 to only be readable by
546 the buildslave account, or 0755 to be world-executable.
547 The default (=None) is to leave it up to the umask of
548 the buildslave process.
549 """
550 name = 'string_download'
551
552 renderables = [ 'slavedest', 's' ]
553
554 - def __init__(self, s, slavedest,
555 workdir=None, maxsize=None, blocksize=16*1024, mode=None,
556 **buildstep_kwargs):
573
575 version = self.slaveVersion("downloadFile")
576 if not version:
577 m = "slave is too old, does not know about downloadFile"
578 raise BuildSlaveTooOldError(m)
579
580
581
582 slavedest = self.slavedest
583 log.msg("StringDownload started, from master to slave %r" % slavedest)
584
585 self.step_status.setText(['downloading', "to",
586 os.path.basename(slavedest)])
587
588
589 fp = StringIO(self.s)
590 fileReader = _FileReader(fp)
591
592
593 args = {
594 'slavedest': slavedest,
595 'maxsize': self.maxsize,
596 'reader': fileReader,
597 'blocksize': self.blocksize,
598 'workdir': self._getWorkdir(),
599 'mode': self.mode,
600 }
601
602 self.cmd = StatusRemoteCommand('downloadFile', args)
603 d = self.runCommand(self.cmd)
604 d.addCallback(self.finished).addErrback(self.failed)
605
607 """
608 Encode object o as a json string and save it on the buildslave
609
610 Arguments::
611
612 ['o'] object to encode and transfer
613 """
614 name = "json_download"
615 - def __init__(self, o, slavedest, **buildstep_kwargs):
621
623 """
624 Download the current build properties as a json string and save it on the
625 buildslave
626 """
627 name = "json_properties_download"
628 - def __init__(self, slavedest, **buildstep_kwargs):
633
646