1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 import os, tarfile, tempfile
17
18 from twisted.python import log
19 from twisted.internet import defer
20
21 from buildslave.commands.base import Command
22
24
26 if self.debug:
27 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc))
28
29
30
31 upd = {'rc': self.rc}
32 if self.stderr:
33 upd['stderr'] = self.stderr
34 self.builder.sendUpdate(upd)
35 return res
36
44
45
46
47
49 """
50 Upload a file from slave to build master
51 Arguments:
52
53 - ['workdir']: base directory to use
54 - ['slavesrc']: name of the slave-side file to read from
55 - ['writer']: RemoteReference to a transfer._FileWriter object
56 - ['maxsize']: max size (in bytes) of file to write
57 - ['blocksize']: max size for each data block
58 - ['keepstamp']: whether to preserve file modified and accessed times
59 """
60 debug = False
61
63 self.workdir = args['workdir']
64 self.filename = args['slavesrc']
65 self.writer = args['writer']
66 self.remaining = args['maxsize']
67 self.blocksize = args['blocksize']
68 self.keepstamp = args.get('keepstamp', False)
69 self.stderr = None
70 self.rc = 0
71
73 if self.debug:
74 log.msg('SlaveFileUploadCommand started')
75
76
77 self.path = os.path.join(self.builder.basedir,
78 self.workdir,
79 os.path.expanduser(self.filename))
80 accessed_modified = None
81 try:
82 if self.keepstamp:
83 accessed_modified = (os.path.getatime(self.path),
84 os.path.getmtime(self.path))
85
86 self.fp = open(self.path, 'rb')
87 if self.debug:
88 log.msg("Opened '%s' for upload" % self.path)
89 except:
90 self.fp = None
91 self.stderr = "Cannot open file '%s' for upload" % self.path
92 self.rc = 1
93 if self.debug:
94 log.msg("Cannot open file '%s' for upload" % self.path)
95
96 self.sendStatus({'header': "sending %s" % self.path})
97
98 d = defer.Deferred()
99 self._reactor.callLater(0, self._loop, d)
100 def _close_ok(res):
101 self.fp = None
102 d1 = self.writer.callRemote("close")
103 def _utime_ok(res):
104 return self.writer.callRemote("utime", accessed_modified)
105 if self.keepstamp:
106 d1.addCallback(_utime_ok)
107 return d1
108 def _close_err(f):
109 self.fp = None
110
111 d1 = self.writer.callRemote("close")
112 def eb(f2):
113 log.msg("ignoring error from remote close():")
114 log.err(f2)
115 d1.addErrback(eb)
116 d1.addBoth(lambda _ : f)
117 return d1
118
119 d.addCallbacks(_close_ok, _close_err)
120 d.addBoth(self.finished)
121 return d
122
123 - def _loop(self, fire_when_done):
124 d = defer.maybeDeferred(self._writeBlock)
125 def _done(finished):
126 if finished:
127 fire_when_done.callback(None)
128 else:
129 self._loop(fire_when_done)
130 def _err(why):
131 fire_when_done.errback(why)
132 d.addCallbacks(_done, _err)
133 return None
134
136 """Write a block of data to the remote writer"""
137
138 if self.interrupted or self.fp is None:
139 if self.debug:
140 log.msg('SlaveFileUploadCommand._writeBlock(): end')
141 return True
142
143 length = self.blocksize
144 if self.remaining is not None and length > self.remaining:
145 length = self.remaining
146
147 if length <= 0:
148 if self.stderr is None:
149 self.stderr = 'Maximum filesize reached, truncating file \'%s\'' \
150 % self.path
151 self.rc = 1
152 data = ''
153 else:
154 data = self.fp.read(length)
155
156 if self.debug:
157 log.msg('SlaveFileUploadCommand._writeBlock(): '+
158 'allowed=%d readlen=%d' % (length, len(data)))
159 if len(data) == 0:
160 log.msg("EOF: callRemote(close)")
161 return True
162
163 if self.remaining is not None:
164 self.remaining = self.remaining - len(data)
165 assert self.remaining >= 0
166 d = self.writer.callRemote('write', data)
167 d.addCallback(lambda res: False)
168 return d
169
170
172 """
173 Upload a directory from slave to build master
174 Arguments:
175
176 - ['workdir']: base directory to use
177 - ['slavesrc']: name of the slave-side directory to read from
178 - ['writer']: RemoteReference to a transfer._DirectoryWriter object
179 - ['maxsize']: max size (in bytes) of file to write
180 - ['blocksize']: max size for each data block
181 - ['compress']: one of [None, 'bz2', 'gz']
182 """
183 debug = False
184
186 self.workdir = args['workdir']
187 self.dirname = args['slavesrc']
188 self.writer = args['writer']
189 self.remaining = args['maxsize']
190 self.blocksize = args['blocksize']
191 self.compress = args['compress']
192 self.stderr = None
193 self.rc = 0
194
196 if self.debug:
197 log.msg('SlaveDirectoryUploadCommand started')
198
199 self.path = os.path.join(self.builder.basedir,
200 self.workdir,
201 os.path.expanduser(self.dirname))
202 if self.debug:
203 log.msg("path: %r" % self.path)
204
205
206 fd, self.tarname = tempfile.mkstemp()
207 fileobj = os.fdopen(fd, 'w')
208 if self.compress == 'bz2':
209 mode='w|bz2'
210 elif self.compress == 'gz':
211 mode='w|gz'
212 else:
213 mode = 'w'
214 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj)
215 archive.add(self.path, '')
216 archive.close()
217 fileobj.close()
218
219
220 self.fp = open(self.tarname, 'rb')
221
222 self.sendStatus({'header': "sending %s" % self.path})
223
224 d = defer.Deferred()
225 self._reactor.callLater(0, self._loop, d)
226 def unpack(res):
227
228 d1 = self.writer.callRemote("unpack")
229 d1.addErrback(log.err)
230 d1.addCallback(lambda ignored: res)
231 return d1
232 d.addCallback(unpack)
233 d.addBoth(self.finished)
234 return d
235
240
241
243 """
244 Download a file from master to slave
245 Arguments:
246
247 - ['workdir']: base directory to use
248 - ['slavedest']: name of the slave-side file to be created
249 - ['reader']: RemoteReference to a transfer._FileReader object
250 - ['maxsize']: max size (in bytes) of file to write
251 - ['blocksize']: max size for each data block
252 - ['mode']: access mode for the new file
253 """
254 debug = False
255
257 self.workdir = args['workdir']
258 self.filename = args['slavedest']
259 self.reader = args['reader']
260 self.bytes_remaining = args['maxsize']
261 self.blocksize = args['blocksize']
262 self.mode = args['mode']
263 self.stderr = None
264 self.rc = 0
265
267 if self.debug:
268 log.msg('SlaveFileDownloadCommand starting')
269
270
271 self.path = os.path.join(self.builder.basedir,
272 self.workdir,
273 os.path.expanduser(self.filename))
274
275 dirname = os.path.dirname(self.path)
276 if not os.path.exists(dirname):
277 os.makedirs(dirname)
278
279 try:
280 self.fp = open(self.path, 'wb')
281 if self.debug:
282 log.msg("Opened '%s' for download" % self.path)
283 if self.mode is not None:
284
285
286
287
288
289
290
291 os.chmod(self.path, self.mode)
292 except IOError:
293
294 self.fp = None
295 self.stderr = "Cannot open file '%s' for download" % self.path
296 self.rc = 1
297 if self.debug:
298 log.msg("Cannot open file '%s' for download" % self.path)
299
300 d = defer.Deferred()
301 self._reactor.callLater(0, self._loop, d)
302 def _close(res):
303
304 d1 = self.reader.callRemote('close')
305 d1.addErrback(log.err, 'while trying to close reader')
306 d1.addCallback(lambda ignored: res)
307 return d1
308 d.addBoth(_close)
309 d.addBoth(self.finished)
310 return d
311
312 - def _loop(self, fire_when_done):
313 d = defer.maybeDeferred(self._readBlock)
314 def _done(finished):
315 if finished:
316 fire_when_done.callback(None)
317 else:
318 self._loop(fire_when_done)
319 def _err(why):
320 fire_when_done.errback(why)
321 d.addCallbacks(_done, _err)
322 return None
323
325 """Read a block of data from the remote reader."""
326
327 if self.interrupted or self.fp is None:
328 if self.debug:
329 log.msg('SlaveFileDownloadCommand._readBlock(): end')
330 return True
331
332 length = self.blocksize
333 if self.bytes_remaining is not None and length > self.bytes_remaining:
334 length = self.bytes_remaining
335
336 if length <= 0:
337 if self.stderr is None:
338 self.stderr = "Maximum filesize reached, truncating file '%s'" \
339 % self.path
340 self.rc = 1
341 return True
342 else:
343 d = self.reader.callRemote('read', length)
344 d.addCallback(self._writeData)
345 return d
346
348 if self.debug:
349 log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
350 len(data))
351 if len(data) == 0:
352 return True
353
354 if self.bytes_remaining is not None:
355 self.bytes_remaining = self.bytes_remaining - len(data)
356 assert self.bytes_remaining >= 0
357 self.fp.write(data)
358 return False
359
365