1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 import os, tarfile, tempfile
17
18 from twisted.python import log
19 from twisted.internet import defer
20
21 from buildslave.commands.base import Command
22
24
26 if self.debug:
27 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc))
28
29
30
31 upd = {'rc': self.rc}
32 if self.stderr:
33 upd['stderr'] = self.stderr
34 self.builder.sendUpdate(upd)
35 return res
36
44
45
46
47
49 """
50 Upload a file from slave to build master
51 Arguments:
52
53 - ['workdir']: base directory to use
54 - ['slavesrc']: name of the slave-side file to read from
55 - ['writer']: RemoteReference to a transfer._FileWriter object
56 - ['maxsize']: max size (in bytes) of file to write
57 - ['blocksize']: max size for each data block
58 - ['keepstamp']: whether to preserve file modified and accessed times
59 """
60 debug = False
61
63 self.workdir = args['workdir']
64 self.filename = args['slavesrc']
65 self.writer = args['writer']
66 self.remaining = args['maxsize']
67 self.blocksize = args['blocksize']
68 self.keepstamp = args.get('keepstamp', False)
69 self.stderr = None
70 self.rc = 0
71
73 if self.debug:
74 log.msg('SlaveFileUploadCommand started')
75
76
77 self.path = os.path.join(self.builder.basedir,
78 self.workdir,
79 os.path.expanduser(self.filename))
80 accessed_modified = None
81 try:
82 if self.keepstamp:
83 accessed_modified = (os.path.getatime(self.path),
84 os.path.getmtime(self.path))
85
86 self.fp = open(self.path, 'rb')
87 if self.debug:
88 log.msg("Opened '%s' for upload" % self.path)
89 except:
90 self.fp = None
91 self.stderr = "Cannot open file '%s' for upload" % self.path
92 self.rc = 1
93 if self.debug:
94 log.msg("Cannot open file '%s' for upload" % self.path)
95
96 self.sendStatus({'header': "sending %s" % self.path})
97
98 d = defer.Deferred()
99 self._reactor.callLater(0, self._loop, d)
100 def _close_ok(res):
101 self.fp = None
102 d1 = self.writer.callRemote("close")
103 def _utime_ok(res):
104 return self.writer.callRemote("utime", accessed_modified)
105 if self.keepstamp:
106 d1.addCallback(_utime_ok)
107 return d1
108 def _close_err(f):
109 self.rc = 1
110 self.fp = None
111
112 d1 = self.writer.callRemote("close")
113 def eb(f2):
114 log.msg("ignoring error from remote close():")
115 log.err(f2)
116 d1.addErrback(eb)
117 d1.addBoth(lambda _ : f)
118 return d1
119
120 d.addCallbacks(_close_ok, _close_err)
121 d.addBoth(self.finished)
122 return d
123
124 - def _loop(self, fire_when_done):
125 d = defer.maybeDeferred(self._writeBlock)
126 def _done(finished):
127 if finished:
128 fire_when_done.callback(None)
129 else:
130 self._loop(fire_when_done)
131 def _err(why):
132 fire_when_done.errback(why)
133 d.addCallbacks(_done, _err)
134 return None
135
137 """Write a block of data to the remote writer"""
138
139 if self.interrupted or self.fp is None:
140 if self.debug:
141 log.msg('SlaveFileUploadCommand._writeBlock(): end')
142 return True
143
144 length = self.blocksize
145 if self.remaining is not None and length > self.remaining:
146 length = self.remaining
147
148 if length <= 0:
149 if self.stderr is None:
150 self.stderr = 'Maximum filesize reached, truncating file \'%s\'' \
151 % self.path
152 self.rc = 1
153 data = ''
154 else:
155 data = self.fp.read(length)
156
157 if self.debug:
158 log.msg('SlaveFileUploadCommand._writeBlock(): '+
159 'allowed=%d readlen=%d' % (length, len(data)))
160 if len(data) == 0:
161 log.msg("EOF: callRemote(close)")
162 return True
163
164 if self.remaining is not None:
165 self.remaining = self.remaining - len(data)
166 assert self.remaining >= 0
167 d = self.writer.callRemote('write', data)
168 d.addCallback(lambda res: False)
169 return d
170
171
173 debug = False
174
176 self.workdir = args['workdir']
177 self.dirname = args['slavesrc']
178 self.writer = args['writer']
179 self.remaining = args['maxsize']
180 self.blocksize = args['blocksize']
181 self.compress = args['compress']
182 self.stderr = None
183 self.rc = 0
184
186 if self.debug:
187 log.msg('SlaveDirectoryUploadCommand started')
188
189 self.path = os.path.join(self.builder.basedir,
190 self.workdir,
191 os.path.expanduser(self.dirname))
192 if self.debug:
193 log.msg("path: %r" % self.path)
194
195
196 fd, self.tarname = tempfile.mkstemp()
197 fileobj = os.fdopen(fd, 'w')
198 if self.compress == 'bz2':
199 mode='w|bz2'
200 elif self.compress == 'gz':
201 mode='w|gz'
202 else:
203 mode = 'w'
204 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj)
205 archive.add(self.path, '')
206 archive.close()
207 fileobj.close()
208
209
210 self.fp = open(self.tarname, 'rb')
211
212 self.sendStatus({'header': "sending %s" % self.path})
213
214 d = defer.Deferred()
215 self._reactor.callLater(0, self._loop, d)
216 def unpack(res):
217 d1 = self.writer.callRemote("unpack")
218 def unpack_err(f):
219 self.rc = 1
220 return f
221 d1.addErrback(unpack_err)
222 d1.addCallback(lambda ignored: res)
223 return d1
224 d.addCallback(unpack)
225 d.addBoth(self.finished)
226 return d
227
232
233
235 """
236 Download a file from master to slave
237 Arguments:
238
239 - ['workdir']: base directory to use
240 - ['slavedest']: name of the slave-side file to be created
241 - ['reader']: RemoteReference to a transfer._FileReader object
242 - ['maxsize']: max size (in bytes) of file to write
243 - ['blocksize']: max size for each data block
244 - ['mode']: access mode for the new file
245 """
246 debug = False
247
249 self.workdir = args['workdir']
250 self.filename = args['slavedest']
251 self.reader = args['reader']
252 self.bytes_remaining = args['maxsize']
253 self.blocksize = args['blocksize']
254 self.mode = args['mode']
255 self.stderr = None
256 self.rc = 0
257
259 if self.debug:
260 log.msg('SlaveFileDownloadCommand starting')
261
262
263 self.path = os.path.join(self.builder.basedir,
264 self.workdir,
265 os.path.expanduser(self.filename))
266
267 dirname = os.path.dirname(self.path)
268 if not os.path.exists(dirname):
269 os.makedirs(dirname)
270
271 try:
272 self.fp = open(self.path, 'wb')
273 if self.debug:
274 log.msg("Opened '%s' for download" % self.path)
275 if self.mode is not None:
276
277
278
279
280
281
282
283 os.chmod(self.path, self.mode)
284 except IOError:
285
286 self.fp = None
287 self.stderr = "Cannot open file '%s' for download" % self.path
288 self.rc = 1
289 if self.debug:
290 log.msg("Cannot open file '%s' for download" % self.path)
291
292 d = defer.Deferred()
293 self._reactor.callLater(0, self._loop, d)
294 def _close(res):
295
296 d1 = self.reader.callRemote('close')
297 d1.addErrback(log.err, 'while trying to close reader')
298 d1.addCallback(lambda ignored: res)
299 return d1
300 d.addBoth(_close)
301 d.addBoth(self.finished)
302 return d
303
304 - def _loop(self, fire_when_done):
305 d = defer.maybeDeferred(self._readBlock)
306 def _done(finished):
307 if finished:
308 fire_when_done.callback(None)
309 else:
310 self._loop(fire_when_done)
311 def _err(why):
312 fire_when_done.errback(why)
313 d.addCallbacks(_done, _err)
314 return None
315
317 """Read a block of data from the remote reader."""
318
319 if self.interrupted or self.fp is None:
320 if self.debug:
321 log.msg('SlaveFileDownloadCommand._readBlock(): end')
322 return True
323
324 length = self.blocksize
325 if self.bytes_remaining is not None and length > self.bytes_remaining:
326 length = self.bytes_remaining
327
328 if length <= 0:
329 if self.stderr is None:
330 self.stderr = "Maximum filesize reached, truncating file '%s'" \
331 % self.path
332 self.rc = 1
333 return True
334 else:
335 d = self.reader.callRemote('read', length)
336 d.addCallback(self._writeData)
337 return d
338
340 if self.debug:
341 log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
342 len(data))
343 if len(data) == 0:
344 return True
345
346 if self.bytes_remaining is not None:
347 self.bytes_remaining = self.bytes_remaining - len(data)
348 assert self.bytes_remaining >= 0
349 self.fp.write(data)
350 return False
351
357