buildslave.commands.transfer

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import os, tarfile, tempfile 17 18 from twisted.python import log 19 from twisted.internet import defer 20 21 from buildslave.commands.base import Command 22

23 -class TransferCommand(Command):

24

25 - def finished(self, res):

26 if self.debug: 27 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 28 29 # don't use self.sendStatus here, since we may no longer be running 30 # if we have been interrupted 31 upd = {'rc': self.rc} 32 if self.stderr: 33 upd['stderr'] = self.stderr 34 self.builder.sendUpdate(upd) 35 return res

36

37 - def interrupt(self):

38 if self.debug: 39 log.msg('interrupted') 40 if self.interrupted: 41 return 42 self.rc = 1 43 self.interrupted = True

44 # now we wait for the next trip around the loop. It abandon the file 45 # when it sees self.interrupted set. 46 47

48 -class SlaveFileUploadCommand(TransferCommand):

49 """ 50 Upload a file from slave to build master 51 Arguments: 52 53 - ['workdir']: base directory to use 54 - ['slavesrc']: name of the slave-side file to read from 55 - ['writer']: RemoteReference to a transfer._FileWriter object 56 - ['maxsize']: max size (in bytes) of file to write 57 - ['blocksize']: max size for each data block 58 - ['keepstamp']: whether to preserve file modified and accessed times 59 """ 60 debug = False 61

62 - def setup(self, args):

63 self.workdir = args['workdir'] 64 self.filename = args['slavesrc'] 65 self.writer = args['writer'] 66 self.remaining = args['maxsize'] 67 self.blocksize = args['blocksize'] 68 self.keepstamp = args.get('keepstamp', False) 69 self.stderr = None 70 self.rc = 0

71

72 - def start(self):

73 if self.debug: 74 log.msg('SlaveFileUploadCommand started') 75 76 # Open file 77 self.path = os.path.join(self.builder.basedir, 78 self.workdir, 79 os.path.expanduser(self.filename)) 80 accessed_modified = None 81 try: 82 if self.keepstamp: 83 accessed_modified = (os.path.getatime(self.path), 84 os.path.getmtime(self.path)) 85 86 self.fp = open(self.path, 'rb') 87 if self.debug: 88 log.msg("Opened '%s' for upload" % self.path) 89 except: 90 self.fp = None 91 self.stderr = "Cannot open file '%s' for upload" % self.path 92 self.rc = 1 93 if self.debug: 94 log.msg("Cannot open file '%s' for upload" % self.path) 95 96 self.sendStatus({'header': "sending %s" % self.path}) 97 98 d = defer.Deferred() 99 self._reactor.callLater(0, self._loop, d) 100 def _close_ok(res): 101 self.fp = None 102 d1 = self.writer.callRemote("close") 103 def _utime_ok(res): 104 return self.writer.callRemote("utime", accessed_modified)

105 if self.keepstamp: 106 d1.addCallback(_utime_ok) 107 return d1

108 def _close_err(f): 109 self.fp = None 110 # call remote's close(), but keep the existing failure 111 d1 = self.writer.callRemote("close") 112 def eb(f2): 113 log.msg("ignoring error from remote close():") 114 log.err(f2) 115 d1.addErrback(eb) 116 d1.addBoth(lambda _ : f) # always return _loop failure 117 return d1 118 119 d.addCallbacks(_close_ok, _close_err) 120 d.addBoth(self.finished) 121 return d 122

123 - def _loop(self, fire_when_done):

124 d = defer.maybeDeferred(self._writeBlock) 125 def _done(finished): 126 if finished: 127 fire_when_done.callback(None) 128 else: 129 self._loop(fire_when_done)

130 def _err(why): 131 fire_when_done.errback(why) 132 d.addCallbacks(_done, _err) 133 return None 134

135 - def _writeBlock(self):

136 """Write a block of data to the remote writer""" 137 138 if self.interrupted or self.fp is None: 139 if self.debug: 140 log.msg('SlaveFileUploadCommand._writeBlock(): end') 141 return True 142 143 length = self.blocksize 144 if self.remaining is not None and length > self.remaining: 145 length = self.remaining 146 147 if length <= 0: 148 if self.stderr is None: 149 self.stderr = 'Maximum filesize reached, truncating file \'%s\'' \ 150 % self.path 151 self.rc = 1 152 data = '' 153 else: 154 data = self.fp.read(length) 155 156 if self.debug: 157 log.msg('SlaveFileUploadCommand._writeBlock(): '+ 158 'allowed=%d readlen=%d' % (length, len(data))) 159 if len(data) == 0: 160 log.msg("EOF: callRemote(close)") 161 return True 162 163 if self.remaining is not None: 164 self.remaining = self.remaining - len(data) 165 assert self.remaining >= 0 166 d = self.writer.callRemote('write', data) 167 d.addCallback(lambda res: False) 168 return d

169 170

171 -class SlaveDirectoryUploadCommand(SlaveFileUploadCommand):

172 """ 173 Upload a directory from slave to build master 174 Arguments: 175 176 - ['workdir']: base directory to use 177 - ['slavesrc']: name of the slave-side directory to read from 178 - ['writer']: RemoteReference to a transfer._DirectoryWriter object 179 - ['maxsize']: max size (in bytes) of file to write 180 - ['blocksize']: max size for each data block 181 - ['compress']: one of [None, 'bz2', 'gz'] 182 """ 183 debug = False 184

185 - def setup(self, args):

186 self.workdir = args['workdir'] 187 self.dirname = args['slavesrc'] 188 self.writer = args['writer'] 189 self.remaining = args['maxsize'] 190 self.blocksize = args['blocksize'] 191 self.compress = args['compress'] 192 self.stderr = None 193 self.rc = 0

194

195 - def start(self):

196 if self.debug: 197 log.msg('SlaveDirectoryUploadCommand started') 198 199 self.path = os.path.join(self.builder.basedir, 200 self.workdir, 201 os.path.expanduser(self.dirname)) 202 if self.debug: 203 log.msg("path: %r" % self.path) 204 205 # Create temporary archive 206 fd, self.tarname = tempfile.mkstemp() 207 fileobj = os.fdopen(fd, 'w') 208 if self.compress == 'bz2': 209 mode='w|bz2' 210 elif self.compress == 'gz': 211 mode='w|gz' 212 else: 213 mode = 'w' 214 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) 215 archive.add(self.path, '') 216 archive.close() 217 fileobj.close() 218 219 # Transfer it 220 self.fp = open(self.tarname, 'rb') 221 222 self.sendStatus({'header': "sending %s" % self.path}) 223 224 d = defer.Deferred() 225 self._reactor.callLater(0, self._loop, d) 226 def unpack(res): 227 # unpack the archive, but pass through any errors from _loop 228 d1 = self.writer.callRemote("unpack") 229 d1.addErrback(log.err) 230 d1.addCallback(lambda ignored: res) 231 return d1

232 d.addCallback(unpack) 233 d.addBoth(self.finished) 234 return d

235

236 - def finished(self, res):

237 self.fp.close() 238 os.remove(self.tarname) 239 return TransferCommand.finished(self, res)

240 241

242 -class SlaveFileDownloadCommand(TransferCommand):

243 """ 244 Download a file from master to slave 245 Arguments: 246 247 - ['workdir']: base directory to use 248 - ['slavedest']: name of the slave-side file to be created 249 - ['reader']: RemoteReference to a transfer._FileReader object 250 - ['maxsize']: max size (in bytes) of file to write 251 - ['blocksize']: max size for each data block 252 - ['mode']: access mode for the new file 253 """ 254 debug = False 255

256 - def setup(self, args):

257 self.workdir = args['workdir'] 258 self.filename = args['slavedest'] 259 self.reader = args['reader'] 260 self.bytes_remaining = args['maxsize'] 261 self.blocksize = args['blocksize'] 262 self.mode = args['mode'] 263 self.stderr = None 264 self.rc = 0

265

266 - def start(self):

267 if self.debug: 268 log.msg('SlaveFileDownloadCommand starting') 269 270 # Open file 271 self.path = os.path.join(self.builder.basedir, 272 self.workdir, 273 os.path.expanduser(self.filename)) 274 275 dirname = os.path.dirname(self.path) 276 if not os.path.exists(dirname): 277 os.makedirs(dirname) 278 279 try: 280 self.fp = open(self.path, 'wb') 281 if self.debug: 282 log.msg("Opened '%s' for download" % self.path) 283 if self.mode is not None: 284 # note: there is a brief window during which the new file 285 # will have the buildslave's default (umask) mode before we 286 # set the new one. Don't use this mode= feature to keep files 287 # private: use the buildslave's umask for that instead. (it 288 # is possible to call os.umask() before and after the open() 289 # call, but cleaning up from exceptions properly is more of a 290 # nuisance that way). 291 os.chmod(self.path, self.mode) 292 except IOError: 293 # TODO: this still needs cleanup 294 self.fp = None 295 self.stderr = "Cannot open file '%s' for download" % self.path 296 self.rc = 1 297 if self.debug: 298 log.msg("Cannot open file '%s' for download" % self.path) 299 300 d = defer.Deferred() 301 self._reactor.callLater(0, self._loop, d) 302 def _close(res): 303 # close the file, but pass through any errors from _loop 304 d1 = self.reader.callRemote('close') 305 d1.addErrback(log.err, 'while trying to close reader') 306 d1.addCallback(lambda ignored: res) 307 return d1

308 d.addBoth(_close) 309 d.addBoth(self.finished) 310 return d

311

312 - def _loop(self, fire_when_done):

313 d = defer.maybeDeferred(self._readBlock) 314 def _done(finished): 315 if finished: 316 fire_when_done.callback(None) 317 else: 318 self._loop(fire_when_done)

319 def _err(why): 320 fire_when_done.errback(why) 321 d.addCallbacks(_done, _err) 322 return None 323

324 - def _readBlock(self):

325 """Read a block of data from the remote reader.""" 326 327 if self.interrupted or self.fp is None: 328 if self.debug: 329 log.msg('SlaveFileDownloadCommand._readBlock(): end') 330 return True 331 332 length = self.blocksize 333 if self.bytes_remaining is not None and length > self.bytes_remaining: 334 length = self.bytes_remaining 335 336 if length <= 0: 337 if self.stderr is None: 338 self.stderr = "Maximum filesize reached, truncating file '%s'" \ 339 % self.path 340 self.rc = 1 341 return True 342 else: 343 d = self.reader.callRemote('read', length) 344 d.addCallback(self._writeData) 345 return d

346

347 - def _writeData(self, data):

348 if self.debug: 349 log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' % 350 len(data)) 351 if len(data) == 0: 352 return True 353 354 if self.bytes_remaining is not None: 355 self.bytes_remaining = self.bytes_remaining - len(data) 356 assert self.bytes_remaining >= 0 357 self.fp.write(data) 358 return False

359

360 - def finished(self, res):

361 if self.fp is not None: 362 self.fp.close() 363 364 return TransferCommand.finished(self, res)

365

Source Code for Module buildslave.commands.transfer