buildslave.commands.transfer

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import os, tarfile, tempfile 17 18 from twisted.python import log 19 from twisted.internet import defer 20 21 from buildslave.commands.base import Command 22

23 -class TransferCommand(Command):

24

25 - def finished(self, res):

26 if self.debug: 27 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 28 29 # don't use self.sendStatus here, since we may no longer be running 30 # if we have been interrupted 31 upd = {'rc': self.rc} 32 if self.stderr: 33 upd['stderr'] = self.stderr 34 self.builder.sendUpdate(upd) 35 return res

36

37 - def interrupt(self):

38 if self.debug: 39 log.msg('interrupted') 40 if self.interrupted: 41 return 42 self.rc = 1 43 self.interrupted = True

44 # now we wait for the next trip around the loop. It abandon the file 45 # when it sees self.interrupted set. 46 47

48 -class SlaveFileUploadCommand(TransferCommand):

49 """ 50 Upload a file from slave to build master 51 Arguments: 52 53 - ['workdir']: base directory to use 54 - ['slavesrc']: name of the slave-side file to read from 55 - ['writer']: RemoteReference to a transfer._FileWriter object 56 - ['maxsize']: max size (in bytes) of file to write 57 - ['blocksize']: max size for each data block 58 - ['keepstamp']: whether to preserve file modified and accessed times 59 """ 60 debug = False 61

62 - def setup(self, args):

63 self.workdir = args['workdir'] 64 self.filename = args['slavesrc'] 65 self.writer = args['writer'] 66 self.remaining = args['maxsize'] 67 self.blocksize = args['blocksize'] 68 self.keepstamp = args.get('keepstamp', False) 69 self.stderr = None 70 self.rc = 0

71

72 - def start(self):

73 if self.debug: 74 log.msg('SlaveFileUploadCommand started') 75 76 # Open file 77 self.path = os.path.join(self.builder.basedir, 78 self.workdir, 79 os.path.expanduser(self.filename)) 80 accessed_modified = None 81 try: 82 if self.keepstamp: 83 accessed_modified = (os.path.getatime(self.path), 84 os.path.getmtime(self.path)) 85 86 self.fp = open(self.path, 'rb') 87 if self.debug: 88 log.msg("Opened '%s' for upload" % self.path) 89 except: 90 self.fp = None 91 self.stderr = "Cannot open file '%s' for upload" % self.path 92 self.rc = 1 93 if self.debug: 94 log.msg("Cannot open file '%s' for upload" % self.path) 95 96 self.sendStatus({'header': "sending %s" % self.path}) 97 98 d = defer.Deferred() 99 self._reactor.callLater(0, self._loop, d) 100 def _close_ok(res): 101 self.fp = None 102 d1 = self.writer.callRemote("close") 103 def _utime_ok(res): 104 return self.writer.callRemote("utime", accessed_modified)

105 if self.keepstamp: 106 d1.addCallback(_utime_ok) 107 return d1

108 def _close_err(f): 109 self.rc = 1 110 self.fp = None 111 # call remote's close(), but keep the existing failure 112 d1 = self.writer.callRemote("close") 113 def eb(f2): 114 log.msg("ignoring error from remote close():") 115 log.err(f2) 116 d1.addErrback(eb) 117 d1.addBoth(lambda _ : f) # always return _loop failure 118 return d1 119 120 d.addCallbacks(_close_ok, _close_err) 121 d.addBoth(self.finished) 122 return d 123

124 - def _loop(self, fire_when_done):

125 d = defer.maybeDeferred(self._writeBlock) 126 def _done(finished): 127 if finished: 128 fire_when_done.callback(None) 129 else: 130 self._loop(fire_when_done)

131 def _err(why): 132 fire_when_done.errback(why) 133 d.addCallbacks(_done, _err) 134 return None 135

136 - def _writeBlock(self):

137 """Write a block of data to the remote writer""" 138 139 if self.interrupted or self.fp is None: 140 if self.debug: 141 log.msg('SlaveFileUploadCommand._writeBlock(): end') 142 return True 143 144 length = self.blocksize 145 if self.remaining is not None and length > self.remaining: 146 length = self.remaining 147 148 if length <= 0: 149 if self.stderr is None: 150 self.stderr = 'Maximum filesize reached, truncating file \'%s\'' \ 151 % self.path 152 self.rc = 1 153 data = '' 154 else: 155 data = self.fp.read(length) 156 157 if self.debug: 158 log.msg('SlaveFileUploadCommand._writeBlock(): '+ 159 'allowed=%d readlen=%d' % (length, len(data))) 160 if len(data) == 0: 161 log.msg("EOF: callRemote(close)") 162 return True 163 164 if self.remaining is not None: 165 self.remaining = self.remaining - len(data) 166 assert self.remaining >= 0 167 d = self.writer.callRemote('write', data) 168 d.addCallback(lambda res: False) 169 return d

170 171

172 -class SlaveDirectoryUploadCommand(SlaveFileUploadCommand):

173 debug = False 174

175 - def setup(self, args):

176 self.workdir = args['workdir'] 177 self.dirname = args['slavesrc'] 178 self.writer = args['writer'] 179 self.remaining = args['maxsize'] 180 self.blocksize = args['blocksize'] 181 self.compress = args['compress'] 182 self.stderr = None 183 self.rc = 0

184

185 - def start(self):

186 if self.debug: 187 log.msg('SlaveDirectoryUploadCommand started') 188 189 self.path = os.path.join(self.builder.basedir, 190 self.workdir, 191 os.path.expanduser(self.dirname)) 192 if self.debug: 193 log.msg("path: %r" % self.path) 194 195 # Create temporary archive 196 fd, self.tarname = tempfile.mkstemp() 197 fileobj = os.fdopen(fd, 'w') 198 if self.compress == 'bz2': 199 mode='w|bz2' 200 elif self.compress == 'gz': 201 mode='w|gz' 202 else: 203 mode = 'w' 204 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) 205 archive.add(self.path, '') 206 archive.close() 207 fileobj.close() 208 209 # Transfer it 210 self.fp = open(self.tarname, 'rb') 211 212 self.sendStatus({'header': "sending %s" % self.path}) 213 214 d = defer.Deferred() 215 self._reactor.callLater(0, self._loop, d) 216 def unpack(res): 217 d1 = self.writer.callRemote("unpack") 218 def unpack_err(f): 219 self.rc = 1 220 return f

221 d1.addErrback(unpack_err) 222 d1.addCallback(lambda ignored: res) 223 return d1

224 d.addCallback(unpack) 225 d.addBoth(self.finished) 226 return d 227

228 - def finished(self, res):

229 self.fp.close() 230 os.remove(self.tarname) 231 return TransferCommand.finished(self, res)

232 233

234 -class SlaveFileDownloadCommand(TransferCommand):

235 """ 236 Download a file from master to slave 237 Arguments: 238 239 - ['workdir']: base directory to use 240 - ['slavedest']: name of the slave-side file to be created 241 - ['reader']: RemoteReference to a transfer._FileReader object 242 - ['maxsize']: max size (in bytes) of file to write 243 - ['blocksize']: max size for each data block 244 - ['mode']: access mode for the new file 245 """ 246 debug = False 247

248 - def setup(self, args):

249 self.workdir = args['workdir'] 250 self.filename = args['slavedest'] 251 self.reader = args['reader'] 252 self.bytes_remaining = args['maxsize'] 253 self.blocksize = args['blocksize'] 254 self.mode = args['mode'] 255 self.stderr = None 256 self.rc = 0

257

258 - def start(self):

259 if self.debug: 260 log.msg('SlaveFileDownloadCommand starting') 261 262 # Open file 263 self.path = os.path.join(self.builder.basedir, 264 self.workdir, 265 os.path.expanduser(self.filename)) 266 267 dirname = os.path.dirname(self.path) 268 if not os.path.exists(dirname): 269 os.makedirs(dirname) 270 271 try: 272 self.fp = open(self.path, 'wb') 273 if self.debug: 274 log.msg("Opened '%s' for download" % self.path) 275 if self.mode is not None: 276 # note: there is a brief window during which the new file 277 # will have the buildslave's default (umask) mode before we 278 # set the new one. Don't use this mode= feature to keep files 279 # private: use the buildslave's umask for that instead. (it 280 # is possible to call os.umask() before and after the open() 281 # call, but cleaning up from exceptions properly is more of a 282 # nuisance that way). 283 os.chmod(self.path, self.mode) 284 except IOError: 285 # TODO: this still needs cleanup 286 self.fp = None 287 self.stderr = "Cannot open file '%s' for download" % self.path 288 self.rc = 1 289 if self.debug: 290 log.msg("Cannot open file '%s' for download" % self.path) 291 292 d = defer.Deferred() 293 self._reactor.callLater(0, self._loop, d) 294 def _close(res): 295 # close the file, but pass through any errors from _loop 296 d1 = self.reader.callRemote('close') 297 d1.addErrback(log.err, 'while trying to close reader') 298 d1.addCallback(lambda ignored: res) 299 return d1

300 d.addBoth(_close) 301 d.addBoth(self.finished) 302 return d

303

304 - def _loop(self, fire_when_done):

305 d = defer.maybeDeferred(self._readBlock) 306 def _done(finished): 307 if finished: 308 fire_when_done.callback(None) 309 else: 310 self._loop(fire_when_done)

311 def _err(why): 312 fire_when_done.errback(why) 313 d.addCallbacks(_done, _err) 314 return None 315

316 - def _readBlock(self):

317 """Read a block of data from the remote reader.""" 318 319 if self.interrupted or self.fp is None: 320 if self.debug: 321 log.msg('SlaveFileDownloadCommand._readBlock(): end') 322 return True 323 324 length = self.blocksize 325 if self.bytes_remaining is not None and length > self.bytes_remaining: 326 length = self.bytes_remaining 327 328 if length <= 0: 329 if self.stderr is None: 330 self.stderr = "Maximum filesize reached, truncating file '%s'" \ 331 % self.path 332 self.rc = 1 333 return True 334 else: 335 d = self.reader.callRemote('read', length) 336 d.addCallback(self._writeData) 337 return d

338

339 - def _writeData(self, data):

340 if self.debug: 341 log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' % 342 len(data)) 343 if len(data) == 0: 344 return True 345 346 if self.bytes_remaining is not None: 347 self.bytes_remaining = self.bytes_remaining - len(data) 348 assert self.bytes_remaining >= 0 349 self.fp.write(data) 350 return False

351

352 - def finished(self, res):

353 if self.fp is not None: 354 self.fp.close() 355 356 return TransferCommand.finished(self, res)

357

Source Code for Module buildslave.commands.transfer