buildslave.commands.transfer

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import os, tarfile, tempfile 17 18 from twisted.python import log 19 from twisted.internet import defer 20 21 from buildslave.commands.base import Command 22

23 -class SlaveFileUploadCommand(Command):

24 """ 25 Upload a file from slave to build master 26 Arguments: 27 28 - ['workdir']: base directory to use 29 - ['slavesrc']: name of the slave-side file to read from 30 - ['writer']: RemoteReference to a transfer._FileWriter object 31 - ['maxsize']: max size (in bytes) of file to write 32 - ['blocksize']: max size for each data block 33 """ 34 debug = False 35

36 - def setup(self, args):

37 self.workdir = args['workdir'] 38 self.filename = args['slavesrc'] 39 self.writer = args['writer'] 40 self.remaining = args['maxsize'] 41 self.blocksize = args['blocksize'] 42 self.stderr = None 43 self.rc = 0

44

45 - def start(self):

46 if self.debug: 47 log.msg('SlaveFileUploadCommand started') 48 49 # Open file 50 self.path = os.path.join(self.builder.basedir, 51 self.workdir, 52 os.path.expanduser(self.filename)) 53 try: 54 self.fp = open(self.path, 'rb') 55 if self.debug: 56 log.msg("Opened '%s' for upload" % self.path) 57 except: 58 self.fp = None 59 self.stderr = "Cannot open file '%s' for upload" % self.path 60 self.rc = 1 61 if self.debug: 62 log.msg("Cannot open file '%s' for upload" % self.path) 63 64 self.sendStatus({'header': "sending %s" % self.path}) 65 66 d = defer.Deferred() 67 self._reactor.callLater(0, self._loop, d) 68 def _close_ok(res): 69 self.fp = None 70 return self.writer.callRemote("close")

71 def _close_err(f): 72 self.fp = None 73 # call remote's close(), but keep the existing failure 74 d1 = self.writer.callRemote("close") 75 def eb(f2): 76 log.msg("ignoring error from remote close():") 77 log.err(f2)

78 d1.addErrback(eb) 79 d1.addBoth(lambda _ : f) # always return _loop failure 80 return d1 81 d.addCallbacks(_close_ok, _close_err) 82 d.addBoth(self.finished) 83 return d 84

85 - def _loop(self, fire_when_done):

86 d = defer.maybeDeferred(self._writeBlock) 87 def _done(finished): 88 if finished: 89 fire_when_done.callback(None) 90 else: 91 self._loop(fire_when_done)

92 def _err(why): 93 fire_when_done.errback(why) 94 d.addCallbacks(_done, _err) 95 return None 96

97 - def _writeBlock(self):

98 """Write a block of data to the remote writer""" 99 100 if self.interrupted or self.fp is None: 101 if self.debug: 102 log.msg('SlaveFileUploadCommand._writeBlock(): end') 103 return True 104 105 length = self.blocksize 106 if self.remaining is not None and length > self.remaining: 107 length = self.remaining 108 109 if length <= 0: 110 if self.stderr is None: 111 self.stderr = 'Maximum filesize reached, truncating file \'%s\'' \ 112 % self.path 113 self.rc = 1 114 data = '' 115 else: 116 data = self.fp.read(length) 117 118 if self.debug: 119 log.msg('SlaveFileUploadCommand._writeBlock(): '+ 120 'allowed=%d readlen=%d' % (length, len(data))) 121 if len(data) == 0: 122 log.msg("EOF: callRemote(close)") 123 return True 124 125 if self.remaining is not None: 126 self.remaining = self.remaining - len(data) 127 assert self.remaining >= 0 128 d = self.writer.callRemote('write', data) 129 d.addCallback(lambda res: False) 130 return d

131

132 - def interrupt(self):

133 if self.debug: 134 log.msg('interrupted') 135 if self.interrupted: 136 return 137 if self.stderr is None: 138 self.stderr = 'Upload of \'%s\' interrupted' % self.path 139 self.rc = 1 140 self.interrupted = True

141 # the next _writeBlock call will notice the .interrupted flag 142

143 - def finished(self, res):

144 if self.debug: 145 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 146 if self.stderr is None: 147 self.sendStatus({'rc': self.rc}) 148 else: 149 self.sendStatus({'stderr': self.stderr, 'rc': self.rc}) 150 return res

151 152

153 -class SlaveDirectoryUploadCommand(SlaveFileUploadCommand):

154 """ 155 Upload a directory from slave to build master 156 Arguments: 157 158 - ['workdir']: base directory to use 159 - ['slavesrc']: name of the slave-side directory to read from 160 - ['writer']: RemoteReference to a transfer._DirectoryWriter object 161 - ['maxsize']: max size (in bytes) of file to write 162 - ['blocksize']: max size for each data block 163 - ['compress']: one of [None, 'bz2', 'gz'] 164 """ 165 debug = False 166

167 - def setup(self, args):

168 self.workdir = args['workdir'] 169 self.dirname = args['slavesrc'] 170 self.writer = args['writer'] 171 self.remaining = args['maxsize'] 172 self.blocksize = args['blocksize'] 173 self.compress = args['compress'] 174 self.stderr = None 175 self.rc = 0

176

177 - def start(self):

178 if self.debug: 179 log.msg('SlaveDirectoryUploadCommand started') 180 181 self.path = os.path.join(self.builder.basedir, 182 self.workdir, 183 os.path.expanduser(self.dirname)) 184 if self.debug: 185 log.msg("path: %r" % self.path) 186 187 # Create temporary archive 188 fd, self.tarname = tempfile.mkstemp() 189 fileobj = os.fdopen(fd, 'w') 190 if self.compress == 'bz2': 191 mode='w|bz2' 192 elif self.compress == 'gz': 193 mode='w|gz' 194 else: 195 mode = 'w' 196 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) 197 archive.add(self.path, '') 198 archive.close() 199 fileobj.close() 200 201 # Transfer it 202 self.fp = open(self.tarname, 'rb') 203 204 self.sendStatus({'header': "sending %s" % self.path}) 205 206 d = defer.Deferred() 207 self._reactor.callLater(0, self._loop, d) 208 def unpack(res): 209 # unpack the archive, but pass through any errors from _loop 210 d1 = self.writer.callRemote("unpack") 211 d1.addErrback(log.err) 212 d1.addCallback(lambda ignored: res) 213 return d1

214 d.addCallback(unpack) 215 d.addBoth(self.finished) 216 return d

217

218 - def finished(self, res):

219 self.fp.close() 220 os.remove(self.tarname) 221 if self.debug: 222 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 223 if self.stderr is None: 224 self.sendStatus({'rc': self.rc}) 225 else: 226 self.sendStatus({'stderr': self.stderr, 'rc': self.rc}) 227 return res

228 229

230 -class SlaveFileDownloadCommand(Command):

231 """ 232 Download a file from master to slave 233 Arguments: 234 235 - ['workdir']: base directory to use 236 - ['slavedest']: name of the slave-side file to be created 237 - ['reader']: RemoteReference to a transfer._FileReader object 238 - ['maxsize']: max size (in bytes) of file to write 239 - ['blocksize']: max size for each data block 240 - ['mode']: access mode for the new file 241 """ 242 debug = False 243

244 - def setup(self, args):

245 self.workdir = args['workdir'] 246 self.filename = args['slavedest'] 247 self.reader = args['reader'] 248 self.bytes_remaining = args['maxsize'] 249 self.blocksize = args['blocksize'] 250 self.mode = args['mode'] 251 self.stderr = None 252 self.rc = 0

253

254 - def start(self):

255 if self.debug: 256 log.msg('SlaveFileDownloadCommand starting') 257 258 # Open file 259 self.path = os.path.join(self.builder.basedir, 260 self.workdir, 261 os.path.expanduser(self.filename)) 262 263 dirname = os.path.dirname(self.path) 264 if not os.path.exists(dirname): 265 os.makedirs(dirname) 266 267 try: 268 self.fp = open(self.path, 'wb') 269 if self.debug: 270 log.msg("Opened '%s' for download" % self.path) 271 if self.mode is not None: 272 # note: there is a brief window during which the new file 273 # will have the buildslave's default (umask) mode before we 274 # set the new one. Don't use this mode= feature to keep files 275 # private: use the buildslave's umask for that instead. (it 276 # is possible to call os.umask() before and after the open() 277 # call, but cleaning up from exceptions properly is more of a 278 # nuisance that way). 279 os.chmod(self.path, self.mode) 280 except IOError: 281 # TODO: this still needs cleanup 282 self.fp = None 283 self.stderr = "Cannot open file '%s' for download" % self.path 284 self.rc = 1 285 if self.debug: 286 log.msg("Cannot open file '%s' for download" % self.path) 287 288 d = defer.Deferred() 289 self._reactor.callLater(0, self._loop, d) 290 def _close(res): 291 # close the file, but pass through any errors from _loop 292 d1 = self.reader.callRemote('close') 293 d1.addErrback(log.err) # ignore errors closing a reader file 294 d1.addCallback(lambda ignored: res) 295 return d1

296 d.addBoth(_close) 297 d.addBoth(self.finished) 298 return d

299

300 - def _loop(self, fire_when_done):

301 d = defer.maybeDeferred(self._readBlock) 302 def _done(finished): 303 if finished: 304 fire_when_done.callback(None) 305 else: 306 self._loop(fire_when_done)

307 def _err(why): 308 fire_when_done.errback(why) 309 d.addCallbacks(_done, _err) 310 return None 311

312 - def _readBlock(self):

313 """Read a block of data from the remote reader.""" 314 315 if self.interrupted or self.fp is None: 316 if self.debug: 317 log.msg('SlaveFileDownloadCommand._readBlock(): end') 318 return True 319 320 length = self.blocksize 321 if self.bytes_remaining is not None and length > self.bytes_remaining: 322 length = self.bytes_remaining 323 324 if length <= 0: 325 if self.stderr is None: 326 self.stderr = "Maximum filesize reached, truncating file '%s'" \ 327 % self.path 328 self.rc = 1 329 return True 330 else: 331 d = self.reader.callRemote('read', length) 332 d.addCallback(self._writeData) 333 return d

334

335 - def _writeData(self, data):

336 if self.debug: 337 log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' % 338 len(data)) 339 if len(data) == 0: 340 return True 341 342 if self.bytes_remaining is not None: 343 self.bytes_remaining = self.bytes_remaining - len(data) 344 assert self.bytes_remaining >= 0 345 self.fp.write(data) 346 return False

347

348 - def interrupt(self):

349 if self.debug: 350 log.msg('interrupted') 351 if self.interrupted: 352 return 353 if self.stderr is None: 354 self.stderr = "Download of '%s' interrupted" % self.path 355 self.rc = 1 356 self.interrupted = True

357 # now we wait for the next read request to return. _readBlock will 358 # abandon the file when it sees self.interrupted set. 359

360 - def finished(self, res):

361 if self.fp is not None: 362 self.fp.close() 363 364 if self.debug: 365 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 366 if self.stderr is None: 367 self.sendStatus({'rc': self.rc}) 368 else: 369 self.sendStatus({'stderr': self.stderr, 'rc': self.rc}) 370 return res

371

Source Code for Module buildslave.commands.transfer