Source Code for Module buildslave.commands.repo

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import os 
 17  import re 
 18   
 19  from twisted.internet import defer 
 20   
 21  from buildslave.commands.base import SourceBaseCommand 
 22  from buildslave import runprocess 
 23  from buildslave.commands.base import AbandonChain 
 24   
 25   
 26 -class Repo(SourceBaseCommand): 
 27      """Repo specific VC operation. In addition to the arguments 
 28      handled by SourceBaseCommand, this command reads the following keys: 
 29   
 30      ['manifest_url'] (required):    The manifests repo repository. 
 31      ['manifest_branch'] (optional): Which manifest repo version (i.e. branch or tag) 
 32                                      to retrieve. Default: "master". 
 33      ['manifest_file'] (optional):   Which manifest file to use. Default: "default.xml". 
 34      ['tarball'] (optional):         The tarball base to accelerate the fetch. 
 35      ['repo_downloads'] (optional):  Repo downloads to do. Computer from GerritChangeSource 
 36                                      and forced build properties. 
 37      """ 
 38   
 39      header = "repo operation" 
 40   
 41 -    def setup(self, args): 
 42          SourceBaseCommand.setup(self, args) 
 43          self.manifest_url = args.get('manifest_url') 
 44          self.manifest_branch = args.get('manifest_branch') 
 45          self.manifest_file =  args.get('manifest_file') 
 46          self.tarball = args.get('tarball') 
 47          self.repo_downloads = args.get('repo_downloads') 
 48          # we're using string instead of an array here, because it will be transferred back 
 49          # to the master as string anyway and using eval() could have security implications. 
 50          self.repo_downloaded = "" 
 51   
 52          self.sourcedata = "%s %s %s" % (self.manifest_url, self.manifest_branch, self.manifest_file) 
 53          self.re_change = re.compile(".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") 
 54          self.re_head = re.compile("^HEAD is now at ([0-9a-f]+)...") 
 55   
 56 -    def _fullSrcdir(self): 
 57          return os.path.join(self.builder.basedir, self.srcdir) 
 58   
 59 -    def sourcedirIsUpdateable(self): 
 60          print os.path.join(self._fullSrcdir(), ".repo") 
 61          print os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 
 62          return os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 
 63   
 64 -    def _repoCmd(self, command, cb=None, abandonOnFailure=True, **kwargs): 
 65          repo = self.getCommand("repo") 
 66          c = runprocess.RunProcess(self.builder, [repo] + command, self._fullSrcdir(), 
 67                           sendRC=False, timeout=self.timeout, 
 68                           maxTime=self.maxTime, usePTY=False, **kwargs) 
 69          self.command = c 
 70          d = c.start() 
 71          if cb: 
 72              if abandonOnFailure: 
 73                  d.addCallback(self._abandonOnFailure) 
 74              d.addCallback(cb) 
 75          return d 
 76   
 77 -    def _tarCmd(self, cmds, callback): 
 78          cmd = ["tar"] + cmds 
 79          c = runprocess.RunProcess(self.builder, cmd, self._fullSrcdir(), 
 80                                    sendRC=False, timeout=self.timeout, 
 81                                    maxTime=self.maxTime, usePTY=False) 
 82          self.command = c 
 83          cmdexec = c.start() 
 84          cmdexec.addCallback(callback) 
 85          return cmdexec 
 86   
 87 -    def _gitCmd(self, subdir, cmds, callback): 
 88          cmd = ["git"] + cmds 
 89          c = runprocess.RunProcess(self.builder, cmd, os.path.join(self._fullSrcdir(), subdir), 
 90                                    sendRC=False, timeout=self.timeout, 
 91                                    maxTime=self.maxTime, usePTY=False) 
 92          self.command = c 
 93          cmdexec = c.start() 
 94          cmdexec.addCallback(callback) 
 95          return cmdexec 
 96   
 97 -    def sourcedataMatches(self): 
 98          try: 
 99              olddata = self.readSourcedata() 
100              return olddata == self.sourcedata 
101          except IOError: 
102              return False 
103   
104 -    def doVCFull(self): 
105          os.makedirs(self._fullSrcdir()) 
106          if self.tarball and os.path.exists(self.tarball): 
107              return self._tarCmd(['-xvzf', self.tarball], self._doInit) 
108          else: 
109              return self._doInit(None) 
110   
111 -    def _doInit(self,res): 
112          # on fresh init, this file may confuse repo. 
113          if os.path.exists(os.path.join(self._fullSrcdir(), ".repo/project.list")): 
114              os.unlink(os.path.join(self._fullSrcdir(), ".repo/project.list")) 
115          return self._repoCmd(['init', '-u', self.manifest_url, '-b', self.manifest_branch, '-m', self.manifest_file], self._didInit) 
116   
117 -    def _didInit(self, res): 
118          return self.doVCUpdate() 
119   
120 -    def doVCUpdate(self): 
121          command = ['forall', '-c', 'git', 'clean', '-f', '-d', '-x'] 
122          return self._repoCmd(command, self._doClean2, abandonOnFailure=False) 
123   
124 -    def _doClean2(self,dummy): 
125          command = ['clean', '-f', '-d', '-x'] 
126          return self._gitCmd(".repo/manifests",command, self._doSync) 
127   
128 -    def _doSync(self, dummy): 
129          command = ['sync'] 
130          self.sendStatus({"header": "synching manifest %s from branch %s from %s\n" 
131                                     % (self.manifest_file, self.manifest_branch, self.manifest_url)}) 
132          return self._repoCmd(command, self._didSync) 
133   
134 -    def _didSync(self, dummy): 
135          if self.tarball and not os.path.exists(self.tarball): 
136              return self._tarCmd(['-cvzf', self.tarball, ".repo"], self._doDownload) 
137          else: 
138              return self._doDownload(None) 
139   
140 -    def _doDownload(self, dummy): 
141          if hasattr(self.command, 'stderr') and self.command.stderr: 
142              lines = self.command.stderr.split('\n') 
143              if len(lines) > 2: 
144                  match1 = self.re_change.match(lines[1]) 
145                  match2 = self.re_head.match(lines[-2]) 
146                  if match1 and match2: 
147                      self.repo_downloaded += "%s/%s %s " % (match1.group(1), match1.group(2), match2.group(1)) 
148   
149          if self.repo_downloads: 
150              # download each changeset while the self.download variable is not empty 
151              download = self.repo_downloads.pop(0) 
152              command = ['download'] + download.split(' ') 
153              self.sendStatus({"header": "downloading changeset %s\n" 
154                                         % (download)}) 
155              return self._repoCmd(command, self._doDownload, keepStderr=True) # call again 
156   
157          if self.repo_downloaded: 
158              self.sendStatus({"repo_downloaded": self.repo_downloaded[:-1]}) 
159          return defer.succeed(0) 
160   
161 -    def maybeNotDoVCFallback(self, res): 
162          # If we were unable to find the branch/SHA on the remote, 
163          # clobbering the repo won't help any, so just abort the chain 
164          if hasattr(self.command, 'stderr'): 
165              if "Couldn't find remote ref" in self.command.stderr: 
166                  raise AbandonChain(-1) 
167