Source Code for Module buildslave.commands.repo

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import os 
 17  import re 
 18   
 19  from twisted.internet import defer 
 20   
 21  from buildslave.commands.base import SourceBaseCommand 
 22  from buildslave import runprocess 
 23  from buildslave.commands.base import AbandonChain 
 24   
 25   
 26 -class Repo(SourceBaseCommand): 
 27      """Repo specific VC operation. In addition to the arguments 
 28      handled by SourceBaseCommand, this command reads the following keys: 
 29   
 30      ['manifest_url'] (required):    The manifests repo repository. 
 31      ['manifest_branch'] (optional): Which manifest repo version (i.e. branch or tag) 
 32                                      to retrieve. Default: "master". 
 33      ['manifest_file'] (optional):   Which manifest file to use. Default: "default.xml". 
 34      ['tarball'] (optional):         The tarball base to accelerate the fetch. 
 35      ['repo_downloads'] (optional):  Repo downloads to do. Computer from GerritChangeSource 
 36                                      and forced build properties. 
 37      """ 
 38   
 39      header = "repo operation" 
 40   
 41 -    def setup(self, args): 
 42          SourceBaseCommand.setup(self, args) 
 43          self.manifest_url = args.get('manifest_url') 
 44          self.manifest_branch = args.get('manifest_branch') 
 45          self.manifest_file =  args.get('manifest_file') 
 46          self.tarball = args.get('tarball') 
 47          self.repo_downloads = args.get('repo_downloads') 
 48          # we're using string instead of an array here, because it will be transferred back 
 49          # to the master as string anyway and using eval() could have security implications. 
 50          self.repo_downloaded = "" 
 51   
 52          self.sourcedata = "%s %s %s" % (self.manifest_url, self.manifest_branch, self.manifest_file) 
 53          self.re_change = re.compile(".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") 
 54          self.re_head = re.compile("^HEAD is now at ([0-9a-f]+)...") 
 55   
 56 -    def _fullSrcdir(self): 
 57          return os.path.join(self.builder.basedir, self.srcdir) 
 58   
 59 -    def sourcedirIsUpdateable(self): 
 60          print os.path.join(self._fullSrcdir(), ".repo") 
 61          print os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 
 62          return os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 
 63   
 64 -    def _repoCmd(self, command, cb=None, abandonOnFailure=True, **kwargs): 
 65          repo = self.getCommand("repo") 
 66          c = runprocess.RunProcess(self.builder, [repo] + command, self._fullSrcdir(), 
 67                           sendRC=False, timeout=self.timeout, 
 68                           maxTime=self.maxTime, usePTY=False, 
 69                           logEnviron=self.logEnviron, **kwargs) 
 70          self.command = c 
 71          d = c.start() 
 72          if cb: 
 73              if abandonOnFailure: 
 74                  d.addCallback(self._abandonOnFailure) 
 75              d.addCallback(cb) 
 76          return d 
 77   
 78 -    def _tarCmd(self, cmds, callback): 
 79          cmd = ["tar"] + cmds 
 80          c = runprocess.RunProcess(self.builder, cmd, self._fullSrcdir(), 
 81                                    sendRC=False, timeout=self.timeout, 
 82                                    maxTime=self.maxTime, usePTY=False, 
 83                                    logEnviron=self.logEnviron) 
 84          self.command = c 
 85          cmdexec = c.start() 
 86          cmdexec.addCallback(callback) 
 87          return cmdexec 
 88   
 89 -    def _gitCmd(self, subdir, cmds, callback): 
 90          cmd = ["git"] + cmds 
 91          c = runprocess.RunProcess(self.builder, cmd, os.path.join(self._fullSrcdir(), subdir), 
 92                                    sendRC=False, timeout=self.timeout, 
 93                                    maxTime=self.maxTime, usePTY=False, 
 94                                    logEnviron=self.logEnviron) 
 95          self.command = c 
 96          cmdexec = c.start() 
 97          cmdexec.addCallback(callback) 
 98          return cmdexec 
 99   
100 -    def sourcedataMatches(self): 
101          try: 
102              olddata = self.readSourcedata() 
103              return olddata == self.sourcedata 
104          except IOError: 
105              return False 
106   
107 -    def doVCFull(self): 
108          os.makedirs(self._fullSrcdir()) 
109          if self.tarball and os.path.exists(self.tarball): 
110              return self._tarCmd(['-xvzf', self.tarball], self._doInit) 
111          else: 
112              return self._doInit(None) 
113   
114 -    def _doInit(self,res): 
115          # on fresh init, this file may confuse repo. 
116          if os.path.exists(os.path.join(self._fullSrcdir(), ".repo/project.list")): 
117              os.unlink(os.path.join(self._fullSrcdir(), ".repo/project.list")) 
118          return self._repoCmd(['init', '-u', self.manifest_url, '-b', self.manifest_branch, '-m', self.manifest_file], self._didInit) 
119   
120 -    def _didInit(self, res): 
121          return self.doVCUpdate() 
122   
123 -    def doVCUpdate(self): 
124          command = ['forall', '-c', 'git', 'clean', '-f', '-d', '-x'] 
125          return self._repoCmd(command, self._doClean2, abandonOnFailure=False) 
126   
127 -    def _doClean2(self, dummy): 
128          command = ['forall', '-c', 'git', 'reset', '--hard', 'HEAD'] 
129          return self._repoCmd(command, self._doClean3, abandonOnFailure=False) 
130   
131 -    def _doClean3(self,dummy): 
132          command = ['clean', '-f', '-d', '-x'] 
133          return self._gitCmd(".repo/manifests",command, self._doSync) 
134   
135 -    def _doSync(self, dummy): 
136          command = ['sync'] 
137          self.sendStatus({"header": "synching manifest %s from branch %s from %s\n" 
138                                     % (self.manifest_file, self.manifest_branch, self.manifest_url)}) 
139          return self._repoCmd(command, self._didSync) 
140   
141 -    def _didSync(self, dummy): 
142          if self.tarball and not os.path.exists(self.tarball): 
143              return self._tarCmd(['-cvzf', self.tarball, ".repo"], self._doDownload) 
144          else: 
145              return self._doDownload(None) 
146   
147 -    def _doDownload(self, dummy): 
148          if hasattr(self.command, 'stderr') and self.command.stderr: 
149              if "Automatic cherry-pick failed" in self.command.stderr: 
150                  command = ['forall','-c' ,'git' ,'diff', 'HEAD'] 
151                  self.cherry_pick_failed = True 
152                  return self._repoCmd(command, self._DownloadAbandon, abandonOnFailure = False, keepStderr=True) # call again 
153   
154              lines = self.command.stderr.split('\n') 
155              if len(lines) > 2: 
156                  match1 = self.re_change.match(lines[1]) 
157                  match2 = self.re_head.match(lines[-2]) 
158                  if match1 and match2: 
159                      self.repo_downloaded += "%s/%s %s " % (match1.group(1), match1.group(2), match2.group(1)) 
160   
161          if self.repo_downloads: 
162              # download each changeset while the self.download variable is not empty 
163              download = self.repo_downloads.pop(0) 
164              command = ['download'] + download.split(' ') 
165              self.sendStatus({"header": "downloading changeset %s\n" 
166                                         % (download)}) 
167              return self._repoCmd(command, self._doDownload, abandonOnFailure = False, keepStderr=True) # call again 
168   
169          if self.repo_downloaded: 
170              self.sendStatus({"repo_downloaded": self.repo_downloaded[:-1]}) 
171          return defer.succeed(0) 
172   
173 -    def maybeNotDoVCFallback(self, res): 
174          # If we were unable to find the branch/SHA on the remote, 
175          # clobbering the repo won't help any, so just abort the chain 
176          if hasattr(self.command, 'stderr'): 
177              if "Couldn't find remote ref" in self.command.stderr: 
178                  raise AbandonChain(-1) 
179              if hasattr(self, 'cherry_pick_failed') or "Automatic cherry-pick failed" in self.command.stderr: 
180                  raise AbandonChain(-1) 
181 -    def _DownloadAbandon(self,dummy): 
182          self.sendStatus({"header": "abandonned due to merge failure\n"}) 
183          raise AbandonChain(-1) 
184