Source Code for Module buildslave.commands.repo

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import os 
 17  import re 
 18  import textwrap 
 19   
 20  from twisted.internet import defer 
 21   
 22  from buildslave.commands.base import SourceBaseCommand 
 23  from buildslave import runprocess 
 24  from buildslave.commands.base import AbandonChain 
 25   
 26   
 27 -class Repo(SourceBaseCommand): 
 28      """Repo specific VC operation. In addition to the arguments 
 29      handled by SourceBaseCommand, this command reads the following keys: 
 30   
 31      ['manifest_url'] (required):    The manifests repo repository. 
 32      ['manifest_branch'] (optional): Which manifest repo version (i.e. branch or tag) 
 33                                      to retrieve. Default: "master". 
 34      ['manifest_file'] (optional):   Which manifest file to use. Default: "default.xml". 
 35      ['manifest_override_url'] (optional):   Which manifest file to use as an overide. Default: None. 
 36                                      This is usually set by forced build to build over a known working base 
 37      ['tarball'] (optional):         The tarball base to accelerate the fetch. 
 38      ['repo_downloads'] (optional):  Repo downloads to do. Computer from GerritChangeSource 
 39                                      and forced build properties. 
 40      """ 
 41   
 42      header = "repo operation" 
 43   
 44 -    def setup(self, args): 
 45          SourceBaseCommand.setup(self, args) 
 46          self.manifest_url = args.get('manifest_url') 
 47          self.manifest_branch = args.get('manifest_branch') 
 48          self.manifest_file =  args.get('manifest_file') 
 49          self.manifest_override_url =  args.get('manifest_override_url') 
 50          self.tarball = args.get('tarball') 
 51          self.repo_downloads = args.get('repo_downloads') 
 52          # we're using string instead of an array here, because it will be transferred back 
 53          # to the master as string anyway and using eval() could have security implications. 
 54          self.repo_downloaded = "" 
 55   
 56          self.sourcedata = "%s %s %s" % (self.manifest_url, self.manifest_branch, self.manifest_file) 
 57          self.re_change = re.compile(".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") 
 58          self.re_head = re.compile("^HEAD is now at ([0-9a-f]+)...") 
 59   
 60 -    def _fullSrcdir(self): 
 61          return os.path.join(self.builder.basedir, self.srcdir) 
 62   
 63 -    def sourcedirIsUpdateable(self): 
 64          print os.path.join(self._fullSrcdir(), ".repo") 
 65          print os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 
 66          return os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 
 67   
 68 -    def _repoCmd(self, command, cb=None, abandonOnFailure=True, **kwargs): 
 69          repo = self.getCommand("repo") 
 70          c = runprocess.RunProcess(self.builder, [repo] + command, self._fullSrcdir(), 
 71                           sendRC=False, timeout=self.timeout, 
 72                           maxTime=self.maxTime, usePTY=False, 
 73                           logEnviron=self.logEnviron, **kwargs) 
 74          self.command = c 
 75          d = c.start() 
 76          if cb: 
 77              if abandonOnFailure: 
 78                  d.addCallback(self._abandonOnFailure) 
 79              d.addCallback(cb) 
 80          return d 
 81   
 82 -    def _Cmd(self, cmds, callback, abandonOnFailure=True): 
 83          c = runprocess.RunProcess(self.builder, cmds, self._fullSrcdir(), 
 84                                    sendRC=False, timeout=self.timeout, 
 85                                    maxTime=self.maxTime, usePTY=False, 
 86                                    logEnviron=self.logEnviron) 
 87          self.command = c 
 88          d = c.start() 
 89          if abandonOnFailure: 
 90              d.addCallback(self._abandonOnFailure) 
 91          d.addCallback(callback) 
 92          return d 
 93   
 94 -    def sourcedataMatches(self): 
 95          try: 
 96              olddata = self.readSourcedata() 
 97              return olddata == self.sourcedata 
 98          except IOError: 
 99              return False 
100   
101 -    def doVCFull(self): 
102          os.makedirs(self._fullSrcdir()) 
103          if self.tarball and os.path.exists(self.tarball): 
104              return self._Cmd(['tar', '-xvzf', self.tarball], self._doPreInitCleanUp) 
105          else: 
106              return self._doInit(None) 
107   
108 -    def _doInit(self,res): 
109          # on fresh init, this file may confuse repo. 
110          if os.path.exists(os.path.join(self._fullSrcdir(), ".repo/project.list")): 
111              os.unlink(os.path.join(self._fullSrcdir(), ".repo/project.list")) 
112          return self._repoCmd(['init', '-u', self.manifest_url, '-b', self.manifest_branch, '-m', self.manifest_file], self._didInit) 
113   
114 -    def _didInit(self, res): 
115          return self.doVCUpdate() 
116   
117 -    def doVCUpdate(self): 
118          if self.repo_downloads: 
119              self.sendStatus({'header': "will download:\n" + "repo download "+ "\nrepo download ".join(self.repo_downloads) + "\n"}) 
120          return self._doPreSyncCleanUp(None) 
121   
122      # a simple shell script to gather all cleanup tweaks... 
123      # doing them one by one just complicate the stuff 
124      # and messup the stdio log 
125 -    def _cleanupCommand(self): 
126          command = textwrap.dedent("""\ 
127              set -v 
128              if [ -d .repo/manifests ] 
129              then 
130                  # repo just refuse to run if manifest is messed up 
131                  # so ensure we are in a known state 
132                      cd .repo/manifests 
133                  git fetch origin 
134                  git reset --hard remotes/origin/%(manifest_branch)s 
135                  git config branch.default.merge %(manifest_branch)s 
136                  cd .. 
137                  ln -sf manifests/%(manifest_file)s manifest.xml 
138                  cd .. 
139               fi 
140               find . -name .git/index.lock -exec rm -f {} \; 
141                 repo forall -c git clean -f -d -x 2>/dev/null 
142                  repo forall -c git reset --hard HEAD 2>/dev/null 
143               """) % self.__dict__ 
144          return "\n".join([ s.strip() for s in command.splitlines()]) 
145       
146 -    def _doPreInitCleanUp(self, dummy): 
147          command = self._cleanupCommand() 
148          return self._Cmd(["bash", "-c", command], self._doInit, abandonOnFailure=False) 
149   
150 -    def _doPreSyncCleanUp(self, dummy): 
151          command = self._cleanupCommand() 
152          return self._Cmd(["bash", "-c", command], self._doManifestOveride, abandonOnFailure=False) 
153   
154 -    def _doManifestOveride(self, dummy): 
155          if self.manifest_override_url: 
156              self.sendStatus({"header": "overriding manifest with %s\n" %(self.manifest_override_url)}) 
157              if os.path.exists(os.path.join(self._fullSrcdir(), self.manifest_override_url)): 
158                  os.system("cd %s; cp -f %s manifest_override.xml"%(self._fullSrcdir(),self.manifest_override_url)) 
159              else: 
160                  command = ["wget", self.manifest_override_url, '-O', 'manifest_override.xml'] 
161                  return self._Cmd(command, self._doSync) 
162          return self._doSync(None) 
163   
164 -    def _doSync(self, dummy): 
165          if self.manifest_override_url: 
166              os.system("cd %s/.repo; ln -sf ../manifest_override.xml manifest.xml"%(self._fullSrcdir())) 
167          command = ['sync'] 
168          self.sendStatus({"header": "synching manifest %s from branch %s from %s\n" 
169                                     % (self.manifest_file, self.manifest_branch, self.manifest_url)}) 
170          return self._repoCmd(command, self._didSync) 
171   
172 -    def _didSync(self, dummy): 
173          if self.tarball and not os.path.exists(self.tarball): 
174              return self._Cmd(['tar', '-cvzf', self.tarball, ".repo"], self._doManifest) 
175          else: 
176              return self._doManifest(None) 
177   
178 -    def _doManifest(self, dummy): 
179          command = ['manifest', '-r', '-o', 'manifest-original.xml'] 
180          return self._repoCmd(command, self._doDownload, abandonOnFailure=False) 
181   
182   
183 -    def _doDownload(self, dummy): 
184          if hasattr(self.command, 'stderr') and self.command.stderr: 
185              if "Automatic cherry-pick failed" in self.command.stderr or "Automatic revert failed" in self.command.stderr: 
186                  command = ['forall','-c' ,'git' ,'diff', 'HEAD'] 
187                  self.cherry_pick_failed = True 
188                  return self._repoCmd(command, self._DownloadAbandon, abandonOnFailure = False, keepStderr=True) # call again 
189   
190              lines = self.command.stderr.split('\n') 
191              if len(lines) > 2: 
192                  match1 = self.re_change.match(lines[1]) 
193                  match2 = self.re_head.match(lines[-2]) 
194                  if match1 and match2: 
195                      self.repo_downloaded += "%s/%s %s " % (match1.group(1), match1.group(2), match2.group(1)) 
196   
197          if self.repo_downloads: 
198              # download each changeset while the self.download variable is not empty 
199              download = self.repo_downloads.pop(0) 
200              command = ['download'] + download.split(' ') 
201              self.sendStatus({"header": "downloading changeset %s\n" 
202                                         % (download)}) 
203              return self._repoCmd(command, self._doDownload, abandonOnFailure = False, keepStderr=True) # call again 
204   
205          if self.repo_downloaded: 
206              self.sendStatus({"repo_downloaded": self.repo_downloaded[:-1]}) 
207          return defer.succeed(0) 
208   
209 -    def maybeNotDoVCFallback(self, res): 
210          # If we were unable to find the branch/SHA on the remote, 
211          # clobbering the repo won't help any, so just abort the chain 
212          if hasattr(self.command, 'stderr'): 
213              if "Couldn't find remote ref" in self.command.stderr: 
214                  raise AbandonChain(-1) 
215              if hasattr(self, 'cherry_pick_failed') or "Automatic cherry-pick failed" in self.command.stderr: 
216                  raise AbandonChain(-1) 
217 -    def _DownloadAbandon(self,dummy): 
218          self.sendStatus({"header": "abandonned due to merge failure\n"}) 
219          raise AbandonChain(-1) 
220