Package buildslave :: Package commands :: Module repo
[frames] | no frames]

Source Code for Module buildslave.commands.repo

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import os 
 17  import re 
 18  import textwrap 
 19   
 20  from twisted.internet import defer 
 21   
 22  from buildslave.commands.base import SourceBaseCommand 
 23  from buildslave import runprocess 
 24  from buildslave.commands.base import AbandonChain 
 25   
 26   
27 -class Repo(SourceBaseCommand):
28 """Repo specific VC operation. In addition to the arguments 29 handled by SourceBaseCommand, this command reads the following keys: 30 31 ['manifest_url'] (required): The manifests repo repository. 32 ['manifest_branch'] (optional): Which manifest repo version (i.e. branch or tag) 33 to retrieve. Default: "master". 34 ['manifest_file'] (optional): Which manifest file to use. Default: "default.xml". 35 ['manifest_override_url'] (optional): Which manifest file to use as an overide. Default: None. 36 This is usually set by forced build to build over a known working base 37 ['tarball'] (optional): The tarball base to accelerate the fetch. 38 ['repo_downloads'] (optional): Repo downloads to do. Computer from GerritChangeSource 39 and forced build properties. 40 """ 41 42 header = "repo operation" 43
44 - def setup(self, args):
45 SourceBaseCommand.setup(self, args) 46 self.manifest_url = args.get('manifest_url') 47 self.manifest_branch = args.get('manifest_branch') 48 self.manifest_file = args.get('manifest_file') 49 self.manifest_override_url = args.get('manifest_override_url') 50 self.tarball = args.get('tarball') 51 self.repo_downloads = args.get('repo_downloads') 52 # we're using string instead of an array here, because it will be transferred back 53 # to the master as string anyway and using eval() could have security implications. 54 self.repo_downloaded = "" 55 56 self.sourcedata = "%s %s %s" % (self.manifest_url, self.manifest_branch, self.manifest_file) 57 self.re_change = re.compile(".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") 58 self.re_head = re.compile("^HEAD is now at ([0-9a-f]+)...")
59
60 - def _fullSrcdir(self):
61 return os.path.join(self.builder.basedir, self.srcdir)
62
63 - def sourcedirIsUpdateable(self):
64 print os.path.join(self._fullSrcdir(), ".repo") 65 print os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 66 return os.path.isdir(os.path.join(self._fullSrcdir(), ".repo"))
67
68 - def _repoCmd(self, command, cb=None, abandonOnFailure=True, **kwargs):
69 repo = self.getCommand("repo") 70 c = runprocess.RunProcess(self.builder, [repo] + command, self._fullSrcdir(), 71 sendRC=False, timeout=self.timeout, 72 maxTime=self.maxTime, usePTY=False, 73 logEnviron=self.logEnviron, **kwargs) 74 self.command = c 75 d = c.start() 76 if cb: 77 if abandonOnFailure: 78 d.addCallback(self._abandonOnFailure) 79 d.addCallback(cb) 80 return d
81
82 - def _Cmd(self, cmds, callback, abandonOnFailure=True):
83 c = runprocess.RunProcess(self.builder, cmds, self._fullSrcdir(), 84 sendRC=False, timeout=self.timeout, 85 maxTime=self.maxTime, usePTY=False, 86 logEnviron=self.logEnviron) 87 self.command = c 88 d = c.start() 89 if abandonOnFailure: 90 d.addCallback(self._abandonOnFailure) 91 d.addCallback(callback) 92 return d
93
94 - def sourcedataMatches(self):
95 try: 96 olddata = self.readSourcedata() 97 return olddata == self.sourcedata 98 except IOError: 99 return False
100
101 - def doVCFull(self):
102 os.makedirs(self._fullSrcdir()) 103 if self.tarball and os.path.exists(self.tarball): 104 return self._Cmd(['tar', '-xvzf', self.tarball], self._doPreInitCleanUp) 105 else: 106 return self._doInit(None)
107
108 - def _doInit(self,res):
109 # on fresh init, this file may confuse repo. 110 if os.path.exists(os.path.join(self._fullSrcdir(), ".repo/project.list")): 111 os.unlink(os.path.join(self._fullSrcdir(), ".repo/project.list")) 112 return self._repoCmd(['init', '-u', self.manifest_url, '-b', self.manifest_branch, '-m', self.manifest_file], self._didInit)
113
114 - def _didInit(self, res):
115 return self.doVCUpdate()
116
117 - def doVCUpdate(self):
118 if self.repo_downloads: 119 self.sendStatus({'header': "will download:\n" + "repo download "+ "\nrepo download ".join(self.repo_downloads) + "\n"}) 120 return self._doPreSyncCleanUp(None)
121 122 # a simple shell script to gather all cleanup tweaks... 123 # doing them one by one just complicate the stuff 124 # and messup the stdio log
125 - def _cleanupCommand(self):
126 command = textwrap.dedent("""\ 127 set -v 128 if [ -d .repo/manifests ] 129 then 130 # repo just refuse to run if manifest is messed up 131 # so ensure we are in a known state 132 cd .repo/manifests 133 git fetch origin 134 git reset --hard remotes/origin/%(manifest_branch)s 135 git config branch.default.merge %(manifest_branch)s 136 cd .. 137 ln -sf manifests/%(manifest_file)s manifest.xml 138 cd .. 139 fi 140 find . -name .git/index.lock -exec rm -f {} \; 141 repo forall -c git clean -f -d -x 2>/dev/null 142 repo forall -c git reset --hard HEAD 2>/dev/null 143 """) % self.__dict__ 144 return "\n".join([ s.strip() for s in command.splitlines()])
145
146 - def _doPreInitCleanUp(self, dummy):
147 command = self._cleanupCommand() 148 return self._Cmd(["bash", "-c", command], self._doInit, abandonOnFailure=False)
149
150 - def _doPreSyncCleanUp(self, dummy):
151 command = self._cleanupCommand() 152 return self._Cmd(["bash", "-c", command], self._doManifestOveride, abandonOnFailure=False)
153
154 - def _doManifestOveride(self, dummy):
155 if self.manifest_override_url: 156 self.sendStatus({"header": "overriding manifest with %s\n" %(self.manifest_override_url)}) 157 if os.path.exists(os.path.join(self._fullSrcdir(), self.manifest_override_url)): 158 os.system("cd %s; cp -f %s manifest_override.xml"%(self._fullSrcdir(),self.manifest_override_url)) 159 else: 160 command = ["wget", self.manifest_override_url, '-O', 'manifest_override.xml'] 161 return self._Cmd(command, self._doSync) 162 return self._doSync(None)
163
164 - def _doSync(self, dummy):
165 if self.manifest_override_url: 166 os.system("cd %s/.repo; ln -sf ../manifest_override.xml manifest.xml"%(self._fullSrcdir())) 167 command = ['sync'] 168 self.sendStatus({"header": "synching manifest %s from branch %s from %s\n" 169 % (self.manifest_file, self.manifest_branch, self.manifest_url)}) 170 return self._repoCmd(command, self._didSync)
171
172 - def _didSync(self, dummy):
173 if self.tarball and not os.path.exists(self.tarball): 174 return self._Cmd(['tar', '-cvzf', self.tarball, ".repo"], self._doManifest) 175 else: 176 return self._doManifest(None)
177
178 - def _doManifest(self, dummy):
179 command = ['manifest', '-r', '-o', 'manifest-original.xml'] 180 return self._repoCmd(command, self._doDownload, abandonOnFailure=False)
181 182
183 - def _doDownload(self, dummy):
184 if hasattr(self.command, 'stderr') and self.command.stderr: 185 if "Automatic cherry-pick failed" in self.command.stderr or "Automatic revert failed" in self.command.stderr: 186 command = ['forall','-c' ,'git' ,'diff', 'HEAD'] 187 self.cherry_pick_failed = True 188 return self._repoCmd(command, self._DownloadAbandon, abandonOnFailure = False, keepStderr=True) # call again 189 190 lines = self.command.stderr.split('\n') 191 if len(lines) > 2: 192 match1 = self.re_change.match(lines[1]) 193 match2 = self.re_head.match(lines[-2]) 194 if match1 and match2: 195 self.repo_downloaded += "%s/%s %s " % (match1.group(1), match1.group(2), match2.group(1)) 196 197 if self.repo_downloads: 198 # download each changeset while the self.download variable is not empty 199 download = self.repo_downloads.pop(0) 200 command = ['download'] + download.split(' ') 201 self.sendStatus({"header": "downloading changeset %s\n" 202 % (download)}) 203 return self._repoCmd(command, self._doDownload, abandonOnFailure = False, keepStderr=True) # call again 204 205 if self.repo_downloaded: 206 self.sendStatus({"repo_downloaded": self.repo_downloaded[:-1]}) 207 return defer.succeed(0)
208
209 - def maybeNotDoVCFallback(self, res):
210 # If we were unable to find the branch/SHA on the remote, 211 # clobbering the repo won't help any, so just abort the chain 212 if hasattr(self.command, 'stderr'): 213 if "Couldn't find remote ref" in self.command.stderr: 214 raise AbandonChain(-1) 215 if hasattr(self, 'cherry_pick_failed') or "Automatic cherry-pick failed" in self.command.stderr: 216 raise AbandonChain(-1)
217 - def _DownloadAbandon(self,dummy):
218 self.sendStatus({"header": "abandonned due to merge failure\n"}) 219 raise AbandonChain(-1)
220