buildslave.commands.repo

28 """Repo specific VC operation. In addition to the arguments 29 handled by SourceBaseCommand, this command reads the following keys: 30 31 ['manifest_url'] (required): The manifests repo repository. 32 ['manifest_branch'] (optional): Which manifest repo version (i.e. branch or tag) 33 to retrieve. Default: "master". 34 ['manifest_file'] (optional): Which manifest file to use. Default: "default.xml". 35 ['manifest_override_url'] (optional): Which manifest file to use as an overide. Default: None. 36 This is usually set by forced build to build over a known working base 37 ['tarball'] (optional): The tarball base to accelerate the fetch. 38 ['repo_downloads'] (optional): Repo downloads to do. Computer from GerritChangeSource 39 and forced build properties. 40 ['jobs'] (optional): number of connections to run in parallel 41 repo tool will use while syncing 42 """ 43 44 header = "repo operation" 45

46 - def setup(self, args):

47 SourceBaseCommand.setup(self, args) 48 self.manifest_url = args.get('manifest_url') 49 self.manifest_branch = args.get('manifest_branch') 50 self.manifest_file = args.get('manifest_file') 51 self.manifest_override_url = args.get('manifest_override_url') 52 self.tarball = args.get('tarball') 53 self.repo_downloads = args.get('repo_downloads') 54 # we're using string instead of an array here, because it will be transferred back 55 # to the master as string anyway and using eval() could have security implications. 56 self.repo_downloaded = "" 57 self.jobs = args.get('jobs') 58 59 self.sourcedata = "%s %s" % (self.manifest_url, self.manifest_file) 60 self.re_change = re.compile(".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") 61 self.re_head = re.compile("^HEAD is now at ([0-9a-f]+)...")

62

63 - def _fullSrcdir(self):

64 return os.path.join(self.builder.basedir, self.srcdir)

65

66 - def sourcedirIsUpdateable(self):

67 print os.path.join(self._fullSrcdir(), ".repo") 68 print os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")) 69 return os.path.isdir(os.path.join(self._fullSrcdir(), ".repo"))

70

71 - def _repoCmd(self, command, cb=None, abandonOnFailure=True, **kwargs):

72 repo = self.getCommand("repo") 73 c = runprocess.RunProcess(self.builder, [repo] + command, self._fullSrcdir(), 74 sendRC=False, timeout=self.timeout, 75 maxTime=self.maxTime, usePTY=False, 76 logEnviron=self.logEnviron, **kwargs) 77 self.command = c 78 d = c.start() 79 if cb: 80 if abandonOnFailure: 81 d.addCallback(self._abandonOnFailure) 82 d.addCallback(cb) 83 return d

84

85 - def _Cmd(self, cmds, callback, abandonOnFailure=True):

86 c = runprocess.RunProcess(self.builder, cmds, self._fullSrcdir(), 87 sendRC=False, timeout=self.timeout, 88 maxTime=self.maxTime, usePTY=False, 89 logEnviron=self.logEnviron) 90 self.command = c 91 d = c.start() 92 if abandonOnFailure: 93 d.addCallback(self._abandonOnFailure) 94 d.addCallback(callback) 95 return d

96

97 - def sourcedataMatches(self):

98 try: 99 olddata = self.readSourcedata() 100 return olddata == self.sourcedata 101 except IOError: 102 return False

103

104 - def doVCFull(self):

105 os.makedirs(self._fullSrcdir()) 106 if self.tarball and os.path.exists(self.tarball): 107 return self._Cmd(['tar', '-xvzf', self.tarball], self._doPreInitCleanUp) 108 else: 109 return self._doInit(None)

110

111 - def _doInit(self,res):

112 # on fresh init, this file may confuse repo. 113 if os.path.exists(os.path.join(self._fullSrcdir(), ".repo/project.list")): 114 os.unlink(os.path.join(self._fullSrcdir(), ".repo/project.list")) 115 return self._repoCmd(['init', '-u', self.manifest_url, '-b', self.manifest_branch, '-m', self.manifest_file], self._didInit)

116

117 - def _didInit(self, res):

118 return self.doVCUpdate()

119

120 - def doVCUpdate(self):

121 if self.repo_downloads: 122 self.sendStatus({'header': "will download:\n" + "repo download "+ "\nrepo download ".join(self.repo_downloads) + "\n"}) 123 return self._doPreSyncCleanUp(None)

124 125 # a simple shell script to gather all cleanup tweaks... 126 # doing them one by one just complicate the stuff 127 # and messup the stdio log

128 - def _cleanupCommand(self):

129 command = textwrap.dedent("""\ 130 set -v 131 if [ -d .repo/manifests ] 132 then 133 # repo just refuse to run if manifest is messed up 134 # so ensure we are in a known state 135 cd .repo/manifests 136 git fetch origin 137 git reset --hard remotes/origin/%(manifest_branch)s 138 git config branch.default.merge %(manifest_branch)s 139 cd .. 140 ln -sf manifests/%(manifest_file)s manifest.xml 141 cd .. 142 fi 143 repo forall -c rm -f .git/index.lock 144 repo forall -c git clean -f -d -x 2>/dev/null 145 repo forall -c git reset --hard HEAD 2>/dev/null 146 """) % self.__dict__ 147 return "\n".join([ s.strip() for s in command.splitlines()])

148

149 - def _doPreInitCleanUp(self, dummy):

150 command = self._cleanupCommand() 151 return self._Cmd(["bash", "-c", command], self._doInit, abandonOnFailure=False)

152

153 - def _doPreSyncCleanUp(self, dummy):

154 command = self._cleanupCommand() 155 return self._Cmd(["bash", "-c", command], self._doManifestOveride, abandonOnFailure=False)

156

157 - def _doManifestOveride(self, dummy):

158 if self.manifest_override_url: 159 self.sendStatus({"header": "overriding manifest with %s\n" %(self.manifest_override_url)}) 160 if os.path.exists(os.path.join(self._fullSrcdir(), self.manifest_override_url)): 161 os.system("cd %s; cp -f %s manifest_override.xml"%(self._fullSrcdir(),self.manifest_override_url)) 162 else: 163 command = ["wget", self.manifest_override_url, '-O', 'manifest_override.xml'] 164 return self._Cmd(command, self._doSync) 165 return self._doSync(None)

166

167 - def _doSync(self, dummy):

168 if self.manifest_override_url: 169 os.system("cd %s/.repo; ln -sf ../manifest_override.xml manifest.xml"%(self._fullSrcdir())) 170 command = ['sync'] 171 if self.jobs: 172 command.append('-j' + str(self.jobs)) 173 self.sendStatus({"header": "synching manifest %s from branch %s from %s\n" 174 % (self.manifest_file, self.manifest_branch, self.manifest_url)}) 175 return self._repoCmd(command, self._didSync)

176

177 - def _didSync(self, dummy):

178 if self.tarball and not os.path.exists(self.tarball): 179 return self._Cmd(['tar', '-cvzf', self.tarball, ".repo"], self._doManifest) 180 else: 181 return self._doManifest(None)

182

183 - def _doManifest(self, dummy):

184 command = ['manifest', '-r', '-o', 'manifest-original.xml'] 185 return self._repoCmd(command, self._doDownload, abandonOnFailure=False)

186 187

188 - def _doDownload(self, dummy):

189 if hasattr(self.command, 'stderr') and self.command.stderr: 190 if "Automatic cherry-pick failed" in self.command.stderr or "Automatic revert failed" in self.command.stderr: 191 command = ['forall','-c' ,'git' ,'diff', 'HEAD'] 192 self.cherry_pick_failed = True 193 return self._repoCmd(command, self._DownloadAbandon, abandonOnFailure = False, keepStderr=True) # call again 194 195 lines = self.command.stderr.split('\n') 196 if len(lines) > 2: 197 match1 = self.re_change.match(lines[1]) 198 match2 = self.re_head.match(lines[-2]) 199 if match1 and match2: 200 self.repo_downloaded += "%s/%s %s " % (match1.group(1), match1.group(2), match2.group(1)) 201 202 if self.repo_downloads: 203 # download each changeset while the self.download variable is not empty 204 download = self.repo_downloads.pop(0) 205 command = ['download'] + download.split(' ') 206 self.sendStatus({"header": "downloading changeset %s\n" 207 % (download)}) 208 return self._repoCmd(command, self._doDownload, abandonOnFailure = False, keepStderr=True) # call again 209 210 if self.repo_downloaded: 211 self.sendStatus({"repo_downloaded": self.repo_downloaded[:-1]}) 212 return defer.succeed(0)

213

214 - def maybeNotDoVCFallback(self, res):

215 # If we were unable to find the branch/SHA on the remote, 216 # clobbering the repo won't help any, so just abort the chain 217 if hasattr(self.command, 'stderr'): 218 if "Couldn't find remote ref" in self.command.stderr: 219 raise AbandonChain(-1) 220 if hasattr(self, 'cherry_pick_failed') or "Automatic cherry-pick failed" in self.command.stderr: 221 raise AbandonChain(-1)

222 - def _DownloadAbandon(self,dummy):

223 self.sendStatus({"header": "abandonned due to merge failure\n"}) 224 raise AbandonChain(-1)

Source Code for Module buildslave.commands.repo