Source Code for Module buildbot.schedulers.trysched

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import os 
 17   
 18  from twisted.internet import defer 
 19  from twisted.python import log 
 20  from twisted.protocols import basic 
 21   
 22  from buildbot import pbutil 
 23  from buildbot.util.maildir import MaildirService 
 24  from buildbot.util import json 
 25  from buildbot.util import netstrings 
 26  from buildbot.process.properties import Properties 
 27  from buildbot.schedulers import base 
 28  from buildbot.status.buildset import BuildSetStatus 
 29   
 30   
 31 -class TryBase(base.BaseScheduler): 
 32   
 33 -    def filterBuilderList(self, builderNames): 
 34          """ 
 35          Make sure that C{builderNames} is a subset of the configured 
 36          C{self.builderNames}, returning an empty list if not.  If 
 37          C{builderNames} is empty, use C{self.builderNames}. 
 38   
 39          @returns: list of builder names to build on 
 40          """ 
 41   
 42          # self.builderNames is the configured list of builders 
 43          # available for try.  If the user supplies a list of builders, 
 44          # it must be restricted to the configured list.  If not, build 
 45          # on all of the configured builders. 
 46          if builderNames: 
 47              for b in builderNames: 
 48                  if not b in self.builderNames: 
 49                      log.msg("%s got with builder %s" % (self, b)) 
 50                      log.msg(" but that wasn't in our list: %s" 
 51                              % (self.builderNames,)) 
 52                      return [] 
 53          else: 
 54              builderNames = self.builderNames 
 55          return builderNames 
 56   
 57   
 58 -class BadJobfile(Exception): 
 59      pass 
 60   
 61   
 62 -class JobdirService(MaildirService): 
 63      # NOTE: tightly coupled with Try_Jobdir, below 
 64   
 65 -    def messageReceived(self, filename): 
 66          f = self.moveToCurDir(filename) 
 67          return self.parent.handleJobFile(filename, f) 
 68   
 69   
 70 -class Try_Jobdir(TryBase): 
 71   
 72      compare_attrs = TryBase.compare_attrs + ('jobdir',) 
 73   
 74 -    def __init__(self, name, builderNames, jobdir, 
 75                   properties={}): 
 76          TryBase.__init__(self, name=name, builderNames=builderNames, 
 77                           properties=properties) 
 78          self.jobdir = jobdir 
 79          self.watcher = JobdirService() 
 80          self.watcher.setServiceParent(self) 
 81   
 82 -    def startService(self): 
 83          # set the watcher's basedir now that we have a master 
 84          jobdir = os.path.join(self.master.basedir, self.jobdir) 
 85          self.watcher.setBasedir(jobdir) 
 86          for subdir in "cur new tmp".split(): 
 87              if not os.path.exists(os.path.join(jobdir, subdir)): 
 88                  os.mkdir(os.path.join(jobdir, subdir)) 
 89          TryBase.startService(self) 
 90   
 91 -    def parseJob(self, f): 
 92          # jobfiles are serialized build requests. Each is a list of 
 93          # serialized netstrings, in the following order: 
 94          #  format version number: 
 95          #  "1" the original 
 96          #  "2" introduces project and repository 
 97          #  "3" introduces who 
 98          #  "4" introduces comment 
 99          #  "5" introduces properties and JSON serialization of values after 
100          #      version 
101          #  jobid: arbitrary string, used to find the buildSet later 
102          #  branch: branch name, "" for default-branch 
103          #  baserev: revision, "" for HEAD 
104          #  patch_level: usually "1" 
105          #  patch_body: patch to be applied for build 
106          #  repository 
107          #  project 
108          #  who: user requesting build 
109          #  comment: comment from user about diff and/or build 
110          #  builderNames: list of builder names 
111          #  properties: dict of build properties 
112          p = netstrings.NetstringParser() 
113          f.seek(0,2) 
114          if f.tell() > basic.NetstringReceiver.MAX_LENGTH: 
115              raise BadJobfile("The patch size is greater that NetStringReceiver.MAX_LENGTH. Please Set this higher in the master.cfg") 
116          f.seek(0,0) 
117          try: 
118              p.feed(f.read()) 
119          except basic.NetstringParseError: 
120              raise BadJobfile("unable to parse netstrings") 
121          if not p.strings: 
122              raise BadJobfile("could not find any complete netstrings") 
123          ver = p.strings.pop(0) 
124   
125          v1_keys = ['jobid', 'branch', 'baserev', 'patch_level', 'patch_body'] 
126          v2_keys = v1_keys + ['repository', 'project'] 
127          v3_keys = v2_keys + ['who'] 
128          v4_keys = v3_keys + ['comment'] 
129          keys = [v1_keys, v2_keys, v3_keys, v4_keys] 
130          # v5 introduces properties and uses JSON serialization 
131   
132          parsed_job = {} 
133   
134          def extract_netstrings(p, keys): 
135              for i, key in enumerate(keys): 
136                  parsed_job[key] = p.strings[i] 
137   
138          def postprocess_parsed_job(): 
139              # apply defaults and handle type casting 
140              parsed_job['branch'] = parsed_job['branch'] or None 
141              parsed_job['baserev'] = parsed_job['baserev'] or None 
142              parsed_job['patch_level'] = int(parsed_job['patch_level']) 
143              for key in 'repository project who comment'.split(): 
144                  parsed_job[key] = parsed_job.get(key, '') 
145              parsed_job['properties'] = parsed_job.get('properties', {}) 
146   
147          if ver <= "4": 
148              i = int(ver) - 1 
149              extract_netstrings(p, keys[i]) 
150              parsed_job['builderNames'] = p.strings[len(keys[i]):] 
151              postprocess_parsed_job() 
152          elif ver == "5": 
153              try: 
154                  parsed_job = json.loads(p.strings[0]) 
155              except ValueError: 
156                  raise BadJobfile("unable to parse JSON") 
157              postprocess_parsed_job() 
158          else: 
159              raise BadJobfile("unknown version '%s'" % ver) 
160          return parsed_job 
161   
162 -    def handleJobFile(self, filename, f): 
163          try: 
164              parsed_job = self.parseJob(f) 
165              builderNames = parsed_job['builderNames'] 
166          except BadJobfile: 
167              log.msg("%s reports a bad jobfile in %s" % (self, filename)) 
168              log.err() 
169              return defer.succeed(None) 
170   
171          # Validate/fixup the builder names. 
172          builderNames = self.filterBuilderList(builderNames) 
173          if not builderNames: 
174              log.msg( 
175                  "incoming Try job did not specify any allowed builder names") 
176              return defer.succeed(None) 
177   
178          who = "" 
179          if parsed_job['who']: 
180              who = parsed_job['who'] 
181   
182          comment = "" 
183          if parsed_job['comment']: 
184              comment = parsed_job['comment'] 
185   
186          d = self.master.db.sourcestampsets.addSourceStampSet() 
187   
188          def addsourcestamp(setid): 
189              self.master.db.sourcestamps.addSourceStamp( 
190                  sourcestampsetid=setid, 
191                  branch=parsed_job['branch'], 
192                  revision=parsed_job['baserev'], 
193                  patch_body=parsed_job['patch_body'], 
194                  patch_level=parsed_job['patch_level'], 
195                  patch_author=who, 
196                  patch_comment=comment, 
197                  patch_subdir='',  # TODO: can't set this remotely - #1769 
198                  project=parsed_job['project'], 
199                  repository=parsed_job['repository']) 
200              return setid 
201   
202          d.addCallback(addsourcestamp) 
203   
204          def create_buildset(setid): 
205              reason = "'try' job" 
206              if parsed_job['who']: 
207                  reason += " by user %s" % parsed_job['who'] 
208              properties = parsed_job['properties'] 
209              requested_props = Properties() 
210              requested_props.update(properties, "try build") 
211              return self.addBuildsetForSourceStamp( 
212                  ssid=None, setid=setid, 
213                  reason=reason, external_idstring=parsed_job['jobid'], 
214                  builderNames=builderNames, properties=requested_props) 
215          d.addCallback(create_buildset) 
216          return d 
217   
218   
219 -class Try_Userpass_Perspective(pbutil.NewCredPerspective): 
220 -    def __init__(self, scheduler, username): 
221          self.scheduler = scheduler 
222          self.username = username 
223   
224      @defer.inlineCallbacks 
225 -    def perspective_try(self, branch, revision, patch, repository, project, 
226                          builderNames, who="", comment="", properties={}): 
227          db = self.scheduler.master.db 
228          log.msg("user %s requesting build on builders %s" % (self.username, 
229                                                               builderNames)) 
230   
231          # build the intersection of the request and our configured list 
232          builderNames = self.scheduler.filterBuilderList(builderNames) 
233          if not builderNames: 
234              return 
235   
236          reason = "'try' job" 
237   
238          if who: 
239              reason += " by user %s" % who 
240   
241          if comment: 
242              reason += " (%s)" % comment 
243   
244          sourcestampsetid = yield db.sourcestampsets.addSourceStampSet() 
245   
246          yield db.sourcestamps.addSourceStamp( 
247              branch=branch, revision=revision, repository=repository, 
248              project=project, patch_level=patch[0], patch_body=patch[1], 
249              patch_subdir='', patch_author=who or '', 
250              patch_comment=comment or '', 
251              sourcestampsetid=sourcestampsetid) 
252                      # note: no way to specify patch subdir - #1769 
253   
254          requested_props = Properties() 
255          requested_props.update(properties, "try build") 
256          (bsid, brids) = yield self.scheduler.addBuildsetForSourceStamp( 
257                  setid=sourcestampsetid, reason=reason, 
258                  properties=requested_props, builderNames=builderNames) 
259   
260          # return a remotely-usable BuildSetStatus object 
261          bsdict = yield db.buildsets.getBuildset(bsid) 
262   
263          bss = BuildSetStatus(bsdict, self.scheduler.master.status) 
264          from buildbot.status.client import makeRemote 
265          defer.returnValue(makeRemote(bss)) 
266   
267 -    def perspective_getAvailableBuilderNames(self): 
268          # Return a list of builder names that are configured 
269          # for the try service 
270          # This is mostly intended for integrating try services 
271          # into other applications 
272          return self.scheduler.listBuilderNames() 
273   
274   
275 -class Try_Userpass(TryBase): 
276      compare_attrs = ('name', 'builderNames', 'port', 'userpass', 'properties') 
277   
278 -    def __init__(self, name, builderNames, port, userpass, 
279                   properties={}): 
280          TryBase.__init__(self, name=name, builderNames=builderNames, 
281                           properties=properties) 
282          self.port = port 
283          self.userpass = userpass 
284   
285 -    def startService(self): 
286          TryBase.startService(self) 
287   
288          # register each user/passwd with the pbmanager 
289          def factory(mind, username): 
290              return Try_Userpass_Perspective(self, username) 
291          self.registrations = [] 
292          for user, passwd in self.userpass: 
293              self.registrations.append( 
294                  self.master.pbmanager.register( 
295                      self.port, user, passwd, factory)) 
296   
297 -    def stopService(self): 
298          d = defer.maybeDeferred(TryBase.stopService, self) 
299   
300          def unreg(_): 
301              return defer.gatherResults( 
302                  [reg.unregister() for reg in self.registrations]) 
303          d.addCallback(unreg) 
304          return d 
305