Source Code for Module buildbot.db.buildrequests

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import itertools 
 17  import sqlalchemy as sa 
 18  from twisted.internet import reactor 
 19  from twisted.python import log 
 20  from buildbot.db import base 
 21  from buildbot.util import epoch2datetime, datetime2epoch 
 22   
 23 -class AlreadyClaimedError(Exception): 
 24      pass 
 25   
 26 -class NotClaimedError(Exception): 
 27      pass 
 28   
 29 -class BrDict(dict): 
 30      pass 
 31   
 32  # private decorator to add a _master_objectid keyword argument, querying from 
 33  # the master 
 34 -def with_master_objectid(fn): 
 35      def wrap(self, *args, **kwargs): 
 36          d = self.db.master.getObjectId() 
 37          d.addCallback(lambda master_objectid : 
 38                  fn(self, _master_objectid=master_objectid, *args, **kwargs)) 
 39          return d 
 40      wrap.__name__ = fn.__name__ 
 41      wrap.__doc__ = fn.__doc__ 
 42      return wrap 
 43   
 44 -class BuildRequestsConnectorComponent(base.DBConnectorComponent): 
 45      # Documentation is in developer/database.rst 
 46   
 47      @with_master_objectid 
 48 -    def getBuildRequest(self, brid, _master_objectid=None): 
 49          def thd(conn): 
 50              reqs_tbl = self.db.model.buildrequests 
 51              claims_tbl = self.db.model.buildrequest_claims 
 52              res = conn.execute(sa.select([ 
 53                  reqs_tbl.outerjoin(claims_tbl, 
 54                                     (reqs_tbl.c.id == claims_tbl.c.brid)) ], 
 55                  whereclause=(reqs_tbl.c.id == brid))) 
 56              row = res.fetchone() 
 57   
 58              rv = None 
 59              if row: 
 60                  rv = self._brdictFromRow(row, _master_objectid) 
 61              res.close() 
 62              return rv 
 63          return self.db.pool.do(thd) 
 64   
 65      @with_master_objectid 
 66 -    def getBuildRequests(self, buildername=None, complete=None, claimed=None, 
 67              bsid=None, _master_objectid=None): 
 68          def thd(conn): 
 69              reqs_tbl = self.db.model.buildrequests 
 70              claims_tbl = self.db.model.buildrequest_claims 
 71              q = sa.select([ reqs_tbl.outerjoin(claims_tbl, 
 72                                      reqs_tbl.c.id == claims_tbl.c.brid) ]) 
 73              if claimed is not None: 
 74                  if not claimed: 
 75                      q = q.where( 
 76                          (claims_tbl.c.claimed_at == None) & 
 77                          (reqs_tbl.c.complete == 0)) 
 78                  elif claimed == "mine": 
 79                      q = q.where( 
 80                          (claims_tbl.c.objectid == _master_objectid)) 
 81                  else: 
 82                      q = q.where( 
 83                          (claims_tbl.c.claimed_at != None)) 
 84              if buildername is not None: 
 85                  q = q.where(reqs_tbl.c.buildername == buildername) 
 86              if complete is not None: 
 87                  if complete: 
 88                      q = q.where(reqs_tbl.c.complete != 0) 
 89                  else: 
 90                      q = q.where(reqs_tbl.c.complete == 0) 
 91              if bsid is not None: 
 92                  q = q.where(reqs_tbl.c.buildsetid == bsid) 
 93              res = conn.execute(q) 
 94   
 95              return [ self._brdictFromRow(row, _master_objectid) 
 96                       for row in res.fetchall() ] 
 97          return self.db.pool.do(thd) 
 98   
 99      @with_master_objectid 
100 -    def claimBuildRequests(self, brids, claimed_at=None, _reactor=reactor, 
101                              _master_objectid=None): 
102          if claimed_at is not None: 
103              claimed_at = datetime2epoch(claimed_at) 
104          else: 
105              claimed_at = _reactor.seconds() 
106   
107          def thd(conn): 
108              transaction = conn.begin() 
109              tbl = self.db.model.buildrequest_claims 
110   
111              try: 
112                  q = tbl.insert() 
113                  conn.execute(q, [ dict(brid=id, objectid=_master_objectid, 
114                                      claimed_at=claimed_at) 
115                                    for id in brids ]) 
116              except (sa.exc.IntegrityError, sa.exc.ProgrammingError): 
117                  transaction.rollback() 
118                  raise AlreadyClaimedError 
119   
120              transaction.commit() 
121   
122          return self.db.pool.do(thd) 
123   
124      @with_master_objectid 
125 -    def reclaimBuildRequests(self, brids, _reactor=reactor, 
126                              _master_objectid=None): 
127          def thd(conn): 
128              transaction = conn.begin() 
129              tbl = self.db.model.buildrequest_claims 
130              claimed_at = _reactor.seconds() 
131   
132              # we'll need to batch the brids into groups of 100, so that the 
133              # parameter lists supported by the DBAPI aren't exhausted 
134              iterator = iter(brids) 
135   
136              while 1: 
137                  batch = list(itertools.islice(iterator, 100)) 
138                  if not batch: 
139                      break # success! 
140   
141                  q = tbl.update(tbl.c.brid.in_(batch) 
142                                  & (tbl.c.objectid==_master_objectid)) 
143                  res = conn.execute(q, claimed_at=claimed_at) 
144   
145                  # if fewer rows were updated than expected, then something 
146                  # went wrong 
147                  if res.rowcount != len(batch): 
148                      transaction.rollback() 
149                      raise AlreadyClaimedError 
150   
151              transaction.commit() 
152          return self.db.pool.do(thd) 
153   
154      @with_master_objectid 
155 -    def unclaimBuildRequests(self, brids, _master_objectid=None): 
156          def thd(conn): 
157              transaction = conn.begin() 
158              claims_tbl = self.db.model.buildrequest_claims 
159   
160              # we'll need to batch the brids into groups of 100, so that the 
161              # parameter lists supported by the DBAPI aren't exhausted 
162              iterator = iter(brids) 
163   
164              while 1: 
165                  batch = list(itertools.islice(iterator, 100)) 
166                  if not batch: 
167                      break # success! 
168   
169                  try: 
170                      q = claims_tbl.delete( 
171                              (claims_tbl.c.brid.in_(batch)) 
172                              & (claims_tbl.c.objectid == _master_objectid)) 
173                      conn.execute(q) 
174                  except: 
175                      transaction.rollback() 
176                      raise 
177   
178              transaction.commit() 
179          return self.db.pool.do(thd) 
180   
181      @with_master_objectid 
182 -    def completeBuildRequests(self, brids, results, complete_at=None, 
183                              _reactor=reactor, _master_objectid=None): 
184          if complete_at is not None: 
185              complete_at = datetime2epoch(complete_at) 
186          else: 
187              complete_at = _reactor.seconds() 
188   
189          def thd(conn): 
190              transaction = conn.begin() 
191   
192              # the update here is simple, but a number of conditions are 
193              # attached to ensure that we do not update a row inappropriately, 
194              # Note that checking that the request is mine would require a 
195              # subquery, so for efficiency that is not checed. 
196   
197              reqs_tbl = self.db.model.buildrequests 
198   
199              # we'll need to batch the brids into groups of 100, so that the 
200              # parameter lists supported by the DBAPI aren't exhausted 
201              iterator = iter(brids) 
202   
203              while 1: 
204                  batch = list(itertools.islice(iterator, 100)) 
205                  if not batch: 
206                      break # success! 
207   
208                  q = reqs_tbl.update() 
209                  q = q.where(reqs_tbl.c.id.in_(batch)) 
210                  q = q.where(reqs_tbl.c.complete != 1) 
211                  res = conn.execute(q, 
212                      complete=1, 
213                      results=results, 
214                      complete_at=complete_at) 
215   
216                  # if an incorrect number of rows were updated, then we failed. 
217                  if res.rowcount != len(batch): 
218                      log.msg("tried to complete %d buildreqests, " 
219                          "but only completed %d" % (len(batch), res.rowcount)) 
220                      transaction.rollback() 
221                      raise NotClaimedError 
222              transaction.commit() 
223          return self.db.pool.do(thd) 
224   
225 -    def unclaimExpiredRequests(self, old, _reactor=reactor): 
226          def thd(conn): 
227              reqs_tbl = self.db.model.buildrequests 
228              claims_tbl = self.db.model.buildrequest_claims 
229              old_epoch = _reactor.seconds() - old 
230   
231              # select any expired requests, and delete each one individually 
232              expired_brids = sa.select([ reqs_tbl.c.id ], 
233                          whereclause=(reqs_tbl.c.complete != 1)) 
234              res = conn.execute(claims_tbl.delete( 
235                          (claims_tbl.c.claimed_at < old_epoch) & 
236                          claims_tbl.c.brid.in_(expired_brids))) 
237              return res.rowcount 
238          d = self.db.pool.do(thd) 
239          def log_nonzero_count(count): 
240              if count != 0: 
241                  log.msg("unclaimed %d expired buildrequests (over %d seconds " 
242                          "old)" % (count, old)) 
243          d.addCallback(log_nonzero_count) 
244          return d 
245   
246 -    def _brdictFromRow(self, row, master_objectid): 
247          claimed = mine = False 
248          claimed_at = None 
249          if row.claimed_at is not None: 
250              claimed_at = row.claimed_at 
251              claimed = True 
252              mine = row.objectid == master_objectid 
253   
254          def mkdt(epoch): 
255              if epoch: 
256                  return epoch2datetime(epoch) 
257          submitted_at = mkdt(row.submitted_at) 
258          complete_at = mkdt(row.complete_at) 
259          claimed_at = mkdt(row.claimed_at) 
260   
261          return BrDict(brid=row.id, buildsetid=row.buildsetid, 
262                  buildername=row.buildername, priority=row.priority, 
263                  claimed=claimed, claimed_at=claimed_at, mine=mine, 
264                  complete=bool(row.complete), results=row.results, 
265                  submitted_at=submitted_at, complete_at=complete_at) 
266