1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 """
17 Support for buildsets in the database
18 """
19
20 import itertools
21 import sqlalchemy as sa
22 from twisted.internet import reactor
23 from twisted.python import log
24 from buildbot.db import base
25 from buildbot.util import epoch2datetime
29
32
35
39 def wrap(self, *args, **kwargs):
40 d = self.db.master.getObjectId()
41 d.addCallback(lambda master_objectid :
42 fn(self, *args, _master_objectid=master_objectid, **kwargs))
43 return d
44 wrap.__name__ = fn.__name__
45 wrap.__doc__ = fn.__doc__
46 return wrap
47
49 """
50 A DBConnectorComponent to handle buildrequests. An instance is available
51 at C{master.db.buildrequests}.
52
53 Build Requests are represented as dictionaries with keys C{brid},
54 C{buildsetid}, C{buildername}, C{priority}, C{claimed} (boolean),
55 C{claimed_at}, C{mine} (boolean), C{complete}, C{results}, C{submitted_at},
56 and C{complete_at}. The two time parameters (C{*_at}) are presented as
57 datetime objects.
58 """
59
60 @with_master_objectid
62 """
63 Get a single BuildRequest, in the format described above. Returns
64 C{None} if there is no such buildrequest.
65
66 Note that build requests are not cached, as the values in the database
67 are not fixed.
68
69 @param brid: build request id
70 @type brid: integer
71
72 @returns: Build request dictionary as above or None, via Deferred
73 """
74 def thd(conn):
75 reqs_tbl = self.db.model.buildrequests
76 claims_tbl = self.db.model.buildrequest_claims
77 res = conn.execute(sa.select([
78 reqs_tbl.outerjoin(claims_tbl,
79 (reqs_tbl.c.id == claims_tbl.c.brid)) ],
80 whereclause=(reqs_tbl.c.id == brid)))
81 row = res.fetchone()
82
83 rv = None
84 if row:
85 rv = self._brdictFromRow(row, _master_objectid)
86 res.close()
87 return rv
88 return self.db.pool.do(thd)
89
90 @with_master_objectid
91 - def getBuildRequests(self, buildername=None, complete=None, claimed=None,
92 bsid=None, _master_objectid=None):
93 """
94 Get a list of build requests matching the given characteristics. Note
95 that C{unclaimed}, C{my_claimed}, and C{other_claimed} all default to
96 C{False}, so at least one must be provided or no results will be
97 returned.
98
99 The C{claimed} parameter can be C{None} (the default) to ignore the
100 claimed status of requests; C{True} to return only claimed builds,
101 C{False} to return only unclaimed builds, or C{"mine"} to return only
102 builds claimed by this master instance. A request is considered
103 unclaimed if its C{claimed_at} column is either NULL or 0, and it is
104 not complete. If C{bsid} is specified, then only build requests for
105 that buildset will be returned.
106
107 A build is considered completed if its C{complete} column is 1; the
108 C{complete_at} column is not consulted.
109
110 Since this method is often used to detect changed build requests, it
111 always bypasses the cache.
112
113 @param buildername: limit results to buildrequests for this builder
114 @type buildername: string
115
116 @param complete: if true, limit to completed buildrequests; if false,
117 limit to incomplete buildrequests; if None, do not limit based on
118 completion.
119
120 @param claimed: see above
121
122 @param bsid: see above
123
124 @returns: List of build request dictionaries as above, via Deferred
125 """
126 def thd(conn):
127 reqs_tbl = self.db.model.buildrequests
128 claims_tbl = self.db.model.buildrequest_claims
129 q = sa.select([ reqs_tbl.outerjoin(claims_tbl,
130 reqs_tbl.c.id == claims_tbl.c.brid) ])
131 if claimed is not None:
132 if not claimed:
133 q = q.where(
134 (claims_tbl.c.claimed_at == None) &
135 (reqs_tbl.c.complete == 0))
136 elif claimed == "mine":
137 q = q.where(
138 (claims_tbl.c.objectid == _master_objectid))
139 else:
140 q = q.where(
141 (claims_tbl.c.claimed_at != None))
142 if buildername is not None:
143 q = q.where(reqs_tbl.c.buildername == buildername)
144 if complete is not None:
145 if complete:
146 q = q.where(reqs_tbl.c.complete != 0)
147 else:
148 q = q.where(reqs_tbl.c.complete == 0)
149 if bsid is not None:
150 q = q.where(reqs_tbl.c.buildsetid == bsid)
151 res = conn.execute(q)
152
153 return [ self._brdictFromRow(row, _master_objectid)
154 for row in res.fetchall() ]
155 return self.db.pool.do(thd)
156
157 @with_master_objectid
160 """
161 Try to "claim" the indicated build requests for this buildmaster
162 instance. The resulting deferred will fire normally on success, or
163 fail with L{AleadyClaimedError} if I{any} of the build requests are
164 already claimed by another master instance. In this case, none of the
165 claims will take effect.
166
167 As of 0.8.5, this method can no longer be used to re-claim build
168 requests. All given brids must be unclaimed. Use
169 L{reclaimBuildRequests} to reclaim.
170
171 On database backends that do not enforce referential integrity (e.g.,
172 SQLite), this method will not prevent claims for nonexistent build
173 requests. On database backends that do not support transactions
174 (MySQL), this method will not properly roll back any partial claims
175 made before an L{AlreadyClaimedError} was generated.
176
177 @param brids: ids of buildrequests to claim
178 @type brids: list
179
180 @param _reactor: reactor to use (for testing)
181
182 @returns: Deferred
183 """
184
185 def thd(conn):
186 transaction = conn.begin()
187 tbl = self.db.model.buildrequest_claims
188
189 try:
190 q = tbl.insert()
191 claimed_at = _reactor.seconds()
192 conn.execute(q, [ dict(brid=id, objectid=_master_objectid,
193 claimed_at=claimed_at)
194 for id in brids ])
195 except sa.exc.IntegrityError:
196 transaction.rollback()
197 raise AlreadyClaimedError
198
199 transaction.commit()
200
201 return self.db.pool.do(thd)
202
203 @with_master_objectid
206 """
207 Re-claim the given build requests, updating the timestamp, but checking
208 that the requsts are owned by this master. The resulting deferred will
209 fire normally on success, or fail with L{AleadyClaimedError} if I{any}
210 of the build requests are already claimed by another master instance,
211 or don't exist. In this case, none of the reclaims will take effect.
212
213 @param brids: ids of buildrequests to reclaim
214 @type brids: list
215
216 @param _reactor: reactor to use (for testing)
217
218 @returns: Deferred
219 """
220
221 def thd(conn):
222 transaction = conn.begin()
223 tbl = self.db.model.buildrequest_claims
224 claimed_at = _reactor.seconds()
225
226
227
228 iterator = iter(brids)
229
230 while 1:
231 batch = list(itertools.islice(iterator, 100))
232 if not batch:
233 break
234
235 q = tbl.update(tbl.c.brid.in_(batch)
236 & (tbl.c.objectid==_master_objectid))
237 res = conn.execute(q, claimed_at=claimed_at)
238
239
240
241 if res.rowcount != len(batch):
242 transaction.rollback()
243 raise AlreadyClaimedError
244
245 transaction.commit()
246 return self.db.pool.do(thd)
247
248 @with_master_objectid
250 """
251 Release this master's claim on all of the given build requests. This
252 will not unclaim requests that are claimed by another master, but will
253 not fail in this case. The method does not check whether a request is
254 completed.
255
256 @param brids: ids of buildrequests to unclaim
257 @type brids: list
258
259 @returns: Deferred
260 """
261 def thd(conn):
262 transaction = conn.begin()
263 claims_tbl = self.db.model.buildrequest_claims
264
265
266
267 iterator = iter(brids)
268
269 while 1:
270 batch = list(itertools.islice(iterator, 100))
271 if not batch:
272 break
273
274 try:
275 q = claims_tbl.delete(
276 (claims_tbl.c.brid.in_(batch))
277 & (claims_tbl.c.objectid == _master_objectid))
278 conn.execute(q)
279 except:
280 transaction.rollback()
281 raise
282
283 transaction.commit()
284 return self.db.pool.do(thd)
285
286 @with_master_objectid
289 """
290 Complete a set of build requests, all of which are owned by this master
291 instance. This will fail with L{NotClaimedError} if the build request
292 is already completed or does not exist.
293
294 @param brids: build request IDs to complete
295 @type brids: integer
296
297 @param results: integer result code
298 @type results: integer
299
300 @param _reactor: reactor to use (for testing)
301
302 @returns: Deferred
303 """
304 def thd(conn):
305 transaction = conn.begin()
306
307
308
309
310
311
312 reqs_tbl = self.db.model.buildrequests
313 complete_at = _reactor.seconds()
314
315
316
317 iterator = iter(brids)
318
319 while 1:
320 batch = list(itertools.islice(iterator, 100))
321 if not batch:
322 break
323
324 q = reqs_tbl.update()
325 q = q.where(reqs_tbl.c.id.in_(batch))
326 q = q.where(reqs_tbl.c.complete != 1)
327 res = conn.execute(q,
328 complete=1,
329 results=results,
330 complete_at=complete_at)
331
332
333 if res.rowcount != len(batch):
334 log.msg("tried to complete %d buildreqests, "
335 "but only completed %d" % (len(batch), res.rowcount))
336 transaction.rollback()
337 raise NotClaimedError
338 transaction.commit()
339 return self.db.pool.do(thd)
340
342 """
343 Find any incomplete claimed builds which are older than C{old} seconds,
344 and clear their claim information.
345
346 This is intended to catch builds that were claimed by a master which
347 has since disappeared. As a side effect, it will log a message if any
348 requests are unclaimed.
349
350 @param old: number of seconds after which a claim is considered old
351 @type old: int
352
353 @param _reactor: for testing
354
355 @returns: Deferred
356 """
357 def thd(conn):
358 reqs_tbl = self.db.model.buildrequests
359 claims_tbl = self.db.model.buildrequest_claims
360 old_epoch = _reactor.seconds() - old
361
362
363 expired_brids = sa.select([ reqs_tbl.c.id ],
364 whereclause=(reqs_tbl.c.complete != 1))
365 res = conn.execute(claims_tbl.delete(
366 (claims_tbl.c.claimed_at < old_epoch) &
367 claims_tbl.c.brid.in_(expired_brids)))
368 return res.rowcount
369 d = self.db.pool.do(thd)
370 def log_nonzero_count(count):
371 if count != 0:
372 log.msg("unclaimed %d expired buildrequests (over %d seconds "
373 "old)" % (count, old))
374 d.addCallback(log_nonzero_count)
375 return d
376
378 claimed = mine = False
379 claimed_at = None
380 if row.claimed_at is not None:
381 claimed_at = row.claimed_at
382 claimed = True
383 mine = row.objectid == master_objectid
384
385 def mkdt(epoch):
386 if epoch:
387 return epoch2datetime(epoch)
388 submitted_at = mkdt(row.submitted_at)
389 complete_at = mkdt(row.complete_at)
390 claimed_at = mkdt(row.claimed_at)
391
392 return BrDict(brid=row.id, buildsetid=row.buildsetid,
393 buildername=row.buildername, priority=row.priority,
394 claimed=claimed, claimed_at=claimed_at, mine=mine,
395 complete=bool(row.complete), results=row.results,
396 submitted_at=submitted_at, complete_at=complete_at)
397