1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 """
17 Storage for the database model (schema)
18 """
19
20 import sqlalchemy as sa
21 import migrate
22 import migrate.versioning.schema
23 import migrate.versioning.repository
24 from twisted.python import util, log
25 from buildbot.db import base
26
27 try:
28 from migrate.versioning import exceptions
29 _hush_pyflakes = exceptions
30 except ImportError:
31 from migrate import exceptions
32
33 -class Model(base.DBConnectorComponent):
34 """
35 DBConnector component to handle the database model; an instance is available
36 at C{master.db.model}.
37
38 This class has attributes for each defined table, as well as methods to
39 handle schema migration (using sqlalchemy-migrate). View the source to see
40 the table definitions.
41
42 Note that the Buildbot metadata is never bound to an engine, since that might
43 lead users to execute queries outside of the thread pool.
44 """
45
46
47
48
49
50 metadata = sa.MetaData()
51
52
53
54
55
56
57
58
59
60
61
62
63
64 buildrequests = sa.Table('buildrequests', metadata,
65 sa.Column('id', sa.Integer, primary_key=True),
66 sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"),
67 nullable=False),
68 sa.Column('buildername', sa.String(length=256), nullable=False),
69 sa.Column('priority', sa.Integer, nullable=False,
70 server_default=sa.DefaultClause("0")),
71
72
73 sa.Column('complete', sa.Integer,
74 server_default=sa.DefaultClause("0")),
75
76
77
78 sa.Column('results', sa.SmallInteger),
79
80
81 sa.Column('submitted_at', sa.Integer, nullable=False),
82
83
84 sa.Column('complete_at', sa.Integer),
85 )
86 """A BuildRequest is a request for a particular build to be performed.
87 Each BuildRequest is a part of a BuildSet. BuildRequests are claimed by
88 masters, to avoid multiple masters running the same build."""
89
90 buildrequest_claims = sa.Table('buildrequest_claims', metadata,
91 sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'),
92 index=True, unique=True),
93 sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'),
94 index=True, nullable=True),
95 sa.Column('claimed_at', sa.Integer, nullable=False),
96 )
97 """Each row in this table represents a claimed build request, where the
98 claim is made by the object referenced by objectid."""
99
100
101
102 builds = sa.Table('builds', metadata,
103 sa.Column('id', sa.Integer, primary_key=True),
104
105
106
107 sa.Column('number', sa.Integer, nullable=False),
108
109 sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False),
110 sa.Column('start_time', sa.Integer, nullable=False),
111 sa.Column('finish_time', sa.Integer),
112 )
113 """This table contains basic information about each build. Note that most data
114 about a build is still stored in on-disk pickles."""
115
116
117
118 buildset_properties = sa.Table('buildset_properties', metadata,
119 sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False),
120 sa.Column('property_name', sa.String(256), nullable=False),
121
122 sa.Column('property_value', sa.String(1024), nullable=False),
123 )
124 """This table contains input properties for buildsets"""
125
126 buildsets = sa.Table('buildsets', metadata,
127 sa.Column('id', sa.Integer, primary_key=True),
128
129
130
131 sa.Column('external_idstring', sa.String(256)),
132
133
134 sa.Column('reason', sa.String(256)),
135 sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False),
136 sa.Column('submitted_at', sa.Integer, nullable=False),
137
138
139 sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")),
140 sa.Column('complete_at', sa.Integer),
141
142
143
144 sa.Column('results', sa.SmallInteger),
145 )
146 """This table represents BuildSets - sets of BuildRequests that share the same
147 original cause and source information."""
148
149
150
151 change_files = sa.Table('change_files', metadata,
152 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
153 sa.Column('filename', sa.String(1024), nullable=False),
154 )
155 """Files touched in changes"""
156
157 change_links = sa.Table('change_links', metadata,
158 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
159 sa.Column('link', sa.String(1024), nullable=False),
160 )
161 """Links (URLs) for changes"""
162
163 change_properties = sa.Table('change_properties', metadata,
164 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
165 sa.Column('property_name', sa.String(256), nullable=False),
166
167 sa.Column('property_value', sa.String(1024), nullable=False),
168 )
169 """Properties for changes"""
170
171 changes = sa.Table('changes', metadata,
172
173 sa.Column('changeid', sa.Integer, primary_key=True),
174
175
176 sa.Column('author', sa.String(256), nullable=False),
177
178
179 sa.Column('comments', sa.String(1024), nullable=False),
180
181
182 sa.Column('is_dir', sa.SmallInteger, nullable=False),
183
184
185
186 sa.Column('branch', sa.String(256)),
187
188
189 sa.Column('revision', sa.String(256)),
190
191
192 sa.Column('revlink', sa.String(256)),
193
194
195
196
197 sa.Column('when_timestamp', sa.Integer, nullable=False),
198
199
200 sa.Column('category', sa.String(256)),
201
202
203
204 sa.Column('repository', sa.String(length=512), nullable=False, server_default=''),
205
206
207
208 sa.Column('project', sa.String(length=512), nullable=False, server_default=''),
209 )
210 """Changes to the source code, produced by ChangeSources"""
211
212
213
214 patches = sa.Table('patches', metadata,
215 sa.Column('id', sa.Integer, primary_key=True),
216
217
218 sa.Column('patchlevel', sa.Integer, nullable=False),
219
220
221 sa.Column('patch_base64', sa.Text, nullable=False),
222
223
224 sa.Column('patch_author', sa.Text, nullable=False),
225
226
227 sa.Column('patch_comment', sa.Text, nullable=False),
228
229
230 sa.Column('subdir', sa.Text),
231 )
232 """Patches for SourceStamps that were generated through the try mechanism"""
233
234 sourcestamp_changes = sa.Table('sourcestamp_changes', metadata,
235 sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False),
236 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
237 )
238 """The changes that led up to a particular source stamp."""
239
240
241 sourcestamps = sa.Table('sourcestamps', metadata,
242 sa.Column('id', sa.Integer, primary_key=True),
243
244
245
246 sa.Column('branch', sa.String(256)),
247
248
249 sa.Column('revision', sa.String(256)),
250
251
252 sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')),
253
254
255 sa.Column('repository', sa.String(length=512), nullable=False, server_default=''),
256
257
258 sa.Column('project', sa.String(length=512), nullable=False, server_default=''),
259 )
260 """A sourcestamp identifies a particular instance of the source code.
261 Ideally, this would always be absolute, but in practice source stamps can
262 also mean "latest" (when revision is NULL), which is of course a
263 time-dependent definition."""
264
265
266
267 scheduler_changes = sa.Table('scheduler_changes', metadata,
268 sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')),
269 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')),
270
271 sa.Column('important', sa.SmallInteger),
272 )
273 """This table references "classified" changes that have not yet been "processed".
274 That is, the scheduler has looked at these changes and determined that
275 something should be done, but that hasn't happened yet. Rows are deleted
276 from this table as soon as the scheduler is done with the change."""
277
278 scheduler_upstream_buildsets = sa.Table('scheduler_upstream_buildsets', metadata,
279 sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')),
280 sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')),
281
282 sa.Column('active', sa.SmallInteger),
283 )
284 """This table references buildsets in which a particular scheduler is
285 interested. On every run, a scheduler checks its upstream buildsets for
286 completion and reacts accordingly. Records are never deleted from this
287 table, but active is set to 0 when the record is no longer necessary."""
288
289
290 schedulers = sa.Table("schedulers", metadata,
291
292 sa.Column('schedulerid', sa.Integer, primary_key=True),
293
294 sa.Column('name', sa.String(128), nullable=False),
295
296 sa.Column('state', sa.String(1024), nullable=False),
297
298 sa.Column('class_name', sa.String(128), nullable=False),
299 )
300 """This table records the "state" for each scheduler. This state is, at least,
301 the last change that was analyzed, but is stored in an opaque JSON object.
302 Note that schedulers are never deleted."""
303
304
305 objects = sa.Table("objects", metadata,
306
307 sa.Column("id", sa.Integer, primary_key=True),
308
309 sa.Column('name', sa.String(128), nullable=False),
310
311 sa.Column('class_name', sa.String(128), nullable=False),
312
313
314 sa.UniqueConstraint('name', 'class_name', name='object_identity'),
315 )
316 """This table uniquely identifies objects that need to maintain state
317 across invocations."""
318
319 object_state = sa.Table("object_state", metadata,
320
321 sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'),
322 nullable=False),
323
324 sa.Column("name", sa.String(length=256), nullable=False),
325
326 sa.Column("value_json", sa.Text, nullable=False),
327
328
329 sa.UniqueConstraint('objectid', 'name', name='name_per_object'),
330 )
331 """This table stores key/value pairs for objects, where the key is a string
332 and the value is a JSON string."""
333
334
335
336 sa.Index('name_and_class', schedulers.c.name, schedulers.c.class_name)
337 sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid)
338 sa.Index('buildrequests_buildername', buildrequests.c.buildername)
339 sa.Index('buildrequests_complete', buildrequests.c.complete)
340 sa.Index('builds_number', builds.c.number)
341 sa.Index('builds_brid', builds.c.brid)
342 sa.Index('buildsets_complete', buildsets.c.complete)
343 sa.Index('buildsets_submitted_at', buildsets.c.submitted_at)
344 sa.Index('buildset_properties_buildsetid', buildset_properties.c.buildsetid)
345 sa.Index('changes_branch', changes.c.branch)
346 sa.Index('changes_revision', changes.c.revision)
347 sa.Index('changes_author', changes.c.author)
348 sa.Index('changes_category', changes.c.category)
349 sa.Index('changes_when_timestamp', changes.c.when_timestamp)
350 sa.Index('change_files_changeid', change_files.c.changeid)
351 sa.Index('change_links_changeid', change_links.c.changeid)
352 sa.Index('change_properties_changeid', change_properties.c.changeid)
353 sa.Index('scheduler_changes_schedulerid', scheduler_changes.c.schedulerid)
354 sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid)
355 sa.Index('scheduler_changes_unique', scheduler_changes.c.schedulerid,
356 scheduler_changes.c.changeid, unique=True)
357 sa.Index('scheduler_upstream_buildsets_buildsetid', scheduler_upstream_buildsets.c.buildsetid)
358 sa.Index('scheduler_upstream_buildsets_schedulerid', scheduler_upstream_buildsets.c.schedulerid)
359 sa.Index('scheduler_upstream_buildsets_active', scheduler_upstream_buildsets.c.active)
360 sa.Index('sourcestamp_changes_sourcestampid', sourcestamp_changes.c.sourcestampid)
361
362
363
364
365
366
367
368
369
370
371
372 repo_path = util.sibpath(__file__, "migrate")
373 "path to the SQLAlchemy-Migrate 'repository'"
374
376 """Returns true (via deferred) if the database's version is up to date."""
377 def thd(engine):
378
379
380 repo = migrate.versioning.repository.Repository(self.repo_path)
381 repo_version = repo.latest
382 try:
383
384 schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path)
385 db_version = schema.version
386 except exceptions.DatabaseNotControlledError:
387 return False
388
389 return db_version == repo_version
390 return self.db.pool.do_with_engine(thd)
391
393 """Upgrade the database to the most recent schema version, returning a
394 deferred."""
395
396
397
398
399
400
401
402 def table_exists(engine, tbl):
403 try:
404 r = engine.execute("select * from %s limit 1" % tbl)
405 r.close()
406 return True
407 except:
408 return False
409
410
411
412
413 def upgrade(engine):
414 schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path)
415 changeset = schema.changeset(None)
416 for version, change in changeset:
417 log.msg('migrating schema version %s -> %d'
418 % (version, version + 1))
419 schema.runchange(version, change, 1)
420
421 def version_control(engine, version=None):
422 migrate.versioning.schema.ControlledSchema.create(engine, self.repo_path, version)
423
424
425 def thd(engine):
426
427
428 if table_exists(engine, 'migrate_version'):
429 upgrade(engine)
430
431
432
433
434 elif table_exists(engine, 'version'):
435
436 r = engine.execute("select version from version limit 1")
437 old_version = r.scalar()
438
439
440 version_control(engine, old_version)
441
442
443
444 table = sa.Table('version', self.metadata,
445 sa.Column('x', sa.Integer))
446 table.drop(bind=engine)
447
448
449 self.metadata.remove(table)
450
451
452 upgrade(engine)
453
454
455
456 else:
457 version_control(engine)
458 upgrade(engine)
459 return self.db.pool.do_with_engine(thd)
460
461
462
463
464 try:
465 import migrate.versioning.exceptions as ex1
466 import migrate.changeset.exceptions as ex2
467 ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning
468 except ImportError:
469 pass
470