Source Code for Module buildbot.db.model

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  """ 
 17  Storage for the database model (schema) 
 18  """ 
 19   
 20  import sqlalchemy as sa 
 21  import migrate 
 22  import migrate.versioning.schema 
 23  import migrate.versioning.repository 
 24  from twisted.python import util, log 
 25  from buildbot.db import base 
 26   
 27  try: 
 28      from migrate.versioning import exceptions 
 29      _hush_pyflakes = exceptions 
 30  except ImportError: 
 31      from migrate import exceptions 
 32   
 33 -class Model(base.DBConnectorComponent): 
 34      """ 
 35      DBConnector component to handle the database model; an instance is available 
 36      at C{master.db.model}. 
 37   
 38      This class has attributes for each defined table, as well as methods to 
 39      handle schema migration (using sqlalchemy-migrate).  View the source to see 
 40      the table definitions. 
 41   
 42      Note that the Buildbot metadata is never bound to an engine, since that might 
 43      lead users to execute queries outside of the thread pool. 
 44      """ 
 45   
 46      # 
 47      # schema 
 48      # 
 49   
 50      metadata = sa.MetaData() 
 51   
 52      # NOTES 
 53   
 54      # * server_defaults here are included to match those added by the migration 
 55      #   scripts, but they should not be depended on - all code accessing these 
 56      #   tables should supply default values as necessary.  The defaults are 
 57      #   required during migration when adding non-nullable columns to existing 
 58      #   tables. 
 59      # 
 60      # * dates are stored as unix timestamps (UTC-ish epoch time) 
 61   
 62      # build requests 
 63   
 64      buildrequests = sa.Table('buildrequests', metadata, 
 65          sa.Column('id', sa.Integer,  primary_key=True), 
 66          sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), 
 67              nullable=False), 
 68          sa.Column('buildername', sa.String(length=256), nullable=False), 
 69          sa.Column('priority', sa.Integer, nullable=False, 
 70              server_default=sa.DefaultClause("0")), # TODO: used? 
 71   
 72          # if this is zero, then the build is still pending 
 73          sa.Column('complete', sa.Integer, 
 74              server_default=sa.DefaultClause("0")), # TODO: boolean 
 75   
 76          # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 
 77          # etc - see master/buildbot/status/builder.py 
 78          sa.Column('results', sa.SmallInteger), 
 79   
 80          # time the buildrequest was created 
 81          sa.Column('submitted_at', sa.Integer, nullable=False), 
 82   
 83          # time the buildrequest was completed, or NULL 
 84          sa.Column('complete_at', sa.Integer), 
 85      ) 
 86      """A BuildRequest is a request for a particular build to be performed. 
 87      Each BuildRequest is a part of a BuildSet.  BuildRequests are claimed by 
 88      masters, to avoid multiple masters running the same build.""" 
 89   
 90      buildrequest_claims = sa.Table('buildrequest_claims', metadata, 
 91          sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), 
 92              index=True, unique=True), 
 93          sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'), 
 94              index=True, nullable=True), 
 95          sa.Column('claimed_at', sa.Integer, nullable=False), 
 96      ) 
 97      """Each row in this table represents a claimed build request, where the 
 98      claim is made by the object referenced by objectid.""" 
 99   
100      # builds 
101   
102      builds = sa.Table('builds', metadata, 
103          sa.Column('id', sa.Integer,  primary_key=True), 
104   
105          # XXX 
106          # the build number is local to the builder and (maybe?) the buildmaster 
107          sa.Column('number', sa.Integer, nullable=False), 
108   
109          sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False), 
110          sa.Column('start_time', sa.Integer, nullable=False), 
111          sa.Column('finish_time', sa.Integer), 
112      ) 
113      """This table contains basic information about each build.  Note that most data 
114      about a build is still stored in on-disk pickles.""" 
115   
116      # buildsets 
117   
118      buildset_properties = sa.Table('buildset_properties', metadata, 
119          sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False), 
120          sa.Column('property_name', sa.String(256), nullable=False), 
121          # JSON-encoded tuple of (value, source) 
122          sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 
123      ) 
124      """This table contains input properties for buildsets""" 
125   
126      buildsets = sa.Table('buildsets', metadata, 
127          sa.Column('id', sa.Integer,  primary_key=True), 
128   
129          # a simple external identifier to track down this buildset later, e.g., 
130          # for try requests 
131          sa.Column('external_idstring', sa.String(256)), 
132   
133          # a short string giving the reason the buildset was created 
134          sa.Column('reason', sa.String(256)), # TODO: sa.Text 
135          sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), 
136          sa.Column('submitted_at', sa.Integer, nullable=False), # TODO: redundant 
137   
138          # if this is zero, then the build set is still pending 
139          sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), # TODO: redundant 
140          sa.Column('complete_at', sa.Integer), # TODO: redundant 
141   
142          # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 
143          # etc - see master/buildbot/status/builder.py 
144          sa.Column('results', sa.SmallInteger), # TODO: synthesize from buildrequests 
145      ) 
146      """This table represents BuildSets - sets of BuildRequests that share the same 
147      original cause and source information.""" 
148   
149      # changes 
150   
151      change_files = sa.Table('change_files', metadata, 
152          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
153          sa.Column('filename', sa.String(1024), nullable=False), # TODO: sa.Text 
154      ) 
155      """Files touched in changes""" 
156   
157      change_links = sa.Table('change_links', metadata, 
158          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
159          sa.Column('link', sa.String(1024), nullable=False), # TODO: sa.Text 
160      ) 
161      """Links (URLs) for changes""" 
162   
163      change_properties = sa.Table('change_properties', metadata, 
164          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
165          sa.Column('property_name', sa.String(256), nullable=False), 
166          # JSON-encoded tuple of (value, source) 
167          sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 
168      ) 
169      """Properties for changes""" 
170   
171      changes = sa.Table('changes', metadata, 
172          # changeid also serves as 'change number' 
173          sa.Column('changeid', sa.Integer,  primary_key=True), # TODO: rename to 'id' 
174   
175          # author's name (usually an email address) 
176          sa.Column('author', sa.String(256), nullable=False), 
177   
178          # commit comment 
179          sa.Column('comments', sa.String(1024), nullable=False), # TODO: too short? 
180   
181          # old, CVS-related boolean 
182          sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS 
183   
184          # The branch where this change occurred.  When branch is NULL, that 
185          # means the main branch (trunk, master, etc.) 
186          sa.Column('branch', sa.String(256)), 
187   
188          # revision identifier for this change 
189          sa.Column('revision', sa.String(256)), # CVS uses NULL 
190   
191          # ?? (TODO) 
192          sa.Column('revlink', sa.String(256)), 
193   
194          # this is the timestamp of the change - it is usually copied from the 
195          # version-control system, and may be long in the past or even in the 
196          # future! 
197          sa.Column('when_timestamp', sa.Integer, nullable=False), 
198   
199          # an arbitrary string used for filtering changes 
200          sa.Column('category', sa.String(256)), 
201   
202          # repository specifies, along with revision and branch, the 
203          # source tree in which this change was detected. 
204          sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 
205   
206          # project names the project this source code represents.  It is used 
207          # later to filter changes 
208          sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 
209      ) 
210      """Changes to the source code, produced by ChangeSources""" 
211   
212      # sourcestamps 
213   
214      patches = sa.Table('patches', metadata, 
215          sa.Column('id', sa.Integer,  primary_key=True), 
216   
217          # number of directory levels to strip off (patch -pN) 
218          sa.Column('patchlevel', sa.Integer, nullable=False), 
219   
220          # base64-encoded version of the patch file 
221          sa.Column('patch_base64', sa.Text, nullable=False), 
222           
223          # patch author, if known 
224          sa.Column('patch_author', sa.Text, nullable=False), 
225           
226          # patch comment 
227          sa.Column('patch_comment', sa.Text, nullable=False), 
228   
229          # subdirectory in which the patch should be applied; NULL for top-level 
230          sa.Column('subdir', sa.Text), 
231      ) 
232      """Patches for SourceStamps that were generated through the try mechanism""" 
233   
234      sourcestamp_changes = sa.Table('sourcestamp_changes', metadata, 
235          sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), 
236          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
237      ) 
238      """The changes that led up to a particular source stamp.""" 
239      # TODO: changes should be the result of the difference of two sourcestamps! 
240   
241      sourcestamps = sa.Table('sourcestamps', metadata, 
242          sa.Column('id', sa.Integer,  primary_key=True), 
243   
244          # the branch to check out.  When branch is NULL, that means 
245          # the main branch (trunk, master, etc.) 
246          sa.Column('branch', sa.String(256)), 
247   
248          # the revision to check out, or the latest if NULL 
249          sa.Column('revision', sa.String(256)), 
250   
251          # the patch to apply to generate this source code 
252          sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), 
253   
254          # the repository from which this source should be checked out 
255          sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 
256   
257          # the project this source code represents 
258          sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 
259      ) 
260      """A sourcestamp identifies a particular instance of the source code. 
261      Ideally, this would always be absolute, but in practice source stamps can 
262      also mean "latest" (when revision is NULL), which is of course a 
263      time-dependent definition.""" 
264   
265      # schedulers 
266   
267      scheduler_changes = sa.Table('scheduler_changes', metadata, 
268          sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')), 
269          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), 
270          # true if this change is important to this scheduler 
271          sa.Column('important', sa.SmallInteger), # TODO: Boolean 
272      ) 
273      """This table references "classified" changes that have not yet been "processed". 
274      That is, the scheduler has looked at these changes and determined that 
275      something should be done, but that hasn't happened yet.  Rows are deleted 
276      from this table as soon as the scheduler is done with the change.""" 
277   
278      scheduler_upstream_buildsets = sa.Table('scheduler_upstream_buildsets', metadata, 
279          sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')), 
280          sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')), 
281          # true if this buildset is still active 
282          sa.Column('active', sa.SmallInteger), # TODO: redundant 
283      ) 
284      """This table references buildsets in which a particular scheduler is 
285      interested.  On every run, a scheduler checks its upstream buildsets for 
286      completion and reacts accordingly.  Records are never deleted from this 
287      table, but active is set to 0 when the record is no longer necessary.""" 
288      # TODO: delete records eventually 
289   
290      schedulers = sa.Table("schedulers", metadata, 
291          # unique ID for scheduler 
292          sa.Column('schedulerid', sa.Integer, primary_key=True), # TODO: rename to id 
293          # scheduler's name in master.cfg 
294          sa.Column('name', sa.String(128), nullable=False), 
295          # JSON-encoded state for this scheduler 
296          sa.Column('state', sa.String(1024), nullable=False), 
297          # scheduler's class name, basically representing a "type" for the state 
298          sa.Column('class_name', sa.String(128), nullable=False), 
299      ) 
300      """This table records the "state" for each scheduler.  This state is, at least, 
301      the last change that was analyzed, but is stored in an opaque JSON object. 
302      Note that schedulers are never deleted.""" 
303      # TODO: delete records eventually 
304   
305      objects = sa.Table("objects", metadata, 
306          # unique ID for this object 
307          sa.Column("id", sa.Integer, primary_key=True), 
308          # object's user-given name 
309          sa.Column('name', sa.String(128), nullable=False), 
310          # object's class name, basically representing a "type" for the state 
311          sa.Column('class_name', sa.String(128), nullable=False), 
312   
313          # prohibit multiple id's for the same object 
314          sa.UniqueConstraint('name', 'class_name', name='object_identity'), 
315      ) 
316      """This table uniquely identifies objects that need to maintain state 
317      across invocations.""" 
318   
319      object_state = sa.Table("object_state", metadata, 
320          # object for which this value is set 
321          sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), 
322              nullable=False), 
323          # name for this value (local to the object) 
324          sa.Column("name", sa.String(length=256), nullable=False), 
325          # value, as a JSON string 
326          sa.Column("value_json", sa.Text, nullable=False), 
327   
328          # prohibit multiple values for the same object and name 
329          sa.UniqueConstraint('objectid', 'name', name='name_per_object'), 
330      ) 
331      """This table stores key/value pairs for objects, where the key is a string 
332      and the value is a JSON string.""" 
333   
334      # indexes 
335   
336      sa.Index('name_and_class', schedulers.c.name, schedulers.c.class_name) 
337      sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid) 
338      sa.Index('buildrequests_buildername', buildrequests.c.buildername) 
339      sa.Index('buildrequests_complete', buildrequests.c.complete) 
340      sa.Index('builds_number', builds.c.number) 
341      sa.Index('builds_brid', builds.c.brid) 
342      sa.Index('buildsets_complete', buildsets.c.complete) 
343      sa.Index('buildsets_submitted_at', buildsets.c.submitted_at) 
344      sa.Index('buildset_properties_buildsetid', buildset_properties.c.buildsetid) 
345      sa.Index('changes_branch', changes.c.branch) 
346      sa.Index('changes_revision', changes.c.revision) 
347      sa.Index('changes_author', changes.c.author) 
348      sa.Index('changes_category', changes.c.category) 
349      sa.Index('changes_when_timestamp', changes.c.when_timestamp) 
350      sa.Index('change_files_changeid', change_files.c.changeid) 
351      sa.Index('change_links_changeid', change_links.c.changeid) 
352      sa.Index('change_properties_changeid', change_properties.c.changeid) 
353      sa.Index('scheduler_changes_schedulerid', scheduler_changes.c.schedulerid) 
354      sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid) 
355      sa.Index('scheduler_changes_unique', scheduler_changes.c.schedulerid, 
356                      scheduler_changes.c.changeid, unique=True) 
357      sa.Index('scheduler_upstream_buildsets_buildsetid', scheduler_upstream_buildsets.c.buildsetid) 
358      sa.Index('scheduler_upstream_buildsets_schedulerid', scheduler_upstream_buildsets.c.schedulerid) 
359      sa.Index('scheduler_upstream_buildsets_active', scheduler_upstream_buildsets.c.active) 
360      sa.Index('sourcestamp_changes_sourcestampid', sourcestamp_changes.c.sourcestampid) 
361   
362      # 
363      # migration support 
364      # 
365   
366      # this is a bit more complicated than might be expected because the first 
367      # seven database versions were once implemented using a homespun migration 
368      # system, and we need to support upgrading masters from that system.  The 
369      # old system used a 'version' table, where SQLAlchemy-Migrate uses 
370      # 'migrate_version' 
371   
372      repo_path = util.sibpath(__file__, "migrate") 
373      "path to the SQLAlchemy-Migrate 'repository'" 
374   
375 -    def is_current(self): 
376          """Returns true (via deferred) if the database's version is up to date.""" 
377          def thd(engine): 
378              # we don't even have to look at the old version table - if there's 
379              # no migrate_version, then we're not up to date. 
380              repo = migrate.versioning.repository.Repository(self.repo_path) 
381              repo_version = repo.latest 
382              try: 
383                  # migrate.api doesn't let us hand in an engine 
384                  schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 
385                  db_version = schema.version 
386              except exceptions.DatabaseNotControlledError: 
387                  return False 
388   
389              return db_version == repo_version 
390          return self.db.pool.do_with_engine(thd) 
391   
392 -    def upgrade(self): 
393          """Upgrade the database to the most recent schema version, returning a 
394          deferred.""" 
395   
396          # here, things are a little tricky.  If we have a 'version' table, then 
397          # we need to version_control the database with the proper version 
398          # number, drop 'version', and then upgrade.  If we have no 'version' 
399          # table and no 'migrate_version' table, then we need to version_control 
400          # the database.  Otherwise, we just need to upgrade it. 
401   
402          def table_exists(engine, tbl): 
403              try: 
404                  r = engine.execute("select * from %s limit 1" % tbl) 
405                  r.close() 
406                  return True 
407              except: 
408                  return False 
409   
410          # due to http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100, we cannot 
411          # use the migrate.versioning.api module.  So these methods perform similar wrapping 
412          # functions to what is done by the API functions, but without disposing of the engine. 
413          def upgrade(engine): 
414              schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 
415              changeset = schema.changeset(None) 
416              for version, change in changeset: 
417                  log.msg('migrating schema version %s -> %d' 
418                          % (version, version + 1)) 
419                  schema.runchange(version, change, 1) 
420   
421          def version_control(engine, version=None): 
422              migrate.versioning.schema.ControlledSchema.create(engine, self.repo_path, version) 
423   
424          # the upgrade process must run in a db thread 
425          def thd(engine): 
426              # if the migrate_version table exists, we can just let migrate 
427              # take care of this process. 
428              if table_exists(engine, 'migrate_version'): 
429                  upgrade(engine) 
430   
431              # if the version table exists, then we can version_control things 
432              # at that version, drop the version table, and let migrate take 
433              # care of the rest. 
434              elif table_exists(engine, 'version'): 
435                  # get the existing version 
436                  r = engine.execute("select version from version limit 1") 
437                  old_version = r.scalar() 
438   
439                  # set up migrate at the same version 
440                  version_control(engine, old_version) 
441   
442                  # drop the no-longer-required version table, using a dummy 
443                  # metadata entry 
444                  table = sa.Table('version', self.metadata, 
445                                   sa.Column('x', sa.Integer)) 
446                  table.drop(bind=engine) 
447   
448                  # clear the dummy metadata entry 
449                  self.metadata.remove(table) 
450   
451                  # and, finally, upgrade using migrate 
452                  upgrade(engine) 
453   
454              # otherwise, this db is uncontrolled, so we just version control it 
455              # and update it. 
456              else: 
457                  version_control(engine) 
458                  upgrade(engine) 
459          return self.db.pool.do_with_engine(thd) 
460   
461  # migrate has a bug in one of its warnings; this is fixed in version control 
462  # (3ba66abc4d), but not yet released. It can't hurt to fix it here, too, so we 
463  # get realistic tracebacks 
464  try: 
465      import migrate.versioning.exceptions as ex1 
466      import migrate.changeset.exceptions as ex2 
467      ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning 
468  except ImportError: 
469      pass 
470