Source Code for Module buildbot.db.model

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import sqlalchemy as sa 
 17  import migrate 
 18  import migrate.versioning.schema 
 19  import migrate.versioning.repository 
 20  from twisted.python import util, log 
 21  from buildbot.db import base 
 22   
 23  try: 
 24      from migrate.versioning import exceptions 
 25      _hush_pyflakes = exceptions 
 26  except ImportError: 
 27      from migrate import exceptions 
 28   
 29 -class Model(base.DBConnectorComponent): 
 30      # 
 31      # schema 
 32      # 
 33   
 34      metadata = sa.MetaData() 
 35   
 36      # NOTES 
 37   
 38      # * server_defaults here are included to match those added by the migration 
 39      #   scripts, but they should not be depended on - all code accessing these 
 40      #   tables should supply default values as necessary.  The defaults are 
 41      #   required during migration when adding non-nullable columns to existing 
 42      #   tables. 
 43      # 
 44      # * dates are stored as unix timestamps (UTC-ish epoch time) 
 45      # 
 46      # * sqlalchemy does not handle sa.Boolean very well on MySQL or Postgres; 
 47      #   use sa.Integer instead 
 48   
 49      # build requests 
 50   
 51      # A BuildRequest is a request for a particular build to be performed.  Each 
 52      # BuildRequest is a part of a Buildset.  BuildRequests are claimed by 
 53      # masters, to avoid multiple masters running the same build. 
 54      buildrequests = sa.Table('buildrequests', metadata, 
 55          sa.Column('id', sa.Integer,  primary_key=True), 
 56          sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), 
 57              nullable=False), 
 58          sa.Column('buildername', sa.String(length=256), nullable=False), 
 59          sa.Column('priority', sa.Integer, nullable=False, 
 60              server_default=sa.DefaultClause("0")), # TODO: used? 
 61   
 62          # if this is zero, then the build is still pending 
 63          sa.Column('complete', sa.Integer, 
 64              server_default=sa.DefaultClause("0")), # TODO: boolean 
 65   
 66          # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 
 67          # etc - see master/buildbot/status/builder.py 
 68          sa.Column('results', sa.SmallInteger), 
 69   
 70          # time the buildrequest was created 
 71          sa.Column('submitted_at', sa.Integer, nullable=False), 
 72   
 73          # time the buildrequest was completed, or NULL 
 74          sa.Column('complete_at', sa.Integer), 
 75      ) 
 76   
 77      # Each row in this table represents a claimed build request, where the 
 78      # claim is made by the object referenced by objectid. 
 79      buildrequest_claims = sa.Table('buildrequest_claims', metadata, 
 80          sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), 
 81              index=True, unique=True), 
 82          sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'), 
 83              index=True, nullable=True), 
 84          sa.Column('claimed_at', sa.Integer, nullable=False), 
 85      ) 
 86   
 87      # builds 
 88   
 89      # This table contains basic information about each build.  Note that most 
 90      # data about a build is still stored in on-disk pickles. 
 91      builds = sa.Table('builds', metadata, 
 92          sa.Column('id', sa.Integer,  primary_key=True), 
 93   
 94          # XXX 
 95          # the build number is local to the builder and (maybe?) the buildmaster 
 96          sa.Column('number', sa.Integer, nullable=False), 
 97   
 98          sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False), 
 99          sa.Column('start_time', sa.Integer, nullable=False), 
100          sa.Column('finish_time', sa.Integer), 
101      ) 
102   
103      # buildsets 
104   
105      # This table contains input properties for buildsets 
106      buildset_properties = sa.Table('buildset_properties', metadata, 
107          sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False), 
108          sa.Column('property_name', sa.String(256), nullable=False), 
109          # JSON-encoded tuple of (value, source) 
110          sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 
111      ) 
112   
113      # This table represents Buildsets - sets of BuildRequests that share the 
114      # same original cause and source information. 
115      buildsets = sa.Table('buildsets', metadata, 
116          sa.Column('id', sa.Integer,  primary_key=True), 
117   
118          # a simple external identifier to track down this buildset later, e.g., 
119          # for try requests 
120          sa.Column('external_idstring', sa.String(256)), 
121   
122          # a short string giving the reason the buildset was created 
123          sa.Column('reason', sa.String(256)), # TODO: sa.Text 
124          sa.Column('submitted_at', sa.Integer, nullable=False), # TODO: redundant 
125   
126          # if this is zero, then the build set is still pending 
127          sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), # TODO: redundant 
128          sa.Column('complete_at', sa.Integer), # TODO: redundant 
129   
130          # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 
131          # etc - see master/buildbot/status/builder.py 
132          sa.Column('results', sa.SmallInteger), # TODO: synthesize from buildrequests 
133   
134          # buildset belongs to all sourcestamps with setid 
135          sa.Column('sourcestampsetid', sa.Integer, sa.ForeignKey('sourcestampsets.id')), 
136      ) 
137   
138      # changes 
139   
140      # Files touched in changes 
141      change_files = sa.Table('change_files', metadata, 
142          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
143          sa.Column('filename', sa.String(1024), nullable=False), # TODO: sa.Text 
144      ) 
145   
146      # Properties for changes 
147      change_properties = sa.Table('change_properties', metadata, 
148          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
149          sa.Column('property_name', sa.String(256), nullable=False), 
150          # JSON-encoded tuple of (value, source) 
151          sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 
152      ) 
153   
154      # users associated with this change; this allows multiple users for 
155      # situations where a version-control system can represent both an author 
156      # and committer, for example. 
157      change_users = sa.Table("change_users", metadata, 
158          sa.Column("changeid", sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
159          # uid for the author of the change with the given changeid 
160          sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), nullable=False) 
161      ) 
162   
163      # Changes to the source code, produced by ChangeSources 
164      changes = sa.Table('changes', metadata, 
165          # changeid also serves as 'change number' 
166          sa.Column('changeid', sa.Integer,  primary_key=True), # TODO: rename to 'id' 
167   
168          # author's name (usually an email address) 
169          sa.Column('author', sa.String(256), nullable=False), 
170   
171          # commit comment 
172          sa.Column('comments', sa.String(1024), nullable=False), # TODO: too short? 
173   
174          # old, CVS-related boolean 
175          sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS 
176   
177          # The branch where this change occurred.  When branch is NULL, that 
178          # means the main branch (trunk, master, etc.) 
179          sa.Column('branch', sa.String(256)), 
180   
181          # revision identifier for this change 
182          sa.Column('revision', sa.String(256)), # CVS uses NULL 
183   
184          # ?? (TODO) 
185          sa.Column('revlink', sa.String(256)), 
186   
187          # this is the timestamp of the change - it is usually copied from the 
188          # version-control system, and may be long in the past or even in the 
189          # future! 
190          sa.Column('when_timestamp', sa.Integer, nullable=False), 
191   
192          # an arbitrary string used for filtering changes 
193          sa.Column('category', sa.String(256)), 
194   
195          # repository specifies, along with revision and branch, the 
196          # source tree in which this change was detected. 
197          sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 
198   
199          # project names the project this source code represents.  It is used 
200          # later to filter changes 
201          sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 
202      ) 
203   
204      # sourcestamps 
205   
206      # Patches for SourceStamps that were generated through the try mechanism 
207      patches = sa.Table('patches', metadata, 
208          sa.Column('id', sa.Integer,  primary_key=True), 
209   
210          # number of directory levels to strip off (patch -pN) 
211          sa.Column('patchlevel', sa.Integer, nullable=False), 
212   
213          # base64-encoded version of the patch file 
214          sa.Column('patch_base64', sa.Text, nullable=False), 
215           
216          # patch author, if known 
217          sa.Column('patch_author', sa.Text, nullable=False), 
218           
219          # patch comment 
220          sa.Column('patch_comment', sa.Text, nullable=False), 
221   
222          # subdirectory in which the patch should be applied; NULL for top-level 
223          sa.Column('subdir', sa.Text), 
224      ) 
225   
226      # The changes that led up to a particular source stamp. 
227      sourcestamp_changes = sa.Table('sourcestamp_changes', metadata, 
228          sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), 
229          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 
230      ) 
231   
232      # A sourcestampset identifies a set of sourcestamps 
233      # A sourcestamp belongs to a particular set if the sourcestamp has the same setid 
234      sourcestampsets = sa.Table('sourcestampsets', metadata, 
235          sa.Column('id', sa.Integer,  primary_key=True), 
236      ) 
237   
238      # A sourcestamp identifies a particular instance of the source code. 
239      # Ideally, this would always be absolute, but in practice source stamps can 
240      # also mean "latest" (when revision is NULL), which is of course a 
241      # time-dependent definition. 
242      sourcestamps = sa.Table('sourcestamps', metadata, 
243          sa.Column('id', sa.Integer,  primary_key=True), 
244   
245          # the branch to check out.  When branch is NULL, that means 
246          # the main branch (trunk, master, etc.) 
247          sa.Column('branch', sa.String(256)), 
248   
249          # the revision to check out, or the latest if NULL 
250          sa.Column('revision', sa.String(256)), 
251   
252          # the patch to apply to generate this source code 
253          sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), 
254   
255          # the repository from which this source should be checked out 
256          sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 
257   
258          # the project this source code represents 
259          sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 
260   
261          # each sourcestamp belongs to a set of sourcestamps 
262          sa.Column('sourcestampsetid', sa.Integer, sa.ForeignKey('sourcestampsets.id')), 
263      ) 
264   
265      # schedulers 
266   
267      # This table references "classified" changes that have not yet been 
268      # "processed".  That is, the scheduler has looked at these changes and 
269      # determined that something should be done, but that hasn't happened yet. 
270      # Rows are deleted from this table as soon as the scheduler is done with 
271      # the change. 
272      scheduler_changes = sa.Table('scheduler_changes', metadata, 
273          sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id')), 
274          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), 
275          # true (nonzero) if this change is important to this scheduler 
276          sa.Column('important', sa.Integer), 
277      ) 
278   
279      # objects 
280   
281      # This table uniquely identifies objects that need to maintain state across 
282      # invocations. 
283      objects = sa.Table("objects", metadata, 
284          # unique ID for this object 
285          sa.Column("id", sa.Integer, primary_key=True), 
286          # object's user-given name 
287          sa.Column('name', sa.String(128), nullable=False), 
288          # object's class name, basically representing a "type" for the state 
289          sa.Column('class_name', sa.String(128), nullable=False), 
290      ) 
291   
292      # This table stores key/value pairs for objects, where the key is a string 
293      # and the value is a JSON string. 
294      object_state = sa.Table("object_state", metadata, 
295          # object for which this value is set 
296          sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), 
297              nullable=False), 
298          # name for this value (local to the object) 
299          sa.Column("name", sa.String(length=256), nullable=False), 
300          # value, as a JSON string 
301          sa.Column("value_json", sa.Text, nullable=False), 
302      ) 
303   
304      #users 
305   
306      # This table identifies individual users, and contains buildbot-specific 
307      # information about those users. 
308      users = sa.Table("users", metadata, 
309          # unique user id number 
310          sa.Column("uid", sa.Integer, primary_key=True), 
311   
312          # identifier (nickname) for this user; used for display 
313          sa.Column("identifier", sa.String(256), nullable=False), 
314   
315          # username portion of user credentials for authentication 
316          sa.Column("bb_username", sa.String(128)), 
317   
318          # password portion of user credentials for authentication 
319          sa.Column("bb_password", sa.String(128)), 
320      ) 
321   
322      # This table stores information identifying a user that's related to a 
323      # particular interface - a version-control system, status plugin, etc. 
324      users_info = sa.Table("users_info", metadata, 
325          # unique user id number 
326          sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), 
327                    nullable=False), 
328   
329          # type of user attribute, such as 'git' 
330          sa.Column("attr_type", sa.String(128), nullable=False), 
331   
332          # data for given user attribute, such as a commit string or password 
333          sa.Column("attr_data", sa.String(128), nullable=False), 
334      ) 
335   
336   
337      # indexes 
338   
339      sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid) 
340      sa.Index('buildrequests_buildername', buildrequests.c.buildername) 
341      sa.Index('buildrequests_complete', buildrequests.c.complete) 
342      sa.Index('builds_number', builds.c.number) 
343      sa.Index('builds_brid', builds.c.brid) 
344      sa.Index('buildsets_complete', buildsets.c.complete) 
345      sa.Index('buildsets_submitted_at', buildsets.c.submitted_at) 
346      sa.Index('buildset_properties_buildsetid', buildset_properties.c.buildsetid) 
347      sa.Index('changes_branch', changes.c.branch) 
348      sa.Index('changes_revision', changes.c.revision) 
349      sa.Index('changes_author', changes.c.author) 
350      sa.Index('changes_category', changes.c.category) 
351      sa.Index('changes_when_timestamp', changes.c.when_timestamp) 
352      sa.Index('change_files_changeid', change_files.c.changeid) 
353      sa.Index('change_properties_changeid', change_properties.c.changeid) 
354      sa.Index('scheduler_changes_objectid', scheduler_changes.c.objectid) 
355      sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid) 
356      sa.Index('scheduler_changes_unique', scheduler_changes.c.objectid, 
357                      scheduler_changes.c.changeid, unique=True) 
358      sa.Index('sourcestamp_changes_sourcestampid', sourcestamp_changes.c.sourcestampid) 
359      sa.Index('sourcestamps_sourcestampsetid', sourcestamps.c.sourcestampsetid, unique=False) 
360      sa.Index('users_identifier', users.c.identifier, unique=True) 
361      sa.Index('users_info_uid', users_info.c.uid) 
362      sa.Index('users_info_uid_attr_type', users_info.c.uid, 
363              users_info.c.attr_type, unique=True) 
364      sa.Index('users_info_attrs', users_info.c.attr_type, 
365              users_info.c.attr_data, unique=True) 
366      sa.Index('change_users_changeid', change_users.c.changeid) 
367      sa.Index('users_bb_user', users.c.bb_username, unique=True) 
368      sa.Index('object_identity', objects.c.name, objects.c.class_name, 
369              unique=True) 
370      sa.Index('name_per_object', object_state.c.objectid, object_state.c.name, 
371              unique=True) 
372   
373      # MySQl creates indexes for foreign keys, and these appear in the 
374      # reflection.  This is a list of (table, index) names that should be 
375      # expected on this platform 
376   
377      implied_indexes = [ 
378          ('change_users',  
379              dict(unique=False, column_names=['uid'], name='uid')), 
380          ('sourcestamps', 
381              dict(unique=False, column_names=['patchid'], name='patchid')), 
382          ('sourcestamp_changes', 
383              dict(unique=False, column_names=['changeid'], name='changeid')), 
384          ('buildsets', 
385              dict(unique=False, column_names=['sourcestampsetid'], 
386                                 name='buildsets_sourcestampsetid_fkey')), 
387      ] 
388   
389      # 
390      # migration support 
391      # 
392   
393      # this is a bit more complicated than might be expected because the first 
394      # seven database versions were once implemented using a homespun migration 
395      # system, and we need to support upgrading masters from that system.  The 
396      # old system used a 'version' table, where SQLAlchemy-Migrate uses 
397      # 'migrate_version' 
398   
399      repo_path = util.sibpath(__file__, "migrate") 
400   
401 -    def is_current(self): 
402          def thd(engine): 
403              # we don't even have to look at the old version table - if there's 
404              # no migrate_version, then we're not up to date. 
405              repo = migrate.versioning.repository.Repository(self.repo_path) 
406              repo_version = repo.latest 
407              try: 
408                  # migrate.api doesn't let us hand in an engine 
409                  schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 
410                  db_version = schema.version 
411              except exceptions.DatabaseNotControlledError: 
412                  return False 
413   
414              return db_version == repo_version 
415          return self.db.pool.do_with_engine(thd) 
416   
417 -    def upgrade(self): 
418   
419          # here, things are a little tricky.  If we have a 'version' table, then 
420          # we need to version_control the database with the proper version 
421          # number, drop 'version', and then upgrade.  If we have no 'version' 
422          # table and no 'migrate_version' table, then we need to version_control 
423          # the database.  Otherwise, we just need to upgrade it. 
424   
425          def table_exists(engine, tbl): 
426              try: 
427                  r = engine.execute("select * from %s limit 1" % tbl) 
428                  r.close() 
429                  return True 
430              except: 
431                  return False 
432   
433          # due to http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100, we cannot 
434          # use the migrate.versioning.api module.  So these methods perform similar wrapping 
435          # functions to what is done by the API functions, but without disposing of the engine. 
436          def upgrade(engine): 
437              schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 
438              changeset = schema.changeset(None) 
439              for version, change in changeset: 
440                  log.msg('migrating schema version %s -> %d' 
441                          % (version, version + 1)) 
442                  schema.runchange(version, change, 1) 
443   
444          def check_sqlalchemy_migrate_version(): 
445              # sqlalchemy-migrate started including a version number in 0.7; we 
446              # support back to 0.6.1, but not 0.6.  We'll use some discovered 
447              # differences between 0.6.1 and 0.6 to get that resolution. 
448              version = getattr(migrate, '__version__', 'old') 
449              if version == 'old': 
450                  try: 
451                      from migrate.versioning import schemadiff 
452                      if hasattr(schemadiff, 'ColDiff'): 
453                          version = "0.6.1" 
454                      else: 
455                          version = "0.6" 
456                  except: 
457                      version = "0.0" 
458              version_tup = tuple(map(int, version.split('.'))) 
459              log.msg("using SQLAlchemy-Migrate version %s" % (version,)) 
460              if version_tup < (0,6,1): 
461                  raise RuntimeError("You are using SQLAlchemy-Migrate %s. " 
462                      "The minimum version is 0.6.1." % (version,)) 
463   
464          def version_control(engine, version=None): 
465              migrate.versioning.schema.ControlledSchema.create(engine, self.repo_path, version) 
466   
467          # the upgrade process must run in a db thread 
468          def thd(engine): 
469              # if the migrate_version table exists, we can just let migrate 
470              # take care of this process. 
471              if table_exists(engine, 'migrate_version'): 
472                  upgrade(engine) 
473   
474              # if the version table exists, then we can version_control things 
475              # at that version, drop the version table, and let migrate take 
476              # care of the rest. 
477              elif table_exists(engine, 'version'): 
478                  # get the existing version 
479                  r = engine.execute("select version from version limit 1") 
480                  old_version = r.scalar() 
481   
482                  # set up migrate at the same version 
483                  version_control(engine, old_version) 
484   
485                  # drop the no-longer-required version table, using a dummy 
486                  # metadata entry 
487                  table = sa.Table('version', self.metadata, 
488                                   sa.Column('x', sa.Integer)) 
489                  table.drop(bind=engine) 
490   
491                  # clear the dummy metadata entry 
492                  self.metadata.remove(table) 
493   
494                  # and, finally, upgrade using migrate 
495                  upgrade(engine) 
496   
497              # otherwise, this db is uncontrolled, so we just version control it 
498              # and update it. 
499              else: 
500                  version_control(engine) 
501                  upgrade(engine) 
502   
503          check_sqlalchemy_migrate_version() 
504          return self.db.pool.do_with_engine(thd) 
505   
506  # migrate has a bug in one of its warnings; this is fixed in version control 
507  # (3ba66abc4d), but not yet released. It can't hurt to fix it here, too, so we 
508  # get realistic tracebacks 
509  try: 
510      import migrate.versioning.exceptions as ex1 
511      import migrate.changeset.exceptions as ex2 
512      ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning 
513  except (ImportError,AttributeError): 
514      pass 
515