Source Code for Module buildbot.db.model

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import sqlalchemy as sa 
 17  import migrate 
 18  import migrate.versioning.schema 
 19  import migrate.versioning.repository 
 20  from twisted.python import util, log 
 21  from buildbot.db import base 
 22   
 23  try: 
 24      from migrate.versioning import exceptions 
 25      _hush_pyflakes = exceptions 
 26  except ImportError: 
 27      from migrate import exceptions 
 28   
 29 -class Model(base.DBConnectorComponent): 
 30      # 
 31      # schema 
 32      # 
 33   
 34      metadata = sa.MetaData() 
 35   
 36      # NOTES 
 37   
 38      # * server_defaults here are included to match those added by the migration 
 39      #   scripts, but they should not be depended on - all code accessing these 
 40      #   tables should supply default values as necessary.  The defaults are 
 41      #   required during migration when adding non-nullable columns to existing 
 42      #   tables. 
 43      # 
 44      # * dates are stored as unix timestamps (UTC-ish epoch time) 
 45      # 
 46      # * sqlalchemy does not handle sa.Boolean very well on MySQL or Postgres; 
 47      #   use sa.Integer instead 
 48   
 49      # build requests 
 50   
 51      # A BuildRequest is a request for a particular build to be performed.  Each 
 52      # BuildRequest is a part of a Buildset.  BuildRequests are claimed by 
 53      # masters, to avoid multiple masters running the same build. 
 54      buildrequests = sa.Table('buildrequests', metadata, 
 55          sa.Column('id', sa.Integer,  primary_key=True), 
 56          sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), 
 57              nullable=False), 
 58          sa.Column('buildername', sa.String(length=256), nullable=False), 
 59          sa.Column('priority', sa.Integer, nullable=False, 
 60              server_default=sa.DefaultClause("0")), 
 61   
 62          # if this is zero, then the build is still pending 
 63          sa.Column('complete', sa.Integer, 
 64              server_default=sa.DefaultClause("0")), 
 65   
 66          # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 
 67          # etc - see master/buildbot/status/builder.py 
 68          sa.Column('results', sa.SmallInteger), 
 69   
 70          # time the buildrequest was created 
 71          sa.Column('submitted_at', sa.Integer, nullable=False), 
 72   
 73          # time the buildrequest was completed, or NULL 
 74          sa.Column('complete_at', sa.Integer), 
 75      ) 
 76   
 77      # Each row in this table represents a claimed build request, where the 
 78      # claim is made by the object referenced by objectid. 
 79      buildrequest_claims = sa.Table('buildrequest_claims', metadata, 
 80          sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), 
 81              index=True, unique=True), 
 82          sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'), 
 83              index=True, nullable=True), 
 84          sa.Column('claimed_at', sa.Integer, nullable=False), 
 85      ) 
 86   
 87      # builds 
 88   
 89      # This table contains basic information about each build.  Note that most 
 90      # data about a build is still stored in on-disk pickles. 
 91      builds = sa.Table('builds', metadata, 
 92          sa.Column('id', sa.Integer,  primary_key=True), 
 93          sa.Column('number', sa.Integer, nullable=False), 
 94          sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), 
 95              nullable=False), 
 96          sa.Column('start_time', sa.Integer, nullable=False), 
 97          sa.Column('finish_time', sa.Integer), 
 98      ) 
 99   
100      # buildsets 
101   
102      # This table contains input properties for buildsets 
103      buildset_properties = sa.Table('buildset_properties', metadata, 
104          sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), 
105              nullable=False), 
106          sa.Column('property_name', sa.String(256), nullable=False), 
107          # JSON-encoded tuple of (value, source) 
108          sa.Column('property_value', sa.String(1024), nullable=False), 
109      ) 
110   
111      # This table represents Buildsets - sets of BuildRequests that share the 
112      # same original cause and source information. 
113      buildsets = sa.Table('buildsets', metadata, 
114          sa.Column('id', sa.Integer,  primary_key=True), 
115   
116          # a simple external identifier to track down this buildset later, e.g., 
117          # for try requests 
118          sa.Column('external_idstring', sa.String(256)), 
119   
120          # a short string giving the reason the buildset was created 
121          sa.Column('reason', sa.String(256)), 
122          sa.Column('submitted_at', sa.Integer, nullable=False), 
123   
124          # if this is zero, then the build set is still pending 
125          sa.Column('complete', sa.SmallInteger, nullable=False, 
126              server_default=sa.DefaultClause("0")), 
127          sa.Column('complete_at', sa.Integer), 
128   
129          # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 
130          # etc - see master/buildbot/status/builder.py 
131          sa.Column('results', sa.SmallInteger), 
132   
133          # buildset belongs to all sourcestamps with setid 
134          sa.Column('sourcestampsetid', sa.Integer, 
135              sa.ForeignKey('sourcestampsets.id')), 
136      ) 
137   
138      # changes 
139   
140      # Files touched in changes 
141      change_files = sa.Table('change_files', metadata, 
142          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), 
143              nullable=False), 
144          sa.Column('filename', sa.String(1024), nullable=False), 
145      ) 
146   
147      # Properties for changes 
148      change_properties = sa.Table('change_properties', metadata, 
149          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), 
150              nullable=False), 
151          sa.Column('property_name', sa.String(256), nullable=False), 
152          # JSON-encoded tuple of (value, source) 
153          sa.Column('property_value', sa.String(1024), nullable=False), 
154      ) 
155   
156      # users associated with this change; this allows multiple users for 
157      # situations where a version-control system can represent both an author 
158      # and committer, for example. 
159      change_users = sa.Table("change_users", metadata, 
160          sa.Column("changeid", sa.Integer, sa.ForeignKey('changes.changeid'), 
161              nullable=False), 
162          # uid for the author of the change with the given changeid 
163          sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), 
164              nullable=False) 
165      ) 
166   
167      # Changes to the source code, produced by ChangeSources 
168      changes = sa.Table('changes', metadata, 
169          # changeid also serves as 'change number' 
170          sa.Column('changeid', sa.Integer,  primary_key=True), 
171   
172          # author's name (usually an email address) 
173          sa.Column('author', sa.String(256), nullable=False), 
174   
175          # commit comment 
176          sa.Column('comments', sa.String(1024), nullable=False), 
177   
178          # old, CVS-related boolean 
179          sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS 
180   
181          # The branch where this change occurred.  When branch is NULL, that 
182          # means the main branch (trunk, master, etc.) 
183          sa.Column('branch', sa.String(256)), 
184   
185          # revision identifier for this change 
186          sa.Column('revision', sa.String(256)), # CVS uses NULL 
187   
188          sa.Column('revlink', sa.String(256)), 
189   
190          # this is the timestamp of the change - it is usually copied from the 
191          # version-control system, and may be long in the past or even in the 
192          # future! 
193          sa.Column('when_timestamp', sa.Integer, nullable=False), 
194   
195          # an arbitrary string used for filtering changes 
196          sa.Column('category', sa.String(256)), 
197   
198          # repository specifies, along with revision and branch, the 
199          # source tree in which this change was detected. 
200          sa.Column('repository', sa.String(length=512), nullable=False, 
201              server_default=''), 
202   
203          # codebase is a logical name to specify what is in the repository 
204          sa.Column('codebase', sa.String(256), nullable=False, 
205              server_default=sa.DefaultClause("")), 
206   
207          # project names the project this source code represents.  It is used 
208          # later to filter changes 
209          sa.Column('project', sa.String(length=512), nullable=False, 
210              server_default=''), 
211      ) 
212   
213      # sourcestamps 
214   
215      # Patches for SourceStamps that were generated through the try mechanism 
216      patches = sa.Table('patches', metadata, 
217          sa.Column('id', sa.Integer,  primary_key=True), 
218   
219          # number of directory levels to strip off (patch -pN) 
220          sa.Column('patchlevel', sa.Integer, nullable=False), 
221   
222          # base64-encoded version of the patch file 
223          sa.Column('patch_base64', sa.Text, nullable=False), 
224   
225          # patch author, if known 
226          sa.Column('patch_author', sa.Text, nullable=False), 
227   
228          # patch comment 
229          sa.Column('patch_comment', sa.Text, nullable=False), 
230   
231          # subdirectory in which the patch should be applied; NULL for top-level 
232          sa.Column('subdir', sa.Text), 
233      ) 
234   
235      # The changes that led up to a particular source stamp. 
236      sourcestamp_changes = sa.Table('sourcestamp_changes', metadata, 
237          sa.Column('sourcestampid', sa.Integer, 
238              sa.ForeignKey('sourcestamps.id'), nullable=False), 
239          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), 
240              nullable=False), 
241      ) 
242   
243      # A sourcestampset identifies a set of sourcestamps. A sourcestamp belongs 
244      # to a particular set if the sourcestamp has the same setid 
245      sourcestampsets = sa.Table('sourcestampsets', metadata, 
246          sa.Column('id', sa.Integer,  primary_key=True), 
247      ) 
248   
249      # A sourcestamp identifies a particular instance of the source code. 
250      # Ideally, this would always be absolute, but in practice source stamps can 
251      # also mean "latest" (when revision is NULL), which is of course a 
252      # time-dependent definition. 
253      sourcestamps = sa.Table('sourcestamps', metadata, 
254          sa.Column('id', sa.Integer,  primary_key=True), 
255   
256          # the branch to check out.  When branch is NULL, that means 
257          # the main branch (trunk, master, etc.) 
258          sa.Column('branch', sa.String(256)), 
259   
260          # the revision to check out, or the latest if NULL 
261          sa.Column('revision', sa.String(256)), 
262   
263          # the patch to apply to generate this source code 
264          sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), 
265   
266          # the repository from which this source should be checked out 
267          sa.Column('repository', sa.String(length=512), nullable=False, 
268              server_default=''), 
269   
270          # codebase is a logical name to specify what is in the repository 
271          sa.Column('codebase', sa.String(256), nullable=False, 
272              server_default=sa.DefaultClause("")), 
273   
274          # the project this source code represents 
275          sa.Column('project', sa.String(length=512), nullable=False, 
276              server_default=''), 
277   
278          # each sourcestamp belongs to a set of sourcestamps 
279          sa.Column('sourcestampsetid', sa.Integer, 
280              sa.ForeignKey('sourcestampsets.id')), 
281      ) 
282   
283      # schedulers 
284   
285      # This table references "classified" changes that have not yet been 
286      # "processed".  That is, the scheduler has looked at these changes and 
287      # determined that something should be done, but that hasn't happened yet. 
288      # Rows are deleted from this table as soon as the scheduler is done with 
289      # the change. 
290      scheduler_changes = sa.Table('scheduler_changes', metadata, 
291          sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id')), 
292          sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), 
293          # true (nonzero) if this change is important to this scheduler 
294          sa.Column('important', sa.Integer), 
295      ) 
296   
297      # objects 
298   
299      # This table uniquely identifies objects that need to maintain state across 
300      # invocations. 
301      objects = sa.Table("objects", metadata, 
302          # unique ID for this object 
303          sa.Column("id", sa.Integer, primary_key=True), 
304          # object's user-given name 
305          sa.Column('name', sa.String(128), nullable=False), 
306          # object's class name, basically representing a "type" for the state 
307          sa.Column('class_name', sa.String(128), nullable=False), 
308      ) 
309   
310      # This table stores key/value pairs for objects, where the key is a string 
311      # and the value is a JSON string. 
312      object_state = sa.Table("object_state", metadata, 
313          # object for which this value is set 
314          sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), 
315              nullable=False), 
316          # name for this value (local to the object) 
317          sa.Column("name", sa.String(length=256), nullable=False), 
318          # value, as a JSON string 
319          sa.Column("value_json", sa.Text, nullable=False), 
320      ) 
321   
322      #users 
323   
324      # This table identifies individual users, and contains buildbot-specific 
325      # information about those users. 
326      users = sa.Table("users", metadata, 
327          # unique user id number 
328          sa.Column("uid", sa.Integer, primary_key=True), 
329   
330          # identifier (nickname) for this user; used for display 
331          sa.Column("identifier", sa.String(256), nullable=False), 
332   
333          # username portion of user credentials for authentication 
334          sa.Column("bb_username", sa.String(128)), 
335   
336          # password portion of user credentials for authentication 
337          sa.Column("bb_password", sa.String(128)), 
338      ) 
339   
340      # This table stores information identifying a user that's related to a 
341      # particular interface - a version-control system, status plugin, etc. 
342      users_info = sa.Table("users_info", metadata, 
343          # unique user id number 
344          sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), 
345                    nullable=False), 
346   
347          # type of user attribute, such as 'git' 
348          sa.Column("attr_type", sa.String(128), nullable=False), 
349   
350          # data for given user attribute, such as a commit string or password 
351          sa.Column("attr_data", sa.String(128), nullable=False), 
352      ) 
353   
354   
355      # indexes 
356   
357      sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid) 
358      sa.Index('buildrequests_buildername', buildrequests.c.buildername) 
359      sa.Index('buildrequests_complete', buildrequests.c.complete) 
360      sa.Index('builds_number', builds.c.number) 
361      sa.Index('builds_brid', builds.c.brid) 
362      sa.Index('buildsets_complete', buildsets.c.complete) 
363      sa.Index('buildsets_submitted_at', buildsets.c.submitted_at) 
364      sa.Index('buildset_properties_buildsetid', 
365              buildset_properties.c.buildsetid) 
366      sa.Index('changes_branch', changes.c.branch) 
367      sa.Index('changes_revision', changes.c.revision) 
368      sa.Index('changes_author', changes.c.author) 
369      sa.Index('changes_category', changes.c.category) 
370      sa.Index('changes_when_timestamp', changes.c.when_timestamp) 
371      sa.Index('change_files_changeid', change_files.c.changeid) 
372      sa.Index('change_properties_changeid', change_properties.c.changeid) 
373      sa.Index('scheduler_changes_objectid', scheduler_changes.c.objectid) 
374      sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid) 
375      sa.Index('scheduler_changes_unique', scheduler_changes.c.objectid, 
376              scheduler_changes.c.changeid, unique=True) 
377      sa.Index('sourcestamp_changes_sourcestampid', 
378              sourcestamp_changes.c.sourcestampid) 
379      sa.Index('sourcestamps_sourcestampsetid', sourcestamps.c.sourcestampsetid, 
380              unique=False) 
381      sa.Index('users_identifier', users.c.identifier, unique=True) 
382      sa.Index('users_info_uid', users_info.c.uid) 
383      sa.Index('users_info_uid_attr_type', users_info.c.uid, 
384              users_info.c.attr_type, unique=True) 
385      sa.Index('users_info_attrs', users_info.c.attr_type, 
386              users_info.c.attr_data, unique=True) 
387      sa.Index('change_users_changeid', change_users.c.changeid) 
388      sa.Index('users_bb_user', users.c.bb_username, unique=True) 
389      sa.Index('object_identity', objects.c.name, objects.c.class_name, 
390              unique=True) 
391      sa.Index('name_per_object', object_state.c.objectid, object_state.c.name, 
392              unique=True) 
393   
394      # MySQl creates indexes for foreign keys, and these appear in the 
395      # reflection.  This is a list of (table, index) names that should be 
396      # expected on this platform 
397   
398      implied_indexes = [ 
399          ('change_users',  
400              dict(unique=False, column_names=['uid'], name='uid')), 
401          ('sourcestamps', 
402              dict(unique=False, column_names=['patchid'], name='patchid')), 
403          ('sourcestamp_changes', 
404              dict(unique=False, column_names=['changeid'], name='changeid')), 
405          ('buildsets', 
406              dict(unique=False, column_names=['sourcestampsetid'], 
407                                 name='buildsets_sourcestampsetid_fkey')), 
408      ] 
409   
410      # 
411      # migration support 
412      # 
413   
414      # this is a bit more complicated than might be expected because the first 
415      # seven database versions were once implemented using a homespun migration 
416      # system, and we need to support upgrading masters from that system.  The 
417      # old system used a 'version' table, where SQLAlchemy-Migrate uses 
418      # 'migrate_version' 
419   
420      repo_path = util.sibpath(__file__, "migrate") 
421   
422 -    def is_current(self): 
423          def thd(engine): 
424              # we don't even have to look at the old version table - if there's 
425              # no migrate_version, then we're not up to date. 
426              repo = migrate.versioning.repository.Repository(self.repo_path) 
427              repo_version = repo.latest 
428              try: 
429                  # migrate.api doesn't let us hand in an engine 
430                  schema = migrate.versioning.schema.ControlledSchema(engine, 
431                                                                  self.repo_path) 
432                  db_version = schema.version 
433              except exceptions.DatabaseNotControlledError: 
434                  return False 
435   
436              return db_version == repo_version 
437          return self.db.pool.do_with_engine(thd) 
438   
439 -    def upgrade(self): 
440   
441          # here, things are a little tricky.  If we have a 'version' table, then 
442          # we need to version_control the database with the proper version 
443          # number, drop 'version', and then upgrade.  If we have no 'version' 
444          # table and no 'migrate_version' table, then we need to version_control 
445          # the database.  Otherwise, we just need to upgrade it. 
446   
447          def table_exists(engine, tbl): 
448              try: 
449                  r = engine.execute("select * from %s limit 1" % tbl) 
450                  r.close() 
451                  return True 
452              except: 
453                  return False 
454   
455          # http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100 
456          # means  we cannot use the migrate.versioning.api module.  So these 
457          # methods perform similar wrapping functions to what is done by the API 
458          # functions, but without disposing of the engine. 
459          def upgrade(engine): 
460              schema = migrate.versioning.schema.ControlledSchema(engine, 
461                      self.repo_path) 
462              changeset = schema.changeset(None) 
463              for version, change in changeset: 
464                  log.msg('migrating schema version %s -> %d' 
465                          % (version, version + 1)) 
466                  schema.runchange(version, change, 1) 
467   
468          def check_sqlalchemy_migrate_version(): 
469              # sqlalchemy-migrate started including a version number in 0.7; we 
470              # support back to 0.6.1, but not 0.6.  We'll use some discovered 
471              # differences between 0.6.1 and 0.6 to get that resolution. 
472              version = getattr(migrate, '__version__', 'old') 
473              if version == 'old': 
474                  try: 
475                      from migrate.versioning import schemadiff 
476                      if hasattr(schemadiff, 'ColDiff'): 
477                          version = "0.6.1" 
478                      else: 
479                          version = "0.6" 
480                  except: 
481                      version = "0.0" 
482              version_tup = tuple(map(int, version.split('.'))) 
483              log.msg("using SQLAlchemy-Migrate version %s" % (version,)) 
484              if version_tup < (0,6,1): 
485                  raise RuntimeError("You are using SQLAlchemy-Migrate %s. " 
486                      "The minimum version is 0.6.1." % (version,)) 
487   
488          def version_control(engine, version=None): 
489              migrate.versioning.schema.ControlledSchema.create(engine, 
490                      self.repo_path, version) 
491   
492          # the upgrade process must run in a db thread 
493          def thd(engine): 
494              # if the migrate_version table exists, we can just let migrate 
495              # take care of this process. 
496              if table_exists(engine, 'migrate_version'): 
497                  upgrade(engine) 
498   
499              # if the version table exists, then we can version_control things 
500              # at that version, drop the version table, and let migrate take 
501              # care of the rest. 
502              elif table_exists(engine, 'version'): 
503                  # get the existing version 
504                  r = engine.execute("select version from version limit 1") 
505                  old_version = r.scalar() 
506   
507                  # set up migrate at the same version 
508                  version_control(engine, old_version) 
509   
510                  # drop the no-longer-required version table, using a dummy 
511                  # metadata entry 
512                  table = sa.Table('version', self.metadata, 
513                                   sa.Column('x', sa.Integer)) 
514                  table.drop(bind=engine) 
515   
516                  # clear the dummy metadata entry 
517                  self.metadata.remove(table) 
518   
519                  # and, finally, upgrade using migrate 
520                  upgrade(engine) 
521   
522              # otherwise, this db is uncontrolled, so we just version control it 
523              # and update it. 
524              else: 
525                  version_control(engine) 
526                  upgrade(engine) 
527   
528          check_sqlalchemy_migrate_version() 
529          return self.db.pool.do_with_engine(thd) 
530   
531  # migrate has a bug in one of its warnings; this is fixed in version control 
532  # (3ba66abc4d), but not yet released. It can't hurt to fix it here, too, so we 
533  # get realistic tracebacks 
534  try: 
535      import migrate.versioning.exceptions as ex1 
536      import migrate.changeset.exceptions as ex2 
537      ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning 
538  except (ImportError,AttributeError): 
539      pass 
540