Package buildbot :: Package db :: Module model
[frames] | no frames]

Source Code for Module buildbot.db.model

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  """ 
 17  Storage for the database model (schema) 
 18  """ 
 19   
 20  import sqlalchemy as sa 
 21  import migrate 
 22  import migrate.versioning.schema 
 23  import migrate.versioning.repository 
 24  from twisted.python import util, log 
 25  from buildbot.db import base 
 26   
 27  try: 
 28      from migrate.versioning import exceptions 
 29      _hush_pyflakes = exceptions 
 30  except ImportError: 
 31      from migrate import exceptions 
 32   
33 -class Model(base.DBConnectorComponent):
34 """ 35 DBConnector component to handle the database model; an instance is available 36 at C{master.db.model}. 37 38 This class has attributes for each defined table, as well as methods to 39 handle schema migration (using sqlalchemy-migrate). View the source to see 40 the table definitions. 41 42 Note that the Buildbot metadata is never bound to an engine, since that might 43 lead users to execute queries outside of the thread pool. 44 """ 45 46 # 47 # schema 48 # 49 50 metadata = sa.MetaData() 51 52 # NOTES 53 54 # * server_defaults here are included to match those added by the migration 55 # scripts, but they should not be depended on - all code accessing these 56 # tables should supply default values as necessary. The defaults are 57 # required during migration when adding non-nullable columns to existing 58 # tables. 59 # 60 # * dates are stored as unix timestamps (UTC-ish epoch time) 61 62 # build requests 63 64 buildrequests = sa.Table('buildrequests', metadata, 65 sa.Column('id', sa.Integer, primary_key=True), 66 sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), 67 nullable=False), 68 sa.Column('buildername', sa.String(length=256), nullable=False), 69 sa.Column('priority', sa.Integer, nullable=False, 70 server_default=sa.DefaultClause("0")), # TODO: used? 71 72 # if this is zero, then the build is still pending 73 sa.Column('complete', sa.Integer, 74 server_default=sa.DefaultClause("0")), # TODO: boolean 75 76 # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 77 # etc - see master/buildbot/status/builder.py 78 sa.Column('results', sa.SmallInteger), 79 80 # time the buildrequest was created 81 sa.Column('submitted_at', sa.Integer, nullable=False), 82 83 # time the buildrequest was completed, or NULL 84 sa.Column('complete_at', sa.Integer), 85 ) 86 """A BuildRequest is a request for a particular build to be performed. 87 Each BuildRequest is a part of a BuildSet. BuildRequests are claimed by 88 masters, to avoid multiple masters running the same build.""" 89 90 buildrequest_claims = sa.Table('buildrequest_claims', metadata, 91 sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), 92 index=True, unique=True), 93 sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'), 94 index=True, nullable=True), 95 sa.Column('claimed_at', sa.Integer, nullable=False), 96 ) 97 """Each row in this table represents a claimed build request, where the 98 claim is made by the object referenced by objectid.""" 99 100 # builds 101 102 builds = sa.Table('builds', metadata, 103 sa.Column('id', sa.Integer, primary_key=True), 104 105 # XXX 106 # the build number is local to the builder and (maybe?) the buildmaster 107 sa.Column('number', sa.Integer, nullable=False), 108 109 sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False), 110 sa.Column('start_time', sa.Integer, nullable=False), 111 sa.Column('finish_time', sa.Integer), 112 ) 113 """This table contains basic information about each build. Note that most data 114 about a build is still stored in on-disk pickles.""" 115 116 # buildsets 117 118 buildset_properties = sa.Table('buildset_properties', metadata, 119 sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False), 120 sa.Column('property_name', sa.String(256), nullable=False), 121 # JSON-encoded tuple of (value, source) 122 sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 123 ) 124 """This table contains input properties for buildsets""" 125 126 buildsets = sa.Table('buildsets', metadata, 127 sa.Column('id', sa.Integer, primary_key=True), 128 129 # a simple external identifier to track down this buildset later, e.g., 130 # for try requests 131 sa.Column('external_idstring', sa.String(256)), 132 133 # a short string giving the reason the buildset was created 134 sa.Column('reason', sa.String(256)), # TODO: sa.Text 135 sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), 136 sa.Column('submitted_at', sa.Integer, nullable=False), # TODO: redundant 137 138 # if this is zero, then the build set is still pending 139 sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), # TODO: redundant 140 sa.Column('complete_at', sa.Integer), # TODO: redundant 141 142 # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 143 # etc - see master/buildbot/status/builder.py 144 sa.Column('results', sa.SmallInteger), # TODO: synthesize from buildrequests 145 ) 146 """This table represents BuildSets - sets of BuildRequests that share the same 147 original cause and source information.""" 148 149 # changes 150 151 change_files = sa.Table('change_files', metadata, 152 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 153 sa.Column('filename', sa.String(1024), nullable=False), # TODO: sa.Text 154 ) 155 """Files touched in changes""" 156 157 change_links = sa.Table('change_links', metadata, 158 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 159 sa.Column('link', sa.String(1024), nullable=False), # TODO: sa.Text 160 ) 161 """Links (URLs) for changes""" 162 163 change_properties = sa.Table('change_properties', metadata, 164 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 165 sa.Column('property_name', sa.String(256), nullable=False), 166 # JSON-encoded tuple of (value, source) 167 sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 168 ) 169 """Properties for changes""" 170 171 changes = sa.Table('changes', metadata, 172 # changeid also serves as 'change number' 173 sa.Column('changeid', sa.Integer, primary_key=True), # TODO: rename to 'id' 174 175 # author's name (usually an email address) 176 sa.Column('author', sa.String(256), nullable=False), 177 178 # commit comment 179 sa.Column('comments', sa.String(1024), nullable=False), # TODO: too short? 180 181 # old, CVS-related boolean 182 sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS 183 184 # The branch where this change occurred. When branch is NULL, that 185 # means the main branch (trunk, master, etc.) 186 sa.Column('branch', sa.String(256)), 187 188 # revision identifier for this change 189 sa.Column('revision', sa.String(256)), # CVS uses NULL 190 191 # ?? (TODO) 192 sa.Column('revlink', sa.String(256)), 193 194 # this is the timestamp of the change - it is usually copied from the 195 # version-control system, and may be long in the past or even in the 196 # future! 197 sa.Column('when_timestamp', sa.Integer, nullable=False), 198 199 # an arbitrary string used for filtering changes 200 sa.Column('category', sa.String(256)), 201 202 # repository specifies, along with revision and branch, the 203 # source tree in which this change was detected. 204 sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 205 206 # project names the project this source code represents. It is used 207 # later to filter changes 208 sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 209 ) 210 """Changes to the source code, produced by ChangeSources""" 211 212 # sourcestamps 213 214 patches = sa.Table('patches', metadata, 215 sa.Column('id', sa.Integer, primary_key=True), 216 217 # number of directory levels to strip off (patch -pN) 218 sa.Column('patchlevel', sa.Integer, nullable=False), 219 220 # base64-encoded version of the patch file 221 sa.Column('patch_base64', sa.Text, nullable=False), 222 223 # patch author, if known 224 sa.Column('patch_author', sa.Text, nullable=False), 225 226 # patch comment 227 sa.Column('patch_comment', sa.Text, nullable=False), 228 229 # subdirectory in which the patch should be applied; NULL for top-level 230 sa.Column('subdir', sa.Text), 231 ) 232 """Patches for SourceStamps that were generated through the try mechanism""" 233 234 sourcestamp_changes = sa.Table('sourcestamp_changes', metadata, 235 sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), 236 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 237 ) 238 """The changes that led up to a particular source stamp.""" 239 # TODO: changes should be the result of the difference of two sourcestamps! 240 241 sourcestamps = sa.Table('sourcestamps', metadata, 242 sa.Column('id', sa.Integer, primary_key=True), 243 244 # the branch to check out. When branch is NULL, that means 245 # the main branch (trunk, master, etc.) 246 sa.Column('branch', sa.String(256)), 247 248 # the revision to check out, or the latest if NULL 249 sa.Column('revision', sa.String(256)), 250 251 # the patch to apply to generate this source code 252 sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), 253 254 # the repository from which this source should be checked out 255 sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 256 257 # the project this source code represents 258 sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 259 ) 260 """A sourcestamp identifies a particular instance of the source code. 261 Ideally, this would always be absolute, but in practice source stamps can 262 also mean "latest" (when revision is NULL), which is of course a 263 time-dependent definition.""" 264 265 # schedulers 266 267 scheduler_changes = sa.Table('scheduler_changes', metadata, 268 sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')), 269 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), 270 # true if this change is important to this scheduler 271 sa.Column('important', sa.SmallInteger), # TODO: Boolean 272 ) 273 """This table references "classified" changes that have not yet been "processed". 274 That is, the scheduler has looked at these changes and determined that 275 something should be done, but that hasn't happened yet. Rows are deleted 276 from this table as soon as the scheduler is done with the change.""" 277 278 scheduler_upstream_buildsets = sa.Table('scheduler_upstream_buildsets', metadata, 279 sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')), 280 sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')), 281 # true if this buildset is still active 282 sa.Column('active', sa.SmallInteger), # TODO: redundant 283 ) 284 """This table references buildsets in which a particular scheduler is 285 interested. On every run, a scheduler checks its upstream buildsets for 286 completion and reacts accordingly. Records are never deleted from this 287 table, but active is set to 0 when the record is no longer necessary.""" 288 # TODO: delete records eventually 289 290 schedulers = sa.Table("schedulers", metadata, 291 # unique ID for scheduler 292 sa.Column('schedulerid', sa.Integer, primary_key=True), # TODO: rename to id 293 # scheduler's name in master.cfg 294 sa.Column('name', sa.String(128), nullable=False), 295 # JSON-encoded state for this scheduler 296 sa.Column('state', sa.String(1024), nullable=False), 297 # scheduler's class name, basically representing a "type" for the state 298 sa.Column('class_name', sa.String(128), nullable=False), 299 ) 300 """This table records the "state" for each scheduler. This state is, at least, 301 the last change that was analyzed, but is stored in an opaque JSON object. 302 Note that schedulers are never deleted.""" 303 # TODO: delete records eventually 304 305 objects = sa.Table("objects", metadata, 306 # unique ID for this object 307 sa.Column("id", sa.Integer, primary_key=True), 308 # object's user-given name 309 sa.Column('name', sa.String(128), nullable=False), 310 # object's class name, basically representing a "type" for the state 311 sa.Column('class_name', sa.String(128), nullable=False), 312 313 # prohibit multiple id's for the same object 314 sa.UniqueConstraint('name', 'class_name', name='object_identity'), 315 ) 316 """This table uniquely identifies objects that need to maintain state 317 across invocations.""" 318 319 object_state = sa.Table("object_state", metadata, 320 # object for which this value is set 321 sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), 322 nullable=False), 323 # name for this value (local to the object) 324 sa.Column("name", sa.String(length=256), nullable=False), 325 # value, as a JSON string 326 sa.Column("value_json", sa.Text, nullable=False), 327 328 # prohibit multiple values for the same object and name 329 sa.UniqueConstraint('objectid', 'name', name='name_per_object'), 330 ) 331 """This table stores key/value pairs for objects, where the key is a string 332 and the value is a JSON string.""" 333 334 # indexes 335 336 sa.Index('name_and_class', schedulers.c.name, schedulers.c.class_name) 337 sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid) 338 sa.Index('buildrequests_buildername', buildrequests.c.buildername) 339 sa.Index('buildrequests_complete', buildrequests.c.complete) 340 sa.Index('builds_number', builds.c.number) 341 sa.Index('builds_brid', builds.c.brid) 342 sa.Index('buildsets_complete', buildsets.c.complete) 343 sa.Index('buildsets_submitted_at', buildsets.c.submitted_at) 344 sa.Index('buildset_properties_buildsetid', buildset_properties.c.buildsetid) 345 sa.Index('changes_branch', changes.c.branch) 346 sa.Index('changes_revision', changes.c.revision) 347 sa.Index('changes_author', changes.c.author) 348 sa.Index('changes_category', changes.c.category) 349 sa.Index('changes_when_timestamp', changes.c.when_timestamp) 350 sa.Index('change_files_changeid', change_files.c.changeid) 351 sa.Index('change_links_changeid', change_links.c.changeid) 352 sa.Index('change_properties_changeid', change_properties.c.changeid) 353 sa.Index('scheduler_changes_schedulerid', scheduler_changes.c.schedulerid) 354 sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid) 355 sa.Index('scheduler_changes_unique', scheduler_changes.c.schedulerid, 356 scheduler_changes.c.changeid, unique=True) 357 sa.Index('scheduler_upstream_buildsets_buildsetid', scheduler_upstream_buildsets.c.buildsetid) 358 sa.Index('scheduler_upstream_buildsets_schedulerid', scheduler_upstream_buildsets.c.schedulerid) 359 sa.Index('scheduler_upstream_buildsets_active', scheduler_upstream_buildsets.c.active) 360 sa.Index('sourcestamp_changes_sourcestampid', sourcestamp_changes.c.sourcestampid) 361 362 # 363 # migration support 364 # 365 366 # this is a bit more complicated than might be expected because the first 367 # seven database versions were once implemented using a homespun migration 368 # system, and we need to support upgrading masters from that system. The 369 # old system used a 'version' table, where SQLAlchemy-Migrate uses 370 # 'migrate_version' 371 372 repo_path = util.sibpath(__file__, "migrate") 373 "path to the SQLAlchemy-Migrate 'repository'" 374
375 - def is_current(self):
376 """Returns true (via deferred) if the database's version is up to date.""" 377 def thd(engine): 378 # we don't even have to look at the old version table - if there's 379 # no migrate_version, then we're not up to date. 380 repo = migrate.versioning.repository.Repository(self.repo_path) 381 repo_version = repo.latest 382 try: 383 # migrate.api doesn't let us hand in an engine 384 schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 385 db_version = schema.version 386 except exceptions.DatabaseNotControlledError: 387 return False 388 389 return db_version == repo_version
390 return self.db.pool.do_with_engine(thd)
391
392 - def upgrade(self):
393 """Upgrade the database to the most recent schema version, returning a 394 deferred.""" 395 396 # here, things are a little tricky. If we have a 'version' table, then 397 # we need to version_control the database with the proper version 398 # number, drop 'version', and then upgrade. If we have no 'version' 399 # table and no 'migrate_version' table, then we need to version_control 400 # the database. Otherwise, we just need to upgrade it. 401 402 def table_exists(engine, tbl): 403 try: 404 r = engine.execute("select * from %s limit 1" % tbl) 405 r.close() 406 return True 407 except: 408 return False
409 410 # due to http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100, we cannot 411 # use the migrate.versioning.api module. So these methods perform similar wrapping 412 # functions to what is done by the API functions, but without disposing of the engine. 413 def upgrade(engine): 414 schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 415 changeset = schema.changeset(None) 416 for version, change in changeset: 417 log.msg('migrating schema version %s -> %d' 418 % (version, version + 1)) 419 schema.runchange(version, change, 1) 420 421 def version_control(engine, version=None): 422 migrate.versioning.schema.ControlledSchema.create(engine, self.repo_path, version) 423 424 # the upgrade process must run in a db thread 425 def thd(engine): 426 # if the migrate_version table exists, we can just let migrate 427 # take care of this process. 428 if table_exists(engine, 'migrate_version'): 429 upgrade(engine) 430 431 # if the version table exists, then we can version_control things 432 # at that version, drop the version table, and let migrate take 433 # care of the rest. 434 elif table_exists(engine, 'version'): 435 # get the existing version 436 r = engine.execute("select version from version limit 1") 437 old_version = r.scalar() 438 439 # set up migrate at the same version 440 version_control(engine, old_version) 441 442 # drop the no-longer-required version table, using a dummy 443 # metadata entry 444 table = sa.Table('version', self.metadata, 445 sa.Column('x', sa.Integer)) 446 table.drop(bind=engine) 447 448 # clear the dummy metadata entry 449 self.metadata.remove(table) 450 451 # and, finally, upgrade using migrate 452 upgrade(engine) 453 454 # otherwise, this db is uncontrolled, so we just version control it 455 # and update it. 456 else: 457 version_control(engine) 458 upgrade(engine) 459 return self.db.pool.do_with_engine(thd) 460 461 # migrate has a bug in one of its warnings; this is fixed in version control 462 # (3ba66abc4d), but not yet released. It can't hurt to fix it here, too, so we 463 # get realistic tracebacks 464 try: 465 import migrate.versioning.exceptions as ex1 466 import migrate.changeset.exceptions as ex2 467 ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning 468 except ImportError: 469 pass 470