Package buildbot :: Package db :: Module model
[frames] | no frames]

Source Code for Module buildbot.db.model

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import sqlalchemy as sa 
 17  import migrate 
 18  import migrate.versioning.schema 
 19  import migrate.versioning.repository 
 20  from twisted.python import util, log 
 21  from buildbot.db import base 
 22   
 23  try: 
 24      from migrate.versioning import exceptions 
 25      _hush_pyflakes = exceptions 
 26  except ImportError: 
 27      from migrate import exceptions 
 28   
29 -class Model(base.DBConnectorComponent):
30 # 31 # schema 32 # 33 34 metadata = sa.MetaData() 35 36 # NOTES 37 38 # * server_defaults here are included to match those added by the migration 39 # scripts, but they should not be depended on - all code accessing these 40 # tables should supply default values as necessary. The defaults are 41 # required during migration when adding non-nullable columns to existing 42 # tables. 43 # 44 # * dates are stored as unix timestamps (UTC-ish epoch time) 45 # 46 # * sqlalchemy does not handle sa.Boolean very well on MySQL or Postgres; 47 # use sa.Integer instead 48 49 # build requests 50 51 # A BuildRequest is a request for a particular build to be performed. Each 52 # BuildRequest is a part of a Buildset. BuildRequests are claimed by 53 # masters, to avoid multiple masters running the same build. 54 buildrequests = sa.Table('buildrequests', metadata, 55 sa.Column('id', sa.Integer, primary_key=True), 56 sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), 57 nullable=False), 58 sa.Column('buildername', sa.String(length=256), nullable=False), 59 sa.Column('priority', sa.Integer, nullable=False, 60 server_default=sa.DefaultClause("0")), # TODO: used? 61 62 # if this is zero, then the build is still pending 63 sa.Column('complete', sa.Integer, 64 server_default=sa.DefaultClause("0")), # TODO: boolean 65 66 # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 67 # etc - see master/buildbot/status/builder.py 68 sa.Column('results', sa.SmallInteger), 69 70 # time the buildrequest was created 71 sa.Column('submitted_at', sa.Integer, nullable=False), 72 73 # time the buildrequest was completed, or NULL 74 sa.Column('complete_at', sa.Integer), 75 ) 76 77 # Each row in this table represents a claimed build request, where the 78 # claim is made by the object referenced by objectid. 79 buildrequest_claims = sa.Table('buildrequest_claims', metadata, 80 sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), 81 index=True, unique=True), 82 sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'), 83 index=True, nullable=True), 84 sa.Column('claimed_at', sa.Integer, nullable=False), 85 ) 86 87 # builds 88 89 # This table contains basic information about each build. Note that most 90 # data about a build is still stored in on-disk pickles. 91 builds = sa.Table('builds', metadata, 92 sa.Column('id', sa.Integer, primary_key=True), 93 94 # XXX 95 # the build number is local to the builder and (maybe?) the buildmaster 96 sa.Column('number', sa.Integer, nullable=False), 97 98 sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False), 99 sa.Column('start_time', sa.Integer, nullable=False), 100 sa.Column('finish_time', sa.Integer), 101 ) 102 103 # buildsets 104 105 # This table contains input properties for buildsets 106 buildset_properties = sa.Table('buildset_properties', metadata, 107 sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False), 108 sa.Column('property_name', sa.String(256), nullable=False), 109 # JSON-encoded tuple of (value, source) 110 sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 111 ) 112 113 # This table represents Buildsets - sets of BuildRequests that share the 114 # same original cause and source information. 115 buildsets = sa.Table('buildsets', metadata, 116 sa.Column('id', sa.Integer, primary_key=True), 117 118 # a simple external identifier to track down this buildset later, e.g., 119 # for try requests 120 sa.Column('external_idstring', sa.String(256)), 121 122 # a short string giving the reason the buildset was created 123 sa.Column('reason', sa.String(256)), # TODO: sa.Text 124 sa.Column('submitted_at', sa.Integer, nullable=False), # TODO: redundant 125 126 # if this is zero, then the build set is still pending 127 sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), # TODO: redundant 128 sa.Column('complete_at', sa.Integer), # TODO: redundant 129 130 # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, 131 # etc - see master/buildbot/status/builder.py 132 sa.Column('results', sa.SmallInteger), # TODO: synthesize from buildrequests 133 134 # buildset belongs to all sourcestamps with setid 135 sa.Column('sourcestampsetid', sa.Integer, sa.ForeignKey('sourcestampsets.id')), 136 ) 137 138 # changes 139 140 # Files touched in changes 141 change_files = sa.Table('change_files', metadata, 142 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 143 sa.Column('filename', sa.String(1024), nullable=False), # TODO: sa.Text 144 ) 145 146 # Properties for changes 147 change_properties = sa.Table('change_properties', metadata, 148 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 149 sa.Column('property_name', sa.String(256), nullable=False), 150 # JSON-encoded tuple of (value, source) 151 sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short? 152 ) 153 154 # users associated with this change; this allows multiple users for 155 # situations where a version-control system can represent both an author 156 # and committer, for example. 157 change_users = sa.Table("change_users", metadata, 158 sa.Column("changeid", sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 159 # uid for the author of the change with the given changeid 160 sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), nullable=False) 161 ) 162 163 # Changes to the source code, produced by ChangeSources 164 changes = sa.Table('changes', metadata, 165 # changeid also serves as 'change number' 166 sa.Column('changeid', sa.Integer, primary_key=True), # TODO: rename to 'id' 167 168 # author's name (usually an email address) 169 sa.Column('author', sa.String(256), nullable=False), 170 171 # commit comment 172 sa.Column('comments', sa.String(1024), nullable=False), # TODO: too short? 173 174 # old, CVS-related boolean 175 sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS 176 177 # The branch where this change occurred. When branch is NULL, that 178 # means the main branch (trunk, master, etc.) 179 sa.Column('branch', sa.String(256)), 180 181 # revision identifier for this change 182 sa.Column('revision', sa.String(256)), # CVS uses NULL 183 184 # ?? (TODO) 185 sa.Column('revlink', sa.String(256)), 186 187 # this is the timestamp of the change - it is usually copied from the 188 # version-control system, and may be long in the past or even in the 189 # future! 190 sa.Column('when_timestamp', sa.Integer, nullable=False), 191 192 # an arbitrary string used for filtering changes 193 sa.Column('category', sa.String(256)), 194 195 # repository specifies, along with revision and branch, the 196 # source tree in which this change was detected. 197 sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 198 199 # project names the project this source code represents. It is used 200 # later to filter changes 201 sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 202 ) 203 204 # sourcestamps 205 206 # Patches for SourceStamps that were generated through the try mechanism 207 patches = sa.Table('patches', metadata, 208 sa.Column('id', sa.Integer, primary_key=True), 209 210 # number of directory levels to strip off (patch -pN) 211 sa.Column('patchlevel', sa.Integer, nullable=False), 212 213 # base64-encoded version of the patch file 214 sa.Column('patch_base64', sa.Text, nullable=False), 215 216 # patch author, if known 217 sa.Column('patch_author', sa.Text, nullable=False), 218 219 # patch comment 220 sa.Column('patch_comment', sa.Text, nullable=False), 221 222 # subdirectory in which the patch should be applied; NULL for top-level 223 sa.Column('subdir', sa.Text), 224 ) 225 226 # The changes that led up to a particular source stamp. 227 sourcestamp_changes = sa.Table('sourcestamp_changes', metadata, 228 sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), 229 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), 230 ) 231 232 # A sourcestampset identifies a set of sourcestamps 233 # A sourcestamp belongs to a particular set if the sourcestamp has the same setid 234 sourcestampsets = sa.Table('sourcestampsets', metadata, 235 sa.Column('id', sa.Integer, primary_key=True), 236 ) 237 238 # A sourcestamp identifies a particular instance of the source code. 239 # Ideally, this would always be absolute, but in practice source stamps can 240 # also mean "latest" (when revision is NULL), which is of course a 241 # time-dependent definition. 242 sourcestamps = sa.Table('sourcestamps', metadata, 243 sa.Column('id', sa.Integer, primary_key=True), 244 245 # the branch to check out. When branch is NULL, that means 246 # the main branch (trunk, master, etc.) 247 sa.Column('branch', sa.String(256)), 248 249 # the revision to check out, or the latest if NULL 250 sa.Column('revision', sa.String(256)), 251 252 # the patch to apply to generate this source code 253 sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), 254 255 # the repository from which this source should be checked out 256 sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), 257 258 # the project this source code represents 259 sa.Column('project', sa.String(length=512), nullable=False, server_default=''), 260 261 # each sourcestamp belongs to a set of sourcestamps 262 sa.Column('sourcestampsetid', sa.Integer, sa.ForeignKey('sourcestampsets.id')), 263 ) 264 265 # schedulers 266 267 # This table references "classified" changes that have not yet been 268 # "processed". That is, the scheduler has looked at these changes and 269 # determined that something should be done, but that hasn't happened yet. 270 # Rows are deleted from this table as soon as the scheduler is done with 271 # the change. 272 scheduler_changes = sa.Table('scheduler_changes', metadata, 273 sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id')), 274 sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), 275 # true (nonzero) if this change is important to this scheduler 276 sa.Column('important', sa.Integer), 277 ) 278 279 # objects 280 281 # This table uniquely identifies objects that need to maintain state across 282 # invocations. 283 objects = sa.Table("objects", metadata, 284 # unique ID for this object 285 sa.Column("id", sa.Integer, primary_key=True), 286 # object's user-given name 287 sa.Column('name', sa.String(128), nullable=False), 288 # object's class name, basically representing a "type" for the state 289 sa.Column('class_name', sa.String(128), nullable=False), 290 ) 291 292 # This table stores key/value pairs for objects, where the key is a string 293 # and the value is a JSON string. 294 object_state = sa.Table("object_state", metadata, 295 # object for which this value is set 296 sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), 297 nullable=False), 298 # name for this value (local to the object) 299 sa.Column("name", sa.String(length=256), nullable=False), 300 # value, as a JSON string 301 sa.Column("value_json", sa.Text, nullable=False), 302 ) 303 304 #users 305 306 # This table identifies individual users, and contains buildbot-specific 307 # information about those users. 308 users = sa.Table("users", metadata, 309 # unique user id number 310 sa.Column("uid", sa.Integer, primary_key=True), 311 312 # identifier (nickname) for this user; used for display 313 sa.Column("identifier", sa.String(256), nullable=False), 314 315 # username portion of user credentials for authentication 316 sa.Column("bb_username", sa.String(128)), 317 318 # password portion of user credentials for authentication 319 sa.Column("bb_password", sa.String(128)), 320 ) 321 322 # This table stores information identifying a user that's related to a 323 # particular interface - a version-control system, status plugin, etc. 324 users_info = sa.Table("users_info", metadata, 325 # unique user id number 326 sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), 327 nullable=False), 328 329 # type of user attribute, such as 'git' 330 sa.Column("attr_type", sa.String(128), nullable=False), 331 332 # data for given user attribute, such as a commit string or password 333 sa.Column("attr_data", sa.String(128), nullable=False), 334 ) 335 336 337 # indexes 338 339 sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid) 340 sa.Index('buildrequests_buildername', buildrequests.c.buildername) 341 sa.Index('buildrequests_complete', buildrequests.c.complete) 342 sa.Index('builds_number', builds.c.number) 343 sa.Index('builds_brid', builds.c.brid) 344 sa.Index('buildsets_complete', buildsets.c.complete) 345 sa.Index('buildsets_submitted_at', buildsets.c.submitted_at) 346 sa.Index('buildset_properties_buildsetid', buildset_properties.c.buildsetid) 347 sa.Index('changes_branch', changes.c.branch) 348 sa.Index('changes_revision', changes.c.revision) 349 sa.Index('changes_author', changes.c.author) 350 sa.Index('changes_category', changes.c.category) 351 sa.Index('changes_when_timestamp', changes.c.when_timestamp) 352 sa.Index('change_files_changeid', change_files.c.changeid) 353 sa.Index('change_properties_changeid', change_properties.c.changeid) 354 sa.Index('scheduler_changes_objectid', scheduler_changes.c.objectid) 355 sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid) 356 sa.Index('scheduler_changes_unique', scheduler_changes.c.objectid, 357 scheduler_changes.c.changeid, unique=True) 358 sa.Index('sourcestamp_changes_sourcestampid', sourcestamp_changes.c.sourcestampid) 359 sa.Index('sourcestamps_sourcestampsetid', sourcestamps.c.sourcestampsetid, unique=False) 360 sa.Index('users_identifier', users.c.identifier, unique=True) 361 sa.Index('users_info_uid', users_info.c.uid) 362 sa.Index('users_info_uid_attr_type', users_info.c.uid, 363 users_info.c.attr_type, unique=True) 364 sa.Index('users_info_attrs', users_info.c.attr_type, 365 users_info.c.attr_data, unique=True) 366 sa.Index('change_users_changeid', change_users.c.changeid) 367 sa.Index('users_bb_user', users.c.bb_username, unique=True) 368 sa.Index('object_identity', objects.c.name, objects.c.class_name, 369 unique=True) 370 sa.Index('name_per_object', object_state.c.objectid, object_state.c.name, 371 unique=True) 372 373 # MySQl creates indexes for foreign keys, and these appear in the 374 # reflection. This is a list of (table, index) names that should be 375 # expected on this platform 376 377 implied_indexes = [ 378 ('change_users', 379 dict(unique=False, column_names=['uid'], name='uid')), 380 ('sourcestamps', 381 dict(unique=False, column_names=['patchid'], name='patchid')), 382 ('sourcestamp_changes', 383 dict(unique=False, column_names=['changeid'], name='changeid')), 384 ('buildsets', 385 dict(unique=False, column_names=['sourcestampsetid'], 386 name='buildsets_sourcestampsetid_fkey')), 387 ] 388 389 # 390 # migration support 391 # 392 393 # this is a bit more complicated than might be expected because the first 394 # seven database versions were once implemented using a homespun migration 395 # system, and we need to support upgrading masters from that system. The 396 # old system used a 'version' table, where SQLAlchemy-Migrate uses 397 # 'migrate_version' 398 399 repo_path = util.sibpath(__file__, "migrate") 400
401 - def is_current(self):
402 def thd(engine): 403 # we don't even have to look at the old version table - if there's 404 # no migrate_version, then we're not up to date. 405 repo = migrate.versioning.repository.Repository(self.repo_path) 406 repo_version = repo.latest 407 try: 408 # migrate.api doesn't let us hand in an engine 409 schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 410 db_version = schema.version 411 except exceptions.DatabaseNotControlledError: 412 return False 413 414 return db_version == repo_version
415 return self.db.pool.do_with_engine(thd)
416
417 - def upgrade(self):
418 419 # here, things are a little tricky. If we have a 'version' table, then 420 # we need to version_control the database with the proper version 421 # number, drop 'version', and then upgrade. If we have no 'version' 422 # table and no 'migrate_version' table, then we need to version_control 423 # the database. Otherwise, we just need to upgrade it. 424 425 def table_exists(engine, tbl): 426 try: 427 r = engine.execute("select * from %s limit 1" % tbl) 428 r.close() 429 return True 430 except: 431 return False
432 433 # due to http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100, we cannot 434 # use the migrate.versioning.api module. So these methods perform similar wrapping 435 # functions to what is done by the API functions, but without disposing of the engine. 436 def upgrade(engine): 437 schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path) 438 changeset = schema.changeset(None) 439 for version, change in changeset: 440 log.msg('migrating schema version %s -> %d' 441 % (version, version + 1)) 442 schema.runchange(version, change, 1) 443 444 def check_sqlalchemy_migrate_version(): 445 # sqlalchemy-migrate started including a version number in 0.7; we 446 # support back to 0.6.1, but not 0.6. We'll use some discovered 447 # differences between 0.6.1 and 0.6 to get that resolution. 448 version = getattr(migrate, '__version__', 'old') 449 if version == 'old': 450 try: 451 from migrate.versioning import schemadiff 452 if hasattr(schemadiff, 'ColDiff'): 453 version = "0.6.1" 454 else: 455 version = "0.6" 456 except: 457 version = "0.0" 458 version_tup = tuple(map(int, version.split('.'))) 459 log.msg("using SQLAlchemy-Migrate version %s" % (version,)) 460 if version_tup < (0,6,1): 461 raise RuntimeError("You are using SQLAlchemy-Migrate %s. " 462 "The minimum version is 0.6.1." % (version,)) 463 464 def version_control(engine, version=None): 465 migrate.versioning.schema.ControlledSchema.create(engine, self.repo_path, version) 466 467 # the upgrade process must run in a db thread 468 def thd(engine): 469 # if the migrate_version table exists, we can just let migrate 470 # take care of this process. 471 if table_exists(engine, 'migrate_version'): 472 upgrade(engine) 473 474 # if the version table exists, then we can version_control things 475 # at that version, drop the version table, and let migrate take 476 # care of the rest. 477 elif table_exists(engine, 'version'): 478 # get the existing version 479 r = engine.execute("select version from version limit 1") 480 old_version = r.scalar() 481 482 # set up migrate at the same version 483 version_control(engine, old_version) 484 485 # drop the no-longer-required version table, using a dummy 486 # metadata entry 487 table = sa.Table('version', self.metadata, 488 sa.Column('x', sa.Integer)) 489 table.drop(bind=engine) 490 491 # clear the dummy metadata entry 492 self.metadata.remove(table) 493 494 # and, finally, upgrade using migrate 495 upgrade(engine) 496 497 # otherwise, this db is uncontrolled, so we just version control it 498 # and update it. 499 else: 500 version_control(engine) 501 upgrade(engine) 502 503 check_sqlalchemy_migrate_version() 504 return self.db.pool.do_with_engine(thd) 505 506 # migrate has a bug in one of its warnings; this is fixed in version control 507 # (3ba66abc4d), but not yet released. It can't hurt to fix it here, too, so we 508 # get realistic tracebacks 509 try: 510 import migrate.versioning.exceptions as ex1 511 import migrate.changeset.exceptions as ex2 512 ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning 513 except (ImportError,AttributeError): 514 pass 515