buildbot.db.schema.v1

Source Code for Module buildbot.db.schema.v1

1 # ***** BEGIN LICENSE BLOCK ***** 2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1 3 # 4 # The contents of this file are subject to the Mozilla Public License Version 5 # 1.1 (the "License"); you may not use this file except in compliance with 6 # the License. You may obtain a copy of the License at 7 # http://www.mozilla.org/MPL/ 8 # 9 # Software distributed under the License is distributed on an "AS IS" basis, 10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 11 # for the specific language governing rights and limitations under the 12 # License. 13 # 14 # The Original Code is Mozilla-specific Buildbot steps. 15 # 16 # The Initial Developer of the Original Code is 17 # Mozilla Foundation. 18 # Portions created by the Initial Developer are Copyright (C) 2009 19 # the Initial Developer. All Rights Reserved. 20 # 21 # Contributor(s): 22 # Brian Warner <warner@lothar.com> 23 # Chris AtLee <catlee@mozilla.com> 24 # 25 # Alternatively, the contents of this file may be used under the terms of 26 # either the GNU General Public License Version 2 or later (the "GPL"), or 27 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), 28 # in which case the provisions of the GPL or the LGPL are applicable instead 29 # of those above. If you wish to allow use of your version of this file only 30 # under the terms of either the GPL or the LGPL, and not to allow others to 31 # use your version of this file under the terms of the MPL, indicate your 32 # decision by deleting the provisions above and replace them with the notice 33 # and other provisions required by the GPL or the LGPL. If you do not delete 34 # the provisions above, a recipient may use your version of this file under 35 # the terms of any one of the MPL, the GPL or the LGPL. 36 # 37 # ***** END LICENSE BLOCK ***** 38 39 import cPickle 40 import textwrap 41 import os 42 import sys 43 44 from twisted.persisted import styles 45 46 from buildbot.db import util 47 from buildbot.db.schema import base 48 from buildbot.util import json 49 50 # This is version 1, so it introduces a lot of new tables over version 0, 51 # which had no database. 52 53 TABLES = [ 54 # the schema here is defined as version 1 55 textwrap.dedent(""" 56 CREATE TABLE version ( 57 version INTEGER NOT NULL -- contains one row, currently set to 1 58 ); 59 """), 60 61 # last_access is used for logging, to record the last time that each 62 # client (or rather class of clients) touched the DB. The idea is that if 63 # something gets weird, you can check this and discover that you have an 64 # older tool (which uses a different schema) mucking things up. 65 textwrap.dedent(""" 66 CREATE TABLE last_access ( 67 `who` VARCHAR(256) NOT NULL, -- like 'buildbot-0.8.0' 68 `writing` INTEGER NOT NULL, -- 1 if you are writing, 0 if you are reading 69 -- PRIMARY KEY (who, writing), 70 `last_access` TIMESTAMP -- seconds since epoch 71 ); 72 """), 73 74 textwrap.dedent(""" 75 CREATE TABLE changes_nextid (next_changeid INTEGER); 76 """), 77 78 textwrap.dedent(""" 79 -- Changes are immutable: once added, never changed 80 CREATE TABLE changes ( 81 `changeid` INTEGER PRIMARY KEY NOT NULL, -- also serves as 'change number' 82 `author` VARCHAR(1024) NOT NULL, 83 `comments` VARCHAR(1024) NOT NULL, -- too short? 84 `is_dir` SMALLINT NOT NULL, -- old, for CVS 85 `branch` VARCHAR(1024) NULL, 86 `revision` VARCHAR(256), -- CVS uses NULL. too short for darcs? 87 `revlink` VARCHAR(256) NULL, 88 `when_timestamp` INTEGER NOT NULL, -- copied from incoming Change 89 `category` VARCHAR(256) NULL 90 ); 91 """), 92 93 textwrap.dedent(""" 94 CREATE TABLE change_links ( 95 `changeid` INTEGER NOT NULL, 96 `link` VARCHAR(1024) NOT NULL 97 ); 98 """), 99 100 textwrap.dedent(""" 101 CREATE TABLE change_files ( 102 `changeid` INTEGER NOT NULL, 103 `filename` VARCHAR(1024) NOT NULL 104 ); 105 """), 106 107 textwrap.dedent(""" 108 CREATE TABLE change_properties ( 109 `changeid` INTEGER NOT NULL, 110 `property_name` VARCHAR(256) NOT NULL, 111 `property_value` VARCHAR(1024) NOT NULL -- too short? 112 ); 113 """), 114 115 # Scheduler tables 116 textwrap.dedent(""" 117 CREATE TABLE schedulers ( 118 `schedulerid` INTEGER PRIMARY KEY, -- joins to other tables 119 `name` VARCHAR(127) UNIQUE NOT NULL, 120 `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary 121 ); 122 """), 123 124 textwrap.dedent(""" 125 CREATE TABLE scheduler_changes ( 126 `schedulerid` INTEGER, 127 `changeid` INTEGER, 128 `important` SMALLINT 129 ); 130 """), 131 132 textwrap.dedent(""" 133 CREATE TABLE scheduler_upstream_buildsets ( 134 `buildsetid` INTEGER, 135 `schedulerid` INTEGER, 136 `active` SMALLINT 137 ); 138 """), 139 140 # SourceStamps 141 textwrap.dedent(""" 142 -- SourceStamps are immutable: once added, never changed 143 CREATE TABLE sourcestamps ( 144 `id` INTEGER PRIMARY KEY, 145 `branch` VARCHAR(256) default NULL, 146 `revision` VARCHAR(256) default NULL, 147 `patchid` INTEGER default NULL 148 ); 149 """), 150 textwrap.dedent(""" 151 CREATE TABLE patches ( 152 `id` INTEGER PRIMARY KEY, 153 `patchlevel` INTEGER NOT NULL, 154 `patch_base64` TEXT NOT NULL, -- encoded bytestring 155 `subdir` TEXT -- usually NULL 156 ); 157 """), 158 textwrap.dedent(""" 159 CREATE TABLE sourcestamp_changes ( 160 `sourcestampid` INTEGER NOT NULL, 161 `changeid` INTEGER NOT NULL 162 ); 163 """), 164 165 # BuildRequests 166 textwrap.dedent(""" 167 -- BuildSets are mutable. Python code may not cache them. Every 168 -- BuildRequest must have exactly one associated BuildSet. 169 CREATE TABLE buildsets ( 170 `id` INTEGER PRIMARY KEY NOT NULL, 171 `external_idstring` VARCHAR(256), 172 `reason` VARCHAR(256), 173 `sourcestampid` INTEGER NOT NULL, 174 `submitted_at` INTEGER NOT NULL, 175 `complete` SMALLINT NOT NULL default 0, 176 `complete_at` INTEGER, 177 `results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py 178 -- results is NULL until complete==1 179 ); 180 """), 181 textwrap.dedent(""" 182 CREATE TABLE buildset_properties ( 183 `buildsetid` INTEGER NOT NULL, 184 `property_name` VARCHAR(256) NOT NULL, 185 `property_value` VARCHAR(1024) NOT NULL -- too short? 186 ); 187 """), 188 189 textwrap.dedent(""" 190 -- the buildrequests table represents the queue of builds that need to be 191 -- done. In an idle buildbot, all requests will have complete=1. 192 -- BuildRequests are mutable. Python code may not cache them. 193 CREATE TABLE buildrequests ( 194 `id` INTEGER PRIMARY KEY NOT NULL, 195 196 -- every BuildRequest has a BuildSet 197 -- the sourcestampid and reason live in the BuildSet 198 `buildsetid` INTEGER NOT NULL, 199 200 `buildername` VARCHAR(256) NOT NULL, 201 202 `priority` INTEGER NOT NULL default 0, 203 204 -- claimed_at is the time at which a master most recently asserted that 205 -- it is responsible for running the build: this will be updated 206 -- periodically to maintain the claim 207 `claimed_at` INTEGER default 0, 208 209 -- claimed_by indicates which buildmaster has claimed this request. The 210 -- 'name' contains hostname/basedir, and will be the same for subsequent 211 -- runs of any given buildmaster. The 'incarnation' contains bootime/pid, 212 -- and will be different for subsequent runs. This allows each buildmaster 213 -- to distinguish their current claims, their old claims, and the claims 214 -- of other buildmasters, to treat them each appropriately. 215 `claimed_by_name` VARCHAR(256) default NULL, 216 `claimed_by_incarnation` VARCHAR(256) default NULL, 217 218 `complete` INTEGER default 0, -- complete=0 means 'pending' 219 220 -- results is only valid when complete==1 221 `results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py 222 223 `submitted_at` INTEGER NOT NULL, 224 225 `complete_at` INTEGER 226 ); 227 """), 228 229 textwrap.dedent(""" 230 -- this records which builds have been started for each request 231 CREATE TABLE builds ( 232 `id` INTEGER PRIMARY KEY NOT NULL, 233 `number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number) 234 -- 'number' is scoped to both the local buildmaster and the buildername 235 `brid` INTEGER NOT NULL, -- matches buildrequests.id 236 `start_time` INTEGER NOT NULL, 237 `finish_time` INTEGER 238 ); 239 """), 240 ] 241

242 -class Upgrader(base.Upgrader):

243 - def upgrade(self):

244 self.test_unicode() 245 self.add_tables() 246 self.migrate_changes() 247 self.set_version()

248

249 - def test_unicode(self):

250 # first, create a test table 251 c = self.conn.cursor() 252 c.execute("CREATE TABLE test_unicode (`name` VARCHAR(100))") 253 q = util.sql_insert(self.dbapi, 'test_unicode', ["name"]) 254 try: 255 val = u"Frosty the \N{SNOWMAN}" 256 c.execute(q, [val]) 257 c.execute("SELECT * FROM test_unicode") 258 row = c.fetchall()[0] 259 if row[0] != val: 260 raise UnicodeError("Your database doesn't support unicode data; for MySQL, set the default collation to utf8_general_ci.") 261 finally: 262 pass 263 c.execute("DROP TABLE test_unicode")

264

265 - def add_tables(self):

266 # first, add all of the tables 267 c = self.conn.cursor() 268 for t in TABLES: 269 try: 270 c.execute(t) 271 except: 272 print >>sys.stderr, "error executing SQL query: %s" % t 273 raise

274

275 - def _addChangeToDatabase(self, change, cursor):

276 # strip None from any of these values, just in case 277 def remove_none(x): 278 if x is None: return u"" 279 elif isinstance(x, str): 280 return x.decode("utf8") 281 else: 282 return x

283 try: 284 values = tuple(remove_none(x) for x in 285 (change.number, change.who, 286 change.comments, change.isdir, 287 change.branch, change.revision, change.revlink, 288 change.when, change.category)) 289 except UnicodeDecodeError, e: 290 raise UnicodeError("Trying to import change data as UTF-8 failed. Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e)) 291 292 q = util.sql_insert(self.dbapi, 'changes', 293 """changeid author comments is_dir branch revision 294 revlink when_timestamp category""".split()) 295 cursor.execute(q, values) 296 297 for link in change.links: 298 cursor.execute(util.sql_insert(self.dbapi, 'change_links', ('changeid', 'link')), 299 (change.number, link)) 300 301 # sometimes change.files contains nested lists -- why, I do not know! But we deal with 302 # it all the same - see bug #915. We'll assume for now that change.files contains *either* 303 # lists of filenames or plain filenames, not both. 304 def flatten(l): 305 if l and type(l[0]) == list: 306 rv = [] 307 for e in l: 308 if type(e) == list: 309 rv.extend(e) 310 else: 311 rv.append(e) 312 return rv 313 else: 314 return l

315 for filename in flatten(change.files): 316 cursor.execute(util.sql_insert(self.dbapi, 'change_files', ('changeid', 'filename')), 317 (change.number, filename)) 318 for propname,propvalue in change.properties.properties.items(): 319 encoded_value = json.dumps(propvalue) 320 cursor.execute(util.sql_insert(self.dbapi, 'change_properties', 321 ('changeid', 'property_name', 'property_value')), 322 (change.number, propname, encoded_value)) 323

324 - def migrate_changes(self):

325 # if we still have a changes.pck, then we need to migrate it 326 changes_pickle = os.path.join(self.basedir, "changes.pck") 327 if os.path.exists(changes_pickle): 328 if not self.quiet: print "migrating changes.pck to database" 329 330 # 'source' will be an old b.c.changes.ChangeMaster instance, with a 331 # .changes attribute 332 source = cPickle.load(open(changes_pickle,"rb")) 333 styles.doUpgrade() 334 335 if not self.quiet: print " (%d Change objects)" % len(source.changes) 336 337 # first, scan for changes without a number. If we find any, then we'll 338 # renumber the changes sequentially 339 have_unnumbered = False 340 for c in source.changes: 341 if c.revision and c.number is None: 342 have_unnumbered = True 343 break 344 if have_unnumbered: 345 n = 1 346 for c in source.changes: 347 if c.revision: 348 c.number = n 349 n = n + 1 350 351 # insert the changes 352 cursor = self.conn.cursor() 353 for c in source.changes: 354 if not c.revision: 355 continue 356 self._addChangeToDatabase(c, cursor) 357 358 # update next_changeid 359 max_changeid = max([ c.number for c in source.changes if c.revision ] + [ 0 ]) 360 cursor.execute("""INSERT into changes_nextid VALUES (%d)""" % (max_changeid+1)) 361 362 if not self.quiet: 363 print "moving changes.pck to changes.pck.old; delete it or keep it as a backup" 364 os.rename(changes_pickle, changes_pickle+".old") 365 else: 366 c = self.conn.cursor() 367 c.execute("""INSERT into changes_nextid VALUES (1)""")

368

369 - def set_version(self):

370 c = self.conn.cursor() 371 c.execute("""INSERT INTO version VALUES (1)""")

372