buildbot.db.schema.v1

Source Code for Module buildbot.db.schema.v1

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import cPickle 17 import textwrap 18 import os 19 import sys 20 21 from twisted.persisted import styles 22 23 from buildbot.db import util 24 from buildbot.db.schema import base 25 from buildbot.util import json 26 27 # This is version 1, so it introduces a lot of new tables over version 0, 28 # which had no database. 29 30 TABLES = [ 31 # the schema here is defined as version 1 32 textwrap.dedent(""" 33 CREATE TABLE version ( 34 version INTEGER NOT NULL -- contains one row, currently set to 1 35 ); 36 """), 37 38 # last_access is used for logging, to record the last time that each 39 # client (or rather class of clients) touched the DB. The idea is that if 40 # something gets weird, you can check this and discover that you have an 41 # older tool (which uses a different schema) mucking things up. 42 textwrap.dedent(""" 43 CREATE TABLE last_access ( 44 `who` VARCHAR(256) NOT NULL, -- like 'buildbot-0.8.0' 45 `writing` INTEGER NOT NULL, -- 1 if you are writing, 0 if you are reading 46 -- PRIMARY KEY (who, writing), 47 `last_access` TIMESTAMP -- seconds since epoch 48 ); 49 """), 50 51 textwrap.dedent(""" 52 CREATE TABLE changes_nextid (next_changeid INTEGER); 53 """), 54 55 textwrap.dedent(""" 56 -- Changes are immutable: once added, never changed 57 CREATE TABLE changes ( 58 `changeid` INTEGER PRIMARY KEY NOT NULL, -- also serves as 'change number' 59 `author` VARCHAR(1024) NOT NULL, 60 `comments` VARCHAR(1024) NOT NULL, -- too short? 61 `is_dir` SMALLINT NOT NULL, -- old, for CVS 62 `branch` VARCHAR(1024) NULL, 63 `revision` VARCHAR(256), -- CVS uses NULL. too short for darcs? 64 `revlink` VARCHAR(256) NULL, 65 `when_timestamp` INTEGER NOT NULL, -- copied from incoming Change 66 `category` VARCHAR(256) NULL 67 ); 68 """), 69 70 textwrap.dedent(""" 71 CREATE TABLE change_links ( 72 `changeid` INTEGER NOT NULL, 73 `link` VARCHAR(1024) NOT NULL 74 ); 75 """), 76 77 textwrap.dedent(""" 78 CREATE TABLE change_files ( 79 `changeid` INTEGER NOT NULL, 80 `filename` VARCHAR(1024) NOT NULL 81 ); 82 """), 83 84 textwrap.dedent(""" 85 CREATE TABLE change_properties ( 86 `changeid` INTEGER NOT NULL, 87 `property_name` VARCHAR(256) NOT NULL, 88 `property_value` VARCHAR(1024) NOT NULL -- too short? 89 ); 90 """), 91 92 # Scheduler tables 93 textwrap.dedent(""" 94 CREATE TABLE schedulers ( 95 `schedulerid` INTEGER PRIMARY KEY, -- joins to other tables 96 `name` VARCHAR(127) UNIQUE NOT NULL, 97 `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary 98 ); 99 """), 100 101 textwrap.dedent(""" 102 CREATE TABLE scheduler_changes ( 103 `schedulerid` INTEGER, 104 `changeid` INTEGER, 105 `important` SMALLINT 106 ); 107 """), 108 109 textwrap.dedent(""" 110 CREATE TABLE scheduler_upstream_buildsets ( 111 `buildsetid` INTEGER, 112 `schedulerid` INTEGER, 113 `active` SMALLINT 114 ); 115 """), 116 117 # SourceStamps 118 textwrap.dedent(""" 119 -- SourceStamps are immutable: once added, never changed 120 CREATE TABLE sourcestamps ( 121 `id` INTEGER PRIMARY KEY, 122 `branch` VARCHAR(256) default NULL, 123 `revision` VARCHAR(256) default NULL, 124 `patchid` INTEGER default NULL 125 ); 126 """), 127 textwrap.dedent(""" 128 CREATE TABLE patches ( 129 `id` INTEGER PRIMARY KEY, 130 `patchlevel` INTEGER NOT NULL, 131 `patch_base64` TEXT NOT NULL, -- encoded bytestring 132 `subdir` TEXT -- usually NULL 133 ); 134 """), 135 textwrap.dedent(""" 136 CREATE TABLE sourcestamp_changes ( 137 `sourcestampid` INTEGER NOT NULL, 138 `changeid` INTEGER NOT NULL 139 ); 140 """), 141 142 # BuildRequests 143 textwrap.dedent(""" 144 -- BuildSets are mutable. Python code may not cache them. Every 145 -- BuildRequest must have exactly one associated BuildSet. 146 CREATE TABLE buildsets ( 147 `id` INTEGER PRIMARY KEY NOT NULL, 148 `external_idstring` VARCHAR(256), 149 `reason` VARCHAR(256), 150 `sourcestampid` INTEGER NOT NULL, 151 `submitted_at` INTEGER NOT NULL, 152 `complete` SMALLINT NOT NULL default 0, 153 `complete_at` INTEGER, 154 `results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py 155 -- results is NULL until complete==1 156 ); 157 """), 158 textwrap.dedent(""" 159 CREATE TABLE buildset_properties ( 160 `buildsetid` INTEGER NOT NULL, 161 `property_name` VARCHAR(256) NOT NULL, 162 `property_value` VARCHAR(1024) NOT NULL -- too short? 163 ); 164 """), 165 166 textwrap.dedent(""" 167 -- the buildrequests table represents the queue of builds that need to be 168 -- done. In an idle buildbot, all requests will have complete=1. 169 -- BuildRequests are mutable. Python code may not cache them. 170 CREATE TABLE buildrequests ( 171 `id` INTEGER PRIMARY KEY NOT NULL, 172 173 -- every BuildRequest has a BuildSet 174 -- the sourcestampid and reason live in the BuildSet 175 `buildsetid` INTEGER NOT NULL, 176 177 `buildername` VARCHAR(256) NOT NULL, 178 179 `priority` INTEGER NOT NULL default 0, 180 181 -- claimed_at is the time at which a master most recently asserted that 182 -- it is responsible for running the build: this will be updated 183 -- periodically to maintain the claim 184 `claimed_at` INTEGER default 0, 185 186 -- claimed_by indicates which buildmaster has claimed this request. The 187 -- 'name' contains hostname/basedir, and will be the same for subsequent 188 -- runs of any given buildmaster. The 'incarnation' contains bootime/pid, 189 -- and will be different for subsequent runs. This allows each buildmaster 190 -- to distinguish their current claims, their old claims, and the claims 191 -- of other buildmasters, to treat them each appropriately. 192 `claimed_by_name` VARCHAR(256) default NULL, 193 `claimed_by_incarnation` VARCHAR(256) default NULL, 194 195 `complete` INTEGER default 0, -- complete=0 means 'pending' 196 197 -- results is only valid when complete==1 198 `results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py 199 200 `submitted_at` INTEGER NOT NULL, 201 202 `complete_at` INTEGER 203 ); 204 """), 205 206 textwrap.dedent(""" 207 -- this records which builds have been started for each request 208 CREATE TABLE builds ( 209 `id` INTEGER PRIMARY KEY NOT NULL, 210 `number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number) 211 -- 'number' is scoped to both the local buildmaster and the buildername 212 `brid` INTEGER NOT NULL, -- matches buildrequests.id 213 `start_time` INTEGER NOT NULL, 214 `finish_time` INTEGER 215 ); 216 """), 217 ] 218

219 -class Upgrader(base.Upgrader):

220 - def upgrade(self):

221 self.test_unicode() 222 self.add_tables() 223 self.migrate_changes() 224 self.set_version()

225

226 - def test_unicode(self):

227 # first, create a test table 228 c = self.conn.cursor() 229 c.execute("CREATE TABLE test_unicode (`name` VARCHAR(100))") 230 q = util.sql_insert(self.dbapi, 'test_unicode', ["name"]) 231 try: 232 val = u"Frosty the \N{SNOWMAN}" 233 c.execute(q, [val]) 234 c.execute("SELECT * FROM test_unicode") 235 row = c.fetchall()[0] 236 if row[0] != val: 237 raise UnicodeError("Your database doesn't support unicode data; for MySQL, set the default collation to utf8_general_ci.") 238 finally: 239 pass 240 c.execute("DROP TABLE test_unicode")

241

242 - def add_tables(self):

243 # first, add all of the tables 244 c = self.conn.cursor() 245 for t in TABLES: 246 try: 247 c.execute(t) 248 except: 249 print >>sys.stderr, "error executing SQL query: %s" % t 250 raise

251

252 - def _addChangeToDatabase(self, change, cursor):

253 # strip None from any of these values, just in case 254 def remove_none(x): 255 if x is None: return u"" 256 elif isinstance(x, str): 257 return x.decode("utf8") 258 else: 259 return x

260 try: 261 values = tuple(remove_none(x) for x in 262 (change.number, change.who, 263 change.comments, change.isdir, 264 change.branch, change.revision, change.revlink, 265 change.when, change.category)) 266 except UnicodeDecodeError, e: 267 raise UnicodeError("Trying to import change data as UTF-8 failed. Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e)) 268 269 q = util.sql_insert(self.dbapi, 'changes', 270 """changeid author comments is_dir branch revision 271 revlink when_timestamp category""".split()) 272 cursor.execute(q, values) 273 274 for link in change.links: 275 cursor.execute(util.sql_insert(self.dbapi, 'change_links', ('changeid', 'link')), 276 (change.number, link)) 277 278 # sometimes change.files contains nested lists -- why, I do not know! But we deal with 279 # it all the same - see bug #915. We'll assume for now that change.files contains *either* 280 # lists of filenames or plain filenames, not both. 281 def flatten(l): 282 if l and type(l[0]) == list: 283 rv = [] 284 for e in l: 285 if type(e) == list: 286 rv.extend(e) 287 else: 288 rv.append(e) 289 return rv 290 else: 291 return l

292 for filename in flatten(change.files): 293 cursor.execute(util.sql_insert(self.dbapi, 'change_files', ('changeid', 'filename')), 294 (change.number, filename)) 295 for propname,propvalue in change.properties.properties.items(): 296 encoded_value = json.dumps(propvalue) 297 cursor.execute(util.sql_insert(self.dbapi, 'change_properties', 298 ('changeid', 'property_name', 'property_value')), 299 (change.number, propname, encoded_value)) 300

301 - def migrate_changes(self):

302 # if we still have a changes.pck, then we need to migrate it 303 changes_pickle = os.path.join(self.basedir, "changes.pck") 304 if os.path.exists(changes_pickle): 305 if not self.quiet: print "migrating changes.pck to database" 306 307 # 'source' will be an old b.c.changes.ChangeMaster instance, with a 308 # .changes attribute 309 source = cPickle.load(open(changes_pickle,"rb")) 310 styles.doUpgrade() 311 312 if not self.quiet: print " (%d Change objects)" % len(source.changes) 313 314 # first, scan for changes without a number. If we find any, then we'll 315 # renumber the changes sequentially 316 have_unnumbered = False 317 for c in source.changes: 318 if c.revision and c.number is None: 319 have_unnumbered = True 320 break 321 if have_unnumbered: 322 n = 1 323 for c in source.changes: 324 if c.revision: 325 c.number = n 326 n = n + 1 327 328 # insert the changes 329 cursor = self.conn.cursor() 330 for c in source.changes: 331 if not c.revision: 332 continue 333 self._addChangeToDatabase(c, cursor) 334 335 # update next_changeid 336 max_changeid = max([ c.number for c in source.changes if c.revision ] + [ 0 ]) 337 cursor.execute("""INSERT into changes_nextid VALUES (%d)""" % (max_changeid+1)) 338 339 if not self.quiet: 340 print "moving changes.pck to changes.pck.old; delete it or keep it as a backup" 341 os.rename(changes_pickle, changes_pickle+".old") 342 else: 343 c = self.conn.cursor() 344 c.execute("""INSERT into changes_nextid VALUES (1)""")

345

346 - def set_version(self):

347 c = self.conn.cursor() 348 c.execute("""INSERT INTO version VALUES (1)""")

349