Package buildbot :: Package db :: Package schema :: Module v1
[frames] | no frames]

Source Code for Module buildbot.db.schema.v1

  1  # ***** BEGIN LICENSE BLOCK ***** 
  2  # Version: MPL 1.1/GPL 2.0/LGPL 2.1 
  3  # 
  4  # The contents of this file are subject to the Mozilla Public License Version 
  5  # 1.1 (the "License"); you may not use this file except in compliance with 
  6  # the License. You may obtain a copy of the License at 
  7  # http://www.mozilla.org/MPL/ 
  8  # 
  9  # Software distributed under the License is distributed on an "AS IS" basis, 
 10  # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 
 11  # for the specific language governing rights and limitations under the 
 12  # License. 
 13  # 
 14  # The Original Code is Mozilla-specific Buildbot steps. 
 15  # 
 16  # The Initial Developer of the Original Code is 
 17  # Mozilla Foundation. 
 18  # Portions created by the Initial Developer are Copyright (C) 2009 
 19  # the Initial Developer. All Rights Reserved. 
 20  # 
 21  # Contributor(s): 
 22  #   Brian Warner <warner@lothar.com> 
 23  #   Chris AtLee <catlee@mozilla.com> 
 24  # 
 25  # Alternatively, the contents of this file may be used under the terms of 
 26  # either the GNU General Public License Version 2 or later (the "GPL"), or 
 27  # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), 
 28  # in which case the provisions of the GPL or the LGPL are applicable instead 
 29  # of those above. If you wish to allow use of your version of this file only 
 30  # under the terms of either the GPL or the LGPL, and not to allow others to 
 31  # use your version of this file under the terms of the MPL, indicate your 
 32  # decision by deleting the provisions above and replace them with the notice 
 33  # and other provisions required by the GPL or the LGPL. If you do not delete 
 34  # the provisions above, a recipient may use your version of this file under 
 35  # the terms of any one of the MPL, the GPL or the LGPL. 
 36  # 
 37  # ***** END LICENSE BLOCK ***** 
 38   
 39  import cPickle 
 40  import textwrap 
 41  import os 
 42  import sys 
 43   
 44  from twisted.persisted import styles 
 45   
 46  from buildbot.db import util 
 47  from buildbot.db.schema import base 
 48  from buildbot.util import json 
 49   
 50  # This is version 1, so it introduces a lot of new tables over version 0, 
 51  # which had no database. 
 52   
 53  TABLES = [ 
 54      # the schema here is defined as version 1 
 55      textwrap.dedent(""" 
 56          CREATE TABLE version ( 
 57              version INTEGER NOT NULL -- contains one row, currently set to 1 
 58          ); 
 59      """), 
 60   
 61      # last_access is used for logging, to record the last time that each 
 62      # client (or rather class of clients) touched the DB. The idea is that if 
 63      # something gets weird, you can check this and discover that you have an 
 64      # older tool (which uses a different schema) mucking things up. 
 65      textwrap.dedent(""" 
 66          CREATE TABLE last_access ( 
 67              `who` VARCHAR(256) NOT NULL, -- like 'buildbot-0.8.0' 
 68              `writing` INTEGER NOT NULL, -- 1 if you are writing, 0 if you are reading 
 69              -- PRIMARY KEY (who, writing), 
 70              `last_access` TIMESTAMP     -- seconds since epoch 
 71          ); 
 72      """), 
 73   
 74      textwrap.dedent(""" 
 75          CREATE TABLE changes_nextid (next_changeid INTEGER); 
 76      """), 
 77   
 78      textwrap.dedent(""" 
 79          -- Changes are immutable: once added, never changed 
 80          CREATE TABLE changes ( 
 81              `changeid` INTEGER PRIMARY KEY NOT NULL, -- also serves as 'change number' 
 82              `author` VARCHAR(1024) NOT NULL, 
 83              `comments` VARCHAR(1024) NOT NULL, -- too short? 
 84              `is_dir` SMALLINT NOT NULL, -- old, for CVS 
 85              `branch` VARCHAR(1024) NULL, 
 86              `revision` VARCHAR(256), -- CVS uses NULL. too short for darcs? 
 87              `revlink` VARCHAR(256) NULL, 
 88              `when_timestamp` INTEGER NOT NULL, -- copied from incoming Change 
 89              `category` VARCHAR(256) NULL 
 90          ); 
 91      """), 
 92   
 93      textwrap.dedent(""" 
 94          CREATE TABLE change_links ( 
 95              `changeid` INTEGER NOT NULL, 
 96              `link` VARCHAR(1024) NOT NULL 
 97          ); 
 98      """), 
 99   
100      textwrap.dedent(""" 
101          CREATE TABLE change_files ( 
102              `changeid` INTEGER NOT NULL, 
103              `filename` VARCHAR(1024) NOT NULL 
104          ); 
105      """), 
106   
107      textwrap.dedent(""" 
108          CREATE TABLE change_properties ( 
109              `changeid` INTEGER NOT NULL, 
110              `property_name` VARCHAR(256) NOT NULL, 
111              `property_value` VARCHAR(1024) NOT NULL -- too short? 
112          ); 
113      """), 
114   
115      # Scheduler tables 
116      textwrap.dedent(""" 
117          CREATE TABLE schedulers ( 
118              `schedulerid` INTEGER PRIMARY KEY, -- joins to other tables 
119              `name` VARCHAR(127) UNIQUE NOT NULL, 
120              `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary 
121          ); 
122      """), 
123   
124      textwrap.dedent(""" 
125          CREATE TABLE scheduler_changes ( 
126              `schedulerid` INTEGER, 
127              `changeid` INTEGER, 
128              `important` SMALLINT 
129          ); 
130      """), 
131   
132      textwrap.dedent(""" 
133          CREATE TABLE scheduler_upstream_buildsets ( 
134              `buildsetid` INTEGER, 
135              `schedulerid` INTEGER, 
136              `active` SMALLINT 
137          ); 
138      """), 
139   
140      # SourceStamps 
141      textwrap.dedent(""" 
142          -- SourceStamps are immutable: once added, never changed 
143          CREATE TABLE sourcestamps ( 
144              `id` INTEGER PRIMARY KEY, 
145              `branch` VARCHAR(256) default NULL, 
146              `revision` VARCHAR(256) default NULL, 
147              `patchid` INTEGER default NULL 
148          ); 
149      """), 
150      textwrap.dedent(""" 
151          CREATE TABLE patches ( 
152              `id` INTEGER PRIMARY KEY, 
153              `patchlevel` INTEGER NOT NULL, 
154              `patch_base64` TEXT NOT NULL, -- encoded bytestring 
155              `subdir` TEXT -- usually NULL 
156          ); 
157      """), 
158      textwrap.dedent(""" 
159          CREATE TABLE sourcestamp_changes ( 
160              `sourcestampid` INTEGER NOT NULL, 
161              `changeid` INTEGER NOT NULL 
162          ); 
163      """), 
164   
165      # BuildRequests 
166      textwrap.dedent(""" 
167          -- BuildSets are mutable. Python code may not cache them. Every 
168          -- BuildRequest must have exactly one associated BuildSet. 
169          CREATE TABLE buildsets ( 
170              `id` INTEGER PRIMARY KEY NOT NULL, 
171              `external_idstring` VARCHAR(256), 
172              `reason` VARCHAR(256), 
173              `sourcestampid` INTEGER NOT NULL, 
174              `submitted_at` INTEGER NOT NULL, 
175              `complete` SMALLINT NOT NULL default 0, 
176              `complete_at` INTEGER, 
177              `results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py 
178               -- results is NULL until complete==1 
179          ); 
180      """), 
181      textwrap.dedent(""" 
182          CREATE TABLE buildset_properties ( 
183              `buildsetid` INTEGER NOT NULL, 
184              `property_name` VARCHAR(256) NOT NULL, 
185              `property_value` VARCHAR(1024) NOT NULL -- too short? 
186          ); 
187      """), 
188   
189      textwrap.dedent(""" 
190          -- the buildrequests table represents the queue of builds that need to be 
191          -- done. In an idle buildbot, all requests will have complete=1. 
192          -- BuildRequests are mutable. Python code may not cache them. 
193          CREATE TABLE buildrequests ( 
194              `id` INTEGER PRIMARY KEY NOT NULL, 
195   
196              -- every BuildRequest has a BuildSet 
197              -- the sourcestampid and reason live in the BuildSet 
198              `buildsetid` INTEGER NOT NULL, 
199   
200              `buildername` VARCHAR(256) NOT NULL, 
201   
202              `priority` INTEGER NOT NULL default 0, 
203   
204              -- claimed_at is the time at which a master most recently asserted that 
205              -- it is responsible for running the build: this will be updated 
206              -- periodically to maintain the claim 
207              `claimed_at` INTEGER default 0, 
208   
209              -- claimed_by indicates which buildmaster has claimed this request. The 
210              -- 'name' contains hostname/basedir, and will be the same for subsequent 
211              -- runs of any given buildmaster. The 'incarnation' contains bootime/pid, 
212              -- and will be different for subsequent runs. This allows each buildmaster 
213              -- to distinguish their current claims, their old claims, and the claims 
214              -- of other buildmasters, to treat them each appropriately. 
215              `claimed_by_name` VARCHAR(256) default NULL, 
216              `claimed_by_incarnation` VARCHAR(256) default NULL, 
217   
218              `complete` INTEGER default 0, -- complete=0 means 'pending' 
219   
220               -- results is only valid when complete==1 
221              `results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py 
222   
223              `submitted_at` INTEGER NOT NULL, 
224   
225              `complete_at` INTEGER 
226          ); 
227      """), 
228   
229      textwrap.dedent(""" 
230          -- this records which builds have been started for each request 
231          CREATE TABLE builds ( 
232              `id` INTEGER PRIMARY KEY NOT NULL, 
233              `number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number) 
234              -- 'number' is scoped to both the local buildmaster and the buildername 
235              `brid` INTEGER NOT NULL, -- matches buildrequests.id 
236              `start_time` INTEGER NOT NULL, 
237              `finish_time` INTEGER 
238          ); 
239      """), 
240  ] 
241   
242 -class Upgrader(base.Upgrader):
243 - def upgrade(self):
244 self.test_unicode() 245 self.add_tables() 246 self.migrate_changes() 247 self.set_version()
248
249 - def test_unicode(self):
250 # first, create a test table 251 c = self.conn.cursor() 252 c.execute("CREATE TABLE test_unicode (`name` VARCHAR(100))") 253 q = util.sql_insert(self.dbapi, 'test_unicode', ["name"]) 254 try: 255 val = u"Frosty the \N{SNOWMAN}" 256 c.execute(q, [val]) 257 c.execute("SELECT * FROM test_unicode") 258 row = c.fetchall()[0] 259 if row[0] != val: 260 raise UnicodeError("Your database doesn't support unicode data; for MySQL, set the default collation to utf8_general_ci.") 261 finally: 262 pass 263 c.execute("DROP TABLE test_unicode")
264
265 - def add_tables(self):
266 # first, add all of the tables 267 c = self.conn.cursor() 268 for t in TABLES: 269 try: 270 c.execute(t) 271 except: 272 print >>sys.stderr, "error executing SQL query: %s" % t 273 raise
274
275 - def _addChangeToDatabase(self, change, cursor):
276 # strip None from any of these values, just in case 277 def remove_none(x): 278 if x is None: return u"" 279 elif isinstance(x, str): 280 return x.decode("utf8") 281 else: 282 return x
283 try: 284 values = tuple(remove_none(x) for x in 285 (change.number, change.who, 286 change.comments, change.isdir, 287 change.branch, change.revision, change.revlink, 288 change.when, change.category)) 289 except UnicodeDecodeError, e: 290 raise UnicodeError("Trying to import change data as UTF-8 failed. Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e)) 291 292 q = util.sql_insert(self.dbapi, 'changes', 293 """changeid author comments is_dir branch revision 294 revlink when_timestamp category""".split()) 295 cursor.execute(q, values) 296 297 for link in change.links: 298 cursor.execute(util.sql_insert(self.dbapi, 'change_links', ('changeid', 'link')), 299 (change.number, link)) 300 301 # sometimes change.files contains nested lists -- why, I do not know! But we deal with 302 # it all the same - see bug #915. We'll assume for now that change.files contains *either* 303 # lists of filenames or plain filenames, not both. 304 def flatten(l): 305 if l and type(l[0]) == list: 306 rv = [] 307 for e in l: 308 if type(e) == list: 309 rv.extend(e) 310 else: 311 rv.append(e) 312 return rv 313 else: 314 return l
315 for filename in flatten(change.files): 316 cursor.execute(util.sql_insert(self.dbapi, 'change_files', ('changeid', 'filename')), 317 (change.number, filename)) 318 for propname,propvalue in change.properties.properties.items(): 319 encoded_value = json.dumps(propvalue) 320 cursor.execute(util.sql_insert(self.dbapi, 'change_properties', 321 ('changeid', 'property_name', 'property_value')), 322 (change.number, propname, encoded_value)) 323
324 - def migrate_changes(self):
325 # if we still have a changes.pck, then we need to migrate it 326 changes_pickle = os.path.join(self.basedir, "changes.pck") 327 if os.path.exists(changes_pickle): 328 if not self.quiet: print "migrating changes.pck to database" 329 330 # 'source' will be an old b.c.changes.ChangeMaster instance, with a 331 # .changes attribute 332 source = cPickle.load(open(changes_pickle,"rb")) 333 styles.doUpgrade() 334 335 if not self.quiet: print " (%d Change objects)" % len(source.changes) 336 337 # first, scan for changes without a number. If we find any, then we'll 338 # renumber the changes sequentially 339 have_unnumbered = False 340 for c in source.changes: 341 if c.revision and c.number is None: 342 have_unnumbered = True 343 break 344 if have_unnumbered: 345 n = 1 346 for c in source.changes: 347 if c.revision: 348 c.number = n 349 n = n + 1 350 351 # insert the changes 352 cursor = self.conn.cursor() 353 for c in source.changes: 354 if not c.revision: 355 continue 356 self._addChangeToDatabase(c, cursor) 357 358 # update next_changeid 359 max_changeid = max([ c.number for c in source.changes if c.revision ] + [ 0 ]) 360 cursor.execute("""INSERT into changes_nextid VALUES (%d)""" % (max_changeid+1)) 361 362 if not self.quiet: 363 print "moving changes.pck to changes.pck.old; delete it or keep it as a backup" 364 os.rename(changes_pickle, changes_pickle+".old") 365 else: 366 c = self.conn.cursor() 367 c.execute("""INSERT into changes_nextid VALUES (1)""")
368
369 - def set_version(self):
370 c = self.conn.cursor() 371 c.execute("""INSERT INTO version VALUES (1)""")
372