buildbot.db.changes

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 """ 17 Support for changes in the database 18 """ 19 20 from buildbot.util import json 21 import sqlalchemy as sa 22 from twisted.internet import defer, reactor 23 from buildbot.db import base 24 from buildbot.util import epoch2datetime, datetime2epoch

25 26 -class ChDict(dict):

27 pass

28

29 -class ChangesConnectorComponent(base.DBConnectorComponent):

30 """ 31 A DBConnectorComponent to handle getting changes into and out of the 32 database. An instance is available at C{master.db.changes}. 33 34 Changes are represented as dictionaries with the following keys: 35 36 - changeid: the ID of this change 37 - author: the author of the change (unicode string) 38 - files: list of source-code filenames changed (unicode strings) 39 - comments: user comments (unicode string) 40 - is_dir: deprecated 41 - links: list of links for this change, e.g., to web views, review 42 (unicode strings) 43 - revision: revision for this change (unicode string), or None if unknown 44 - when_timestamp: time of the commit (datetime instance) 45 - branch: branch on which the change took place (unicode string), or None 46 for the "default branch", whatever that might mean 47 - category: user-defined category of this change (unicode string or None) 48 - revlink: link to a web view of this change (unicode string or None) 49 - properties: user-specified properties for this change, represented as a 50 dictionary mapping keys to (value, source) 51 - repository: repository where this change occurred (unicode string) 52 - project: user-defined project to which this change corresponds (unicode 53 string) 54 """ 55

56 - def addChange(self, author=None, files=None, comments=None, is_dir=0, 57 links=None, revision=None, when_timestamp=None, branch=None, 58 category=None, revlink='', properties={}, repository='', 59 project='', _reactor=reactor):

60 """Add a Change with the given attributes to the database; returns 61 a Change instance via a deferred. All arguments are keyword arguments. 62 63 @param author: the author of this change 64 @type author: unicode string 65 66 @param files: a list of filenames that were changed 67 @type branch: list of unicode strings 68 69 @param comments: user comments on the change 70 @type branch: unicode string 71 72 @param is_dir: deprecated 73 74 @param links: a list of links related to this change, e.g., to web 75 viewers or review pages 76 @type links: list of unicode strings 77 78 @param revision: the revision identifier for this change 79 @type revision: unicode string 80 81 @param when_timestamp: when this change occurred, or the current time 82 if None 83 @type when_timestamp: datetime instance or None 84 85 @param branch: the branch on which this change took place 86 @type branch: unicode string 87 88 @param category: category for this change (arbitrary use by Buildbot 89 users) 90 @type category: unicode string 91 92 @param revlink: link to a web view of this revision 93 @type revlink: unicode string 94 95 @param properties: properties to set on this change 96 @type properties: dictionary, where values are tuples of (value, 97 source). At the moment, the source must be C{'Change'}, although 98 this may be relaxed in later versions. 99 100 @param repository: the repository in which this change took place 101 @type repository: unicode string 102 103 @param project: the project this change is a part of 104 @type project: unicode string 105 106 @param _reactor: for testing 107 108 @returns: new change's ID via Deferred 109 """ 110 assert project is not None, "project must be a string, not None" 111 assert repository is not None, "repository must be a string, not None" 112 113 if when_timestamp is None: 114 when_timestamp = epoch2datetime(_reactor.seconds()) 115 116 # verify that source is 'Change' for each property 117 for pv in properties.values(): 118 assert pv[1] == 'Change', ("properties must be qualified with" 119 "source 'Change'") 120 121 def thd(conn): 122 # note that in a read-uncommitted database like SQLite this 123 # transaction does not buy atomicitiy - other database users may 124 # still come across a change without its links, files, properties, 125 # etc. That's OK, since we don't announce the change until it's 126 # all in the database, but beware. 127 128 transaction = conn.begin() 129 130 ins = self.db.model.changes.insert() 131 r = conn.execute(ins, dict( 132 author=author, 133 comments=comments, 134 is_dir=is_dir, 135 branch=branch, 136 revision=revision, 137 revlink=revlink, 138 when_timestamp=datetime2epoch(when_timestamp), 139 category=category, 140 repository=repository, 141 project=project)) 142 changeid = r.inserted_primary_key[0] 143 if links: 144 ins = self.db.model.change_links.insert() 145 conn.execute(ins, [ 146 dict(changeid=changeid, link=l) 147 for l in links 148 ]) 149 if files: 150 ins = self.db.model.change_files.insert() 151 conn.execute(ins, [ 152 dict(changeid=changeid, filename=f) 153 for f in files 154 ]) 155 if properties: 156 ins = self.db.model.change_properties.insert() 157 conn.execute(ins, [ 158 dict(changeid=changeid, 159 property_name=k, 160 property_value=json.dumps(v)) 161 for k,v in properties.iteritems() 162 ]) 163 164 transaction.commit() 165 166 return changeid

167 d = self.db.pool.do(thd) 168 return d

169 170 @base.cached("chdicts")

171 - def getChange(self, changeid):

172 """ 173 Get a change dictionary for the given changeid, or None if no such 174 change exists. 175 176 @param changeid: the id of the change instance to fetch 177 178 @param no_cache: bypass cache and always fetch from database 179 @type no_cache: boolean 180 181 @returns: Change dictionary via Deferred 182 """ 183 assert changeid >= 0 184 def thd(conn): 185 # get the row from the 'changes' table 186 changes_tbl = self.db.model.changes 187 q = changes_tbl.select(whereclause=(changes_tbl.c.changeid == changeid)) 188 rp = conn.execute(q) 189 row = rp.fetchone() 190 if not row: 191 return None 192 # and fetch the ancillary data (links, files, properties) 193 return self._chdict_from_change_row_thd(conn, row)

194 d = self.db.pool.do(thd) 195 return d 196

197 - def getRecentChanges(self, count):

198 """ 199 Get a list of the C{count} most recent changes, represented as 200 dictionaies; returns fewer if that many do not exist. 201 202 @param count: maximum number of instances to return 203 204 @returns: list of dictionaries via Deferred, ordered by changeid 205 """ 206 def thd(conn): 207 # get the changeids from the 'changes' table 208 changes_tbl = self.db.model.changes 209 q = sa.select([changes_tbl.c.changeid], 210 order_by=[sa.desc(changes_tbl.c.changeid)], 211 limit=count) 212 rp = conn.execute(q) 213 changeids = [ row.changeid for row in rp ] 214 rp.close() 215 return list(reversed(changeids))

216 d = self.db.pool.do(thd) 217 218 # then turn those into changes, using the cache 219 def get_changes(changeids): 220 return defer.gatherResults([ self.getChange(changeid) 221 for changeid in changeids ]) 222 d.addCallback(get_changes) 223 return d 224

225 - def getLatestChangeid(self):

226 """ 227 Get the most-recently-assigned changeid, or None if there are no 228 changes at all. 229 230 @returns: changeid via Deferred 231 """ 232 def thd(conn): 233 changes_tbl = self.db.model.changes 234 q = sa.select([ changes_tbl.c.changeid ], 235 order_by=sa.desc(changes_tbl.c.changeid), 236 limit=1) 237 return conn.scalar(q)

238 d = self.db.pool.do(thd) 239 return d 240 241 # utility methods 242

243 - def pruneChanges(self, changeHorizon):

244 if not changeHorizon: 245 return defer.succeed(None) 246 def thd(conn): 247 changes_tbl = self.db.model.changes 248 249 # First, get the list of changes to delete. This could be written 250 # as a subquery but then that subquery would be run for every 251 # table, which is very inefficient; also, MySQL's subquery support 252 # leaves much to be desired, and doesn't support this particular 253 # form. 254 q = sa.select([changes_tbl.c.changeid], 255 order_by=[sa.desc(changes_tbl.c.changeid)], 256 offset=changeHorizon) 257 res = conn.execute(q) 258 ids_to_delete = [ r.changeid for r in res ] 259 260 # and delete from all relevant tables, in dependency order 261 for table_name in ('scheduler_changes', 'sourcestamp_changes', 262 'change_files', 'change_links', 263 'change_properties', 'changes'): 264 table = self.db.model.metadata.tables[table_name] 265 conn.execute( 266 table.delete(table.c.changeid.in_(ids_to_delete)))

267 return self.db.pool.do(thd) 268

269 - def _chdict_from_change_row_thd(self, conn, ch_row):

270 # This method must be run in a db.pool thread, and returns a chdict 271 # given a row from the 'changes' table 272 change_links_tbl = self.db.model.change_links 273 change_files_tbl = self.db.model.change_files 274 change_properties_tbl = self.db.model.change_properties 275 276 def mkdt(epoch): 277 if epoch: 278 return epoch2datetime(epoch)

279 280 chdict = ChDict( 281 changeid=ch_row.changeid, 282 author=ch_row.author, 283 files=[], # see below 284 comments=ch_row.comments, 285 is_dir=ch_row.is_dir, 286 links=[], # see below 287 revision=ch_row.revision, 288 when_timestamp=mkdt(ch_row.when_timestamp), 289 branch=ch_row.branch, 290 category=ch_row.category, 291 revlink=ch_row.revlink, 292 properties={}, # see below 293 repository=ch_row.repository, 294 project=ch_row.project) 295 296 query = change_links_tbl.select( 297 whereclause=(change_links_tbl.c.changeid == ch_row.changeid)) 298 rows = conn.execute(query) 299 for r in rows: 300 chdict['links'].append(r.link) 301 302 query = change_files_tbl.select( 303 whereclause=(change_files_tbl.c.changeid == ch_row.changeid)) 304 rows = conn.execute(query) 305 for r in rows: 306 chdict['files'].append(r.filename) 307 308 # and properties must be given without a source, so strip that, but 309 # be flexible in case users have used a development version where the 310 # change properties were recorded incorrectly 311 def split_vs(vs): 312 try: 313 v,s = vs 314 if s != "Change": 315 v,s = vs, "Change" 316 except: 317 v,s = vs, "Change" 318 return v, s 319 320 query = change_properties_tbl.select( 321 whereclause=(change_properties_tbl.c.changeid == ch_row.changeid)) 322 rows = conn.execute(query) 323 for r in rows: 324 v, s = split_vs(json.loads(r.property_value)) 325 chdict['properties'][r.property_name] = (v,s) 326 327 return chdict 328

Source Code for Module buildbot.db.changes