Source Code for Module buildbot.db.changes

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  """ 
 17  Support for changes in the database 
 18  """ 
 19   
 20  from buildbot.util import json 
 21  import sqlalchemy as sa 
 22  from twisted.internet import defer, reactor 
 23  from buildbot.db import base 
 24  from buildbot.util import epoch2datetime, datetime2epoch 
 25   
 26 -class ChDict(dict): 
 27      pass 
 28   
 29 -class ChangesConnectorComponent(base.DBConnectorComponent): 
 30      # Documentation is in developer/database.rst 
 31   
 32 -    def addChange(self, author=None, files=None, comments=None, is_dir=0, 
 33              revision=None, when_timestamp=None, branch=None, 
 34              category=None, revlink='', properties={}, repository='', 
 35              project='', uid=None, _reactor=reactor): 
 36          assert project is not None, "project must be a string, not None" 
 37          assert repository is not None, "repository must be a string, not None" 
 38   
 39          if when_timestamp is None: 
 40              when_timestamp = epoch2datetime(_reactor.seconds()) 
 41   
 42          # verify that source is 'Change' for each property 
 43          for pv in properties.values(): 
 44              assert pv[1] == 'Change', ("properties must be qualified with" 
 45                                         "source 'Change'") 
 46   
 47          def thd(conn): 
 48              # note that in a read-uncommitted database like SQLite this 
 49              # transaction does not buy atomicitiy - other database users may 
 50              # still come across a change without its files, properties, 
 51              # etc.  That's OK, since we don't announce the change until it's 
 52              # all in the database, but beware. 
 53   
 54              transaction = conn.begin() 
 55   
 56              ch_tbl = self.db.model.changes 
 57   
 58              self.check_length(ch_tbl.c.author, author) 
 59              self.check_length(ch_tbl.c.comments, comments) 
 60              self.check_length(ch_tbl.c.branch, branch) 
 61              self.check_length(ch_tbl.c.revision, revision) 
 62              self.check_length(ch_tbl.c.revlink, revlink) 
 63              self.check_length(ch_tbl.c.category, category) 
 64              self.check_length(ch_tbl.c.repository, repository) 
 65              self.check_length(ch_tbl.c.project, project) 
 66   
 67              r = conn.execute(ch_tbl.insert(), dict( 
 68                  author=author, 
 69                  comments=comments, 
 70                  is_dir=is_dir, 
 71                  branch=branch, 
 72                  revision=revision, 
 73                  revlink=revlink, 
 74                  when_timestamp=datetime2epoch(when_timestamp), 
 75                  category=category, 
 76                  repository=repository, 
 77                  project=project)) 
 78              changeid = r.inserted_primary_key[0] 
 79              if files: 
 80                  tbl = self.db.model.change_files 
 81                  for f in files: 
 82                      self.check_length(tbl.c.filename, f) 
 83                  conn.execute(tbl.insert(), [ 
 84                      dict(changeid=changeid, filename=f) 
 85                          for f in files 
 86                      ]) 
 87              if properties: 
 88                  tbl = self.db.model.change_properties 
 89                  inserts = [ 
 90                      dict(changeid=changeid, 
 91                          property_name=k, 
 92                          property_value=json.dumps(v)) 
 93                      for k,v in properties.iteritems() 
 94                  ] 
 95                  for i in inserts: 
 96                      self.check_length(tbl.c.property_name, 
 97                              i['property_name']) 
 98                      self.check_length(tbl.c.property_value, 
 99                              i['property_value']) 
100   
101                  conn.execute(tbl.insert(), inserts) 
102              if uid: 
103                  ins = self.db.model.change_users.insert() 
104                  conn.execute(ins, dict(changeid=changeid, uid=uid)) 
105   
106              transaction.commit() 
107   
108              return changeid 
109          d = self.db.pool.do(thd) 
110          return d 
111   
112      @base.cached("chdicts") 
113 -    def getChange(self, changeid): 
114          assert changeid >= 0 
115          def thd(conn): 
116              # get the row from the 'changes' table 
117              changes_tbl = self.db.model.changes 
118              q = changes_tbl.select(whereclause=(changes_tbl.c.changeid == changeid)) 
119              rp = conn.execute(q) 
120              row = rp.fetchone() 
121              if not row: 
122                  return None 
123              # and fetch the ancillary data (files, properties) 
124              return self._chdict_from_change_row_thd(conn, row) 
125          d = self.db.pool.do(thd) 
126          return d 
127   
128 -    def getChangeUids(self, changeid): 
129          assert changeid >= 0 
130          def thd(conn): 
131              cu_tbl = self.db.model.change_users 
132              q = cu_tbl.select(whereclause=(cu_tbl.c.changeid == changeid)) 
133              res = conn.execute(q) 
134              rows = res.fetchall() 
135              row_uids = [ row.uid for row in rows ] 
136              return row_uids 
137          d = self.db.pool.do(thd) 
138          return d 
139   
140 -    def getRecentChanges(self, count): 
141          def thd(conn): 
142              # get the changeids from the 'changes' table 
143              changes_tbl = self.db.model.changes 
144              q = sa.select([changes_tbl.c.changeid], 
145                      order_by=[sa.desc(changes_tbl.c.changeid)], 
146                      limit=count) 
147              rp = conn.execute(q) 
148              changeids = [ row.changeid for row in rp ] 
149              rp.close() 
150              return list(reversed(changeids)) 
151          d = self.db.pool.do(thd) 
152   
153          # then turn those into changes, using the cache 
154          def get_changes(changeids): 
155              return defer.gatherResults([ self.getChange(changeid) 
156                                           for changeid in changeids ]) 
157          d.addCallback(get_changes) 
158          return d 
159   
160 -    def getLatestChangeid(self): 
161          def thd(conn): 
162              changes_tbl = self.db.model.changes 
163              q = sa.select([ changes_tbl.c.changeid ], 
164                      order_by=sa.desc(changes_tbl.c.changeid), 
165                      limit=1) 
166              return conn.scalar(q) 
167          d = self.db.pool.do(thd) 
168          return d 
169   
170      # utility methods 
171   
172 -    def pruneChanges(self, changeHorizon): 
173          """ 
174          Called periodically by DBConnector, this method deletes changes older 
175          than C{changeHorizon}. 
176          """ 
177   
178          if not changeHorizon: 
179              return defer.succeed(None) 
180          def thd(conn): 
181              changes_tbl = self.db.model.changes 
182   
183              # First, get the list of changes to delete.  This could be written 
184              # as a subquery but then that subquery would be run for every 
185              # table, which is very inefficient; also, MySQL's subquery support 
186              # leaves much to be desired, and doesn't support this particular 
187              # form. 
188              q = sa.select([changes_tbl.c.changeid], 
189                            order_by=[sa.desc(changes_tbl.c.changeid)], 
190                            offset=changeHorizon) 
191              res = conn.execute(q) 
192              ids_to_delete = [ r.changeid for r in res ] 
193   
194              # and delete from all relevant tables, in dependency order 
195              for table_name in ('scheduler_changes', 'sourcestamp_changes', 
196                                 'change_files', 'change_properties', 'changes', 
197                                 'change_users'): 
198                  table = self.db.model.metadata.tables[table_name] 
199                  conn.execute( 
200                      table.delete(table.c.changeid.in_(ids_to_delete))) 
201          return self.db.pool.do(thd) 
202   
203 -    def _chdict_from_change_row_thd(self, conn, ch_row): 
204          # This method must be run in a db.pool thread, and returns a chdict 
205          # given a row from the 'changes' table 
206          change_files_tbl = self.db.model.change_files 
207          change_properties_tbl = self.db.model.change_properties 
208   
209          chdict = ChDict( 
210                  changeid=ch_row.changeid, 
211                  author=ch_row.author, 
212                  files=[], # see below 
213                  comments=ch_row.comments, 
214                  is_dir=ch_row.is_dir, 
215                  revision=ch_row.revision, 
216                  when_timestamp=epoch2datetime(ch_row.when_timestamp), 
217                  branch=ch_row.branch, 
218                  category=ch_row.category, 
219                  revlink=ch_row.revlink, 
220                  properties={}, # see below 
221                  repository=ch_row.repository, 
222                  project=ch_row.project) 
223   
224          query = change_files_tbl.select( 
225                  whereclause=(change_files_tbl.c.changeid == ch_row.changeid)) 
226          rows = conn.execute(query) 
227          for r in rows: 
228              chdict['files'].append(r.filename) 
229   
230          # and properties must be given without a source, so strip that, but 
231          # be flexible in case users have used a development version where the 
232          # change properties were recorded incorrectly 
233          def split_vs(vs): 
234              try: 
235                  v,s = vs 
236                  if s != "Change": 
237                      v,s = vs, "Change" 
238              except: 
239                  v,s = vs, "Change" 
240              return v, s 
241   
242          query = change_properties_tbl.select( 
243                  whereclause=(change_properties_tbl.c.changeid == ch_row.changeid)) 
244          rows = conn.execute(query) 
245          for r in rows: 
246              try: 
247                  v, s = split_vs(json.loads(r.property_value)) 
248                  chdict['properties'][r.property_name] = (v,s) 
249              except ValueError: 
250                  pass 
251   
252          return chdict 
253