Package buildbot :: Module locks
[frames] | no frames]

Source Code for Module buildbot.locks

  1  # -*- test-case-name: buildbot.test.test_locks -*- 
  2   
  3  from twisted.python import log 
  4  from twisted.internet import reactor, defer 
  5  from buildbot import util 
  6   
  7  if False: # for debugging 
  8      debuglog = log.msg 
  9  else: 
 10      debuglog = lambda m: None 
 11   
12 -class BaseLock:
13 """ 14 Class handling claiming and releasing of L{self}, and keeping track of 15 current and waiting owners. 16 17 @note: Ideally, we'd like to maintain FIFO order. The place to do that 18 would be the L{isAvailable()} function. However, this function is 19 called by builds/steps both for the first time, and after waking 20 them up by L{self} from the L{self.waiting} queue. There is 21 currently no way of distinguishing between them. 22 """ 23 description = "<BaseLock>" 24
25 - def __init__(self, name, maxCount=1):
26 self.name = name # Name of the lock 27 self.waiting = [] # Current queue, tuples (LockAccess, deferred) 28 self.owners = [] # Current owners, tuples (owner, LockAccess) 29 self.maxCount=maxCount # maximal number of counting owners
30
31 - def __repr__(self):
32 return self.description
33
34 - def _getOwnersCount(self):
35 """ Return the number of current exclusive and counting owners. 36 37 @return: Tuple (number exclusive owners, number counting owners) 38 """ 39 num_excl, num_counting = 0, 0 40 for owner in self.owners: 41 if owner[1].mode == 'exclusive': 42 num_excl = num_excl + 1 43 else: # mode == 'counting' 44 num_counting = num_counting + 1 45 46 assert (num_excl == 1 and num_counting == 0) \ 47 or (num_excl == 0 and num_counting <= self.maxCount) 48 return num_excl, num_counting
49 50
51 - def isAvailable(self, access):
52 """ Return a boolean whether the lock is available for claiming """ 53 debuglog("%s isAvailable(%s): self.owners=%r" 54 % (self, access, self.owners)) 55 num_excl, num_counting = self._getOwnersCount() 56 if access.mode == 'counting': 57 # Wants counting access 58 return num_excl == 0 and num_counting < self.maxCount 59 else: 60 # Wants exclusive access 61 return num_excl == 0 and num_counting == 0
62
63 - def claim(self, owner, access):
64 """ Claim the lock (lock must be available) """ 65 debuglog("%s claim(%s, %s)" % (self, owner, access.mode)) 66 assert owner is not None 67 assert self.isAvailable(access), "ask for isAvailable() first" 68 69 assert isinstance(access, LockAccess) 70 assert access.mode in ['counting', 'exclusive'] 71 self.owners.append((owner, access)) 72 debuglog(" %s is claimed '%s'" % (self, access.mode))
73
74 - def release(self, owner, access):
75 """ Release the lock """ 76 assert isinstance(access, LockAccess) 77 78 debuglog("%s release(%s, %s)" % (self, owner, access.mode)) 79 entry = (owner, access) 80 assert entry in self.owners 81 self.owners.remove(entry) 82 # who can we wake up? 83 # After an exclusive access, we may need to wake up several waiting. 84 # Break out of the loop when the first waiting client should not be awakened. 85 num_excl, num_counting = self._getOwnersCount() 86 while len(self.waiting) > 0: 87 access, d = self.waiting[0] 88 if access.mode == 'counting': 89 if num_excl > 0 or num_counting == self.maxCount: 90 break 91 else: 92 num_counting = num_counting + 1 93 else: 94 # access.mode == 'exclusive' 95 if num_excl > 0 or num_counting > 0: 96 break 97 else: 98 num_excl = num_excl + 1 99 100 del self.waiting[0] 101 reactor.callLater(0, d.callback, self)
102
103 - def waitUntilMaybeAvailable(self, owner, access):
104 """Fire when the lock *might* be available. The caller will need to 105 check with isAvailable() when the deferred fires. This loose form is 106 used to avoid deadlocks. If we were interested in a stronger form, 107 this would be named 'waitUntilAvailable', and the deferred would fire 108 after the lock had been claimed. 109 """ 110 debuglog("%s waitUntilAvailable(%s)" % (self, owner)) 111 assert isinstance(access, LockAccess) 112 if self.isAvailable(access): 113 return defer.succeed(self) 114 d = defer.Deferred() 115 self.waiting.append((access, d)) 116 return d
117 118
119 -class RealMasterLock(BaseLock):
120 - def __init__(self, lockid):
121 BaseLock.__init__(self, lockid.name, lockid.maxCount) 122 self.description = "<MasterLock(%s, %s)>" % (self.name, self.maxCount)
123
124 - def getLock(self, slave):
125 return self
126
127 -class RealSlaveLock:
128 - def __init__(self, lockid):
129 self.name = lockid.name 130 self.maxCount = lockid.maxCount 131 self.maxCountForSlave = lockid.maxCountForSlave 132 self.description = "<SlaveLock(%s, %s, %s)>" % (self.name, 133 self.maxCount, 134 self.maxCountForSlave) 135 self.locks = {}
136
137 - def __repr__(self):
138 return self.description
139
140 - def getLock(self, slavebuilder):
141 slavename = slavebuilder.slave.slavename 142 if not self.locks.has_key(slavename): 143 maxCount = self.maxCountForSlave.get(slavename, 144 self.maxCount) 145 lock = self.locks[slavename] = BaseLock(self.name, maxCount) 146 desc = "<SlaveLock(%s, %s)[%s] %d>" % (self.name, maxCount, 147 slavename, id(lock)) 148 lock.description = desc 149 self.locks[slavename] = lock 150 return self.locks[slavename]
151 152
153 -class LockAccess(util.ComparableMixin):
154 """ I am an object representing a way to access a lock. 155 156 @param lockid: LockId instance that should be accessed. 157 @type lockid: A MasterLock or SlaveLock instance. 158 159 @param mode: Mode of accessing the lock. 160 @type mode: A string, either 'counting' or 'exclusive'. 161 """ 162 163 compare_attrs = ['lockid', 'mode']
164 - def __init__(self, lockid, mode):
165 self.lockid = lockid 166 self.mode = mode 167 168 assert isinstance(lockid, (MasterLock, SlaveLock)) 169 assert mode in ['counting', 'exclusive']
170 171
172 -class BaseLockId(util.ComparableMixin):
173 """ Abstract base class for LockId classes. 174 175 Sets up the 'access()' function for the LockId's available to the user 176 (MasterLock and SlaveLock classes). 177 Derived classes should add 178 - Comparison with the L{util.ComparableMixin} via the L{compare_attrs} 179 class variable. 180 - Link to the actual lock class should be added with the L{lockClass} 181 class variable. 182 """
183 - def access(self, mode):
184 """ Express how the lock should be accessed """ 185 assert mode in ['counting', 'exclusive'] 186 return LockAccess(self, mode)
187
188 - def defaultAccess(self):
189 """ For buildbot 0.7.7 compability: When user doesn't specify an access 190 mode, this one is chosen. 191 """ 192 return self.access('counting')
193 194 195 196 # master.cfg should only reference the following MasterLock and SlaveLock 197 # classes. They are identifiers that will be turned into real Locks later, 198 # via the BotMaster.getLockByID method. 199
200 -class MasterLock(BaseLockId):
201 """I am a semaphore that limits the number of simultaneous actions. 202 203 Builds and BuildSteps can declare that they wish to claim me as they run. 204 Only a limited number of such builds or steps will be able to run 205 simultaneously. By default this number is one, but my maxCount parameter 206 can be raised to allow two or three or more operations to happen at the 207 same time. 208 209 Use this to protect a resource that is shared among all builders and all 210 slaves, for example to limit the load on a common SVN repository. 211 """ 212 213 compare_attrs = ['name', 'maxCount'] 214 lockClass = RealMasterLock
215 - def __init__(self, name, maxCount=1):
216 self.name = name 217 self.maxCount = maxCount
218
219 -class SlaveLock(BaseLockId):
220 """I am a semaphore that limits simultaneous actions on each buildslave. 221 222 Builds and BuildSteps can declare that they wish to claim me as they run. 223 Only a limited number of such builds or steps will be able to run 224 simultaneously on any given buildslave. By default this number is one, 225 but my maxCount parameter can be raised to allow two or three or more 226 operations to happen on a single buildslave at the same time. 227 228 Use this to protect a resource that is shared among all the builds taking 229 place on each slave, for example to limit CPU or memory load on an 230 underpowered machine. 231 232 Each buildslave will get an independent copy of this semaphore. By 233 default each copy will use the same owner count (set with maxCount), but 234 you can provide maxCountForSlave with a dictionary that maps slavename to 235 owner count, to allow some slaves more parallelism than others. 236 237 """ 238 239 compare_attrs = ['name', 'maxCount', '_maxCountForSlaveList'] 240 lockClass = RealSlaveLock
241 - def __init__(self, name, maxCount=1, maxCountForSlave={}):
242 self.name = name 243 self.maxCount = maxCount 244 self.maxCountForSlave = maxCountForSlave 245 # for comparison purposes, turn this dictionary into a stably-sorted 246 # list of tuples 247 self._maxCountForSlaveList = self.maxCountForSlave.items() 248 self._maxCountForSlaveList.sort() 249 self._maxCountForSlaveList = tuple(self._maxCountForSlaveList)
250