Package buildbot :: Module ec2buildslave
[frames] | no frames]

Source Code for Module buildbot.ec2buildslave

  1  """A LatentSlave that uses EC2 to instantiate the slaves on demand. 
  2   
  3  Tested with Python boto 1.5c 
  4  """ 
  5   
  6  # Portions copyright Canonical Ltd. 2009 
  7   
  8  import os 
  9  import re 
 10  import time 
 11   
 12  import boto 
 13  import boto.exception 
 14  from twisted.internet import defer, threads 
 15  from twisted.python import log 
 16   
 17  from buildbot.buildslave import AbstractLatentBuildSlave 
 18  from buildbot import interfaces 
 19   
 20  PENDING = 'pending' 
 21  RUNNING = 'running' 
 22  SHUTTINGDOWN = 'shutting-down' 
 23  TERMINATED = 'terminated' 
 24   
25 -class EC2LatentBuildSlave(AbstractLatentBuildSlave):
26 27 instance = image = None 28 _poll_resolution = 5 # hook point for tests 29
30 - def __init__(self, name, password, instance_type, ami=None, 31 valid_ami_owners=None, valid_ami_location_regex=None, 32 elastic_ip=None, identifier=None, secret_identifier=None, 33 aws_id_file_path=None, user_data=None, 34 keypair_name='latent_buildbot_slave', 35 security_name='latent_buildbot_slave', 36 max_builds=None, notify_on_missing=[], missing_timeout=60*20, 37 build_wait_timeout=60*10, properties={}):
38 AbstractLatentBuildSlave.__init__( 39 self, name, password, max_builds, notify_on_missing, 40 missing_timeout, build_wait_timeout, properties) 41 if not ((ami is not None) ^ 42 (valid_ami_owners is not None or 43 valid_ami_location_regex is not None)): 44 raise ValueError( 45 'You must provide either a specific ami, or one or both of ' 46 'valid_ami_location_regex and valid_ami_owners') 47 self.ami = ami 48 if valid_ami_owners is not None: 49 if isinstance(valid_ami_owners, (int, long)): 50 valid_ami_owners = (valid_ami_owners,) 51 else: 52 for element in valid_ami_owners: 53 if not isinstance(element, (int, long)): 54 raise ValueError( 55 'valid_ami_owners should be int or iterable ' 56 'of ints', element) 57 if valid_ami_location_regex is not None: 58 if not isinstance(valid_ami_location_regex, basestring): 59 raise ValueError( 60 'valid_ami_location_regex should be a string') 61 else: 62 # verify that regex will compile 63 re.compile(valid_ami_location_regex) 64 self.valid_ami_owners = valid_ami_owners 65 self.valid_ami_location_regex = valid_ami_location_regex 66 self.instance_type = instance_type 67 self.keypair_name = keypair_name 68 self.security_name = security_name 69 self.user_data = user_data 70 if identifier is None: 71 assert secret_identifier is None, ( 72 'supply both or neither of identifier, secret_identifier') 73 if aws_id_file_path is None: 74 home = os.environ['HOME'] 75 aws_id_file_path = os.path.join(home, '.ec2', 'aws_id') 76 if not os.path.exists(aws_id_file_path): 77 raise ValueError( 78 "Please supply your AWS access key identifier and secret " 79 "access key identifier either when instantiating this %s " 80 "or in the %s file (on two lines).\n" % 81 (self.__class__.__name__, aws_id_file_path)) 82 aws_file = open(aws_id_file_path, 'r') 83 try: 84 identifier = aws_file.readline().strip() 85 secret_identifier = aws_file.readline().strip() 86 finally: 87 aws_file.close() 88 else: 89 assert aws_id_file_path is None, \ 90 'if you supply the identifier and secret_identifier, ' \ 91 'do not specify the aws_id_file_path' 92 assert secret_identifier is not None, \ 93 'supply both or neither of identifier, secret_identifier' 94 # Make the EC2 connection. 95 self.conn = boto.connect_ec2(identifier, secret_identifier) 96 97 # Make a keypair 98 # 99 # We currently discard the keypair data because we don't need it. 100 # If we do need it in the future, we will always recreate the keypairs 101 # because there is no way to 102 # programmatically retrieve the private key component, unless we 103 # generate it and store it on the filesystem, which is an unnecessary 104 # usage requirement. 105 try: 106 key_pair = self.conn.get_all_key_pairs(keypair_name)[0] 107 # key_pair.delete() # would be used to recreate 108 except boto.exception.EC2ResponseError, e: 109 if e.code != 'InvalidKeyPair.NotFound': 110 if e.code == 'AuthFailure': 111 print ('POSSIBLE CAUSES OF ERROR:\n' 112 ' Did you sign up for EC2?\n' 113 ' Did you put a credit card number in your AWS ' 114 'account?\n' 115 'Please doublecheck before reporting a problem.\n') 116 raise 117 # make one; we would always do this, and stash the result, if we 118 # needed the key (for instance, to SSH to the box). We'd then 119 # use paramiko to use the key to connect. 120 self.conn.create_key_pair(keypair_name) 121 122 # create security group 123 try: 124 group = self.conn.get_all_security_groups(security_name)[0] 125 except boto.exception.EC2ResponseError, e: 126 if e.code == 'InvalidGroup.NotFound': 127 self.security_group = self.conn.create_security_group( 128 security_name, 129 'Authorization to access the buildbot instance.') 130 # Authorize the master as necessary 131 # TODO this is where we'd open the hole to do the reverse pb 132 # connect to the buildbot 133 # ip = urllib.urlopen( 134 # 'http://checkip.amazonaws.com').read().strip() 135 # self.security_group.authorize('tcp', 22, 22, '%s/32' % ip) 136 # self.security_group.authorize('tcp', 80, 80, '%s/32' % ip) 137 else: 138 raise 139 140 # get the image 141 if self.ami is not None: 142 self.image = self.conn.get_image(self.ami) 143 else: 144 # verify we have access to at least one acceptable image 145 discard = self.get_image() 146 147 # get the specified elastic IP, if any 148 if elastic_ip is not None: 149 elastic_ip = self.conn.get_all_addresses([elastic_ip])[0] 150 self.elastic_ip = elastic_ip
151
152 - def get_image(self):
153 if self.image is not None: 154 return self.image 155 if self.valid_ami_location_regex: 156 level = 0 157 options = [] 158 get_match = re.compile(self.valid_ami_location_regex).match 159 for image in self.conn.get_all_images( 160 owners=self.valid_ami_owners): 161 # gather sorting data 162 match = get_match(image.location) 163 if match: 164 alpha_sort = int_sort = None 165 if level < 2: 166 try: 167 alpha_sort = match.group(1) 168 except IndexError: 169 level = 2 170 else: 171 if level == 0: 172 try: 173 int_sort = int(alpha_sort) 174 except ValueError: 175 level = 1 176 options.append([int_sort, alpha_sort, 177 image.location, image.id, image]) 178 if level: 179 log.msg('sorting images at level %d' % level) 180 options = [candidate[level:] for candidate in options] 181 else: 182 options = [(image.location, image.id, image) for image 183 in self.conn.get_all_images( 184 owners=self.valid_ami_owners)] 185 options.sort() 186 log.msg('sorted images (last is chosen): %s' % 187 (', '.join( 188 ['%s (%s)' % (candidate[-1].id, candidate[-1].location) 189 for candidate in options]))) 190 if not options: 191 raise ValueError('no available images match constraints') 192 return options[-1][-1]
193
194 - def dns(self):
195 if self.instance is None: 196 return None 197 return self.instance.public_dns_name
198 dns = property(dns) 199
200 - def start_instance(self):
201 if self.instance is not None: 202 raise ValueError('instance active') 203 return threads.deferToThread(self._start_instance)
204
205 - def _start_instance(self):
206 image = self.get_image() 207 reservation = image.run( 208 key_name=self.keypair_name, security_groups=[self.security_name], 209 instance_type=self.instance_type, user_data=self.user_data) 210 self.instance = reservation.instances[0] 211 log.msg('%s %s starting instance %s' % 212 (self.__class__.__name__, self.slavename, self.instance.id)) 213 duration = 0 214 interval = self._poll_resolution 215 while self.instance.state == PENDING: 216 time.sleep(interval) 217 duration += interval 218 if duration % 60 == 0: 219 log.msg('%s %s has waited %d minutes for instance %s' % 220 (self.__class__.__name__, self.slavename, duration//60, 221 self.instance.id)) 222 self.instance.update() 223 if self.instance.state == RUNNING: 224 self.output = self.instance.get_console_output() 225 minutes = duration//60 226 seconds = duration%60 227 log.msg('%s %s instance %s started on %s ' 228 'in about %d minutes %d seconds (%s)' % 229 (self.__class__.__name__, self.slavename, 230 self.instance.id, self.dns, minutes, seconds, 231 self.output.output)) 232 if self.elastic_ip is not None: 233 self.instance.use_ip(self.elastic_ip) 234 return [self.instance.id, 235 image.id, 236 '%02d:%02d:%02d' % (minutes//60, minutes%60, seconds)] 237 else: 238 log.msg('%s %s failed to start instance %s (%s)' % 239 (self.__class__.__name__, self.slavename, 240 self.instance.id, self.instance.state)) 241 raise interfaces.LatentBuildSlaveFailedToSubstantiate( 242 self.instance.id, self.instance.state)
243
244 - def stop_instance(self, fast=False):
245 if self.instance is None: 246 # be gentle. Something may just be trying to alert us that an 247 # instance never attached, and it's because, somehow, we never 248 # started. 249 return defer.succeed(None) 250 instance = self.instance 251 self.output = self.instance = None 252 return threads.deferToThread( 253 self._stop_instance, instance, fast)
254
255 - def _stop_instance(self, instance, fast):
256 if self.elastic_ip is not None: 257 self.conn.disassociate_address(self.elastic_ip.public_ip) 258 instance.update() 259 if instance.state not in (SHUTTINGDOWN, TERMINATED): 260 instance.stop() 261 log.msg('%s %s terminating instance %s' % 262 (self.__class__.__name__, self.slavename, instance.id)) 263 duration = 0 264 interval = self._poll_resolution 265 if fast: 266 goal = (SHUTTINGDOWN, TERMINATED) 267 instance.update() 268 else: 269 goal = (TERMINATED,) 270 while instance.state not in goal: 271 time.sleep(interval) 272 duration += interval 273 if duration % 60 == 0: 274 log.msg( 275 '%s %s has waited %d minutes for instance %s to end' % 276 (self.__class__.__name__, self.slavename, duration//60, 277 instance.id)) 278 instance.update() 279 log.msg('%s %s instance %s %s ' 280 'after about %d minutes %d seconds' % 281 (self.__class__.__name__, self.slavename, 282 instance.id, goal, duration//60, duration%60))
283