buildbot.ec2buildslave

26 27 instance = image = None 28 _poll_resolution = 5 # hook point for tests 29

30 - def __init__(self, name, password, instance_type, ami=None, 31 valid_ami_owners=None, valid_ami_location_regex=None, 32 elastic_ip=None, identifier=None, secret_identifier=None, 33 aws_id_file_path=None, user_data=None, 34 keypair_name='latent_buildbot_slave', 35 security_name='latent_buildbot_slave', 36 max_builds=None, notify_on_missing=[], missing_timeout=60*20, 37 build_wait_timeout=60*10, properties={}, locks=None):

38 AbstractLatentBuildSlave.__init__( 39 self, name, password, max_builds, notify_on_missing, 40 missing_timeout, build_wait_timeout, properties, locks) 41 if not ((ami is not None) ^ 42 (valid_ami_owners is not None or 43 valid_ami_location_regex is not None)): 44 raise ValueError( 45 'You must provide either a specific ami, or one or both of ' 46 'valid_ami_location_regex and valid_ami_owners') 47 self.ami = ami 48 if valid_ami_owners is not None: 49 if isinstance(valid_ami_owners, (int, long)): 50 valid_ami_owners = (valid_ami_owners,) 51 else: 52 for element in valid_ami_owners: 53 if not isinstance(element, (int, long)): 54 raise ValueError( 55 'valid_ami_owners should be int or iterable ' 56 'of ints', element) 57 if valid_ami_location_regex is not None: 58 if not isinstance(valid_ami_location_regex, basestring): 59 raise ValueError( 60 'valid_ami_location_regex should be a string') 61 else: 62 # verify that regex will compile 63 re.compile(valid_ami_location_regex) 64 self.valid_ami_owners = valid_ami_owners 65 self.valid_ami_location_regex = valid_ami_location_regex 66 self.instance_type = instance_type 67 self.keypair_name = keypair_name 68 self.security_name = security_name 69 self.user_data = user_data 70 if identifier is None: 71 assert secret_identifier is None, ( 72 'supply both or neither of identifier, secret_identifier') 73 if aws_id_file_path is None: 74 home = os.environ['HOME'] 75 aws_id_file_path = os.path.join(home, '.ec2', 'aws_id') 76 if not os.path.exists(aws_id_file_path): 77 raise ValueError( 78 "Please supply your AWS access key identifier and secret " 79 "access key identifier either when instantiating this %s " 80 "or in the %s file (on two lines).\n" % 81 (self.__class__.__name__, aws_id_file_path)) 82 aws_file = open(aws_id_file_path, 'r') 83 try: 84 identifier = aws_file.readline().strip() 85 secret_identifier = aws_file.readline().strip() 86 finally: 87 aws_file.close() 88 else: 89 assert aws_id_file_path is None, \ 90 'if you supply the identifier and secret_identifier, ' \ 91 'do not specify the aws_id_file_path' 92 assert secret_identifier is not None, \ 93 'supply both or neither of identifier, secret_identifier' 94 # Make the EC2 connection. 95 self.conn = boto.connect_ec2(identifier, secret_identifier) 96 97 # Make a keypair 98 # 99 # We currently discard the keypair data because we don't need it. 100 # If we do need it in the future, we will always recreate the keypairs 101 # because there is no way to 102 # programmatically retrieve the private key component, unless we 103 # generate it and store it on the filesystem, which is an unnecessary 104 # usage requirement. 105 try: 106 key_pair = self.conn.get_all_key_pairs(keypair_name)[0] 107 # key_pair.delete() # would be used to recreate 108 except boto.exception.EC2ResponseError, e: 109 if 'InvalidKeyPair.NotFound' not in e.body: 110 if 'AuthFailure' in e.body: 111 print ('POSSIBLE CAUSES OF ERROR:\n' 112 ' Did you sign up for EC2?\n' 113 ' Did you put a credit card number in your AWS ' 114 'account?\n' 115 'Please doublecheck before reporting a problem.\n') 116 raise 117 # make one; we would always do this, and stash the result, if we 118 # needed the key (for instance, to SSH to the box). We'd then 119 # use paramiko to use the key to connect. 120 self.conn.create_key_pair(keypair_name) 121 122 # create security group 123 try: 124 group = self.conn.get_all_security_groups(security_name)[0] 125 except boto.exception.EC2ResponseError, e: 126 if 'InvalidGroup.NotFound' in e.body: 127 self.security_group = self.conn.create_security_group( 128 security_name, 129 'Authorization to access the buildbot instance.') 130 # Authorize the master as necessary 131 # TODO this is where we'd open the hole to do the reverse pb 132 # connect to the buildbot 133 # ip = urllib.urlopen( 134 # 'http://checkip.amazonaws.com').read().strip() 135 # self.security_group.authorize('tcp', 22, 22, '%s/32' % ip) 136 # self.security_group.authorize('tcp', 80, 80, '%s/32' % ip) 137 else: 138 raise 139 140 # get the image 141 if self.ami is not None: 142 self.image = self.conn.get_image(self.ami) 143 else: 144 # verify we have access to at least one acceptable image 145 discard = self.get_image() 146 147 # get the specified elastic IP, if any 148 if elastic_ip is not None: 149 elastic_ip = self.conn.get_all_addresses([elastic_ip])[0] 150 self.elastic_ip = elastic_ip

151

152 - def get_image(self):

153 if self.image is not None: 154 return self.image 155 if self.valid_ami_location_regex: 156 level = 0 157 options = [] 158 get_match = re.compile(self.valid_ami_location_regex).match 159 for image in self.conn.get_all_images( 160 owners=self.valid_ami_owners): 161 # gather sorting data 162 match = get_match(image.location) 163 if match: 164 alpha_sort = int_sort = None 165 if level < 2: 166 try: 167 alpha_sort = match.group(1) 168 except IndexError: 169 level = 2 170 else: 171 if level == 0: 172 try: 173 int_sort = int(alpha_sort) 174 except ValueError: 175 level = 1 176 options.append([int_sort, alpha_sort, 177 image.location, image.id, image]) 178 if level: 179 log.msg('sorting images at level %d' % level) 180 options = [candidate[level:] for candidate in options] 181 else: 182 options = [(image.location, image.id, image) for image 183 in self.conn.get_all_images( 184 owners=self.valid_ami_owners)] 185 options.sort() 186 log.msg('sorted images (last is chosen): %s' % 187 (', '.join( 188 ['%s (%s)' % (candidate[-1].id, candidate[-1].location) 189 for candidate in options]))) 190 if not options: 191 raise ValueError('no available images match constraints') 192 return options[-1][-1]

193

194 - def dns(self):

195 if self.instance is None: 196 return None 197 return self.instance.public_dns_name

198 dns = property(dns) 199

200 - def start_instance(self):

201 if self.instance is not None: 202 raise ValueError('instance active') 203 return threads.deferToThread(self._start_instance)

204

205 - def _start_instance(self):

206 image = self.get_image() 207 reservation = image.run( 208 key_name=self.keypair_name, security_groups=[self.security_name], 209 instance_type=self.instance_type, user_data=self.user_data) 210 self.instance = reservation.instances[0] 211 log.msg('%s %s starting instance %s' % 212 (self.__class__.__name__, self.slavename, self.instance.id)) 213 duration = 0 214 interval = self._poll_resolution 215 while self.instance.state == PENDING: 216 time.sleep(interval) 217 duration += interval 218 if duration % 60 == 0: 219 log.msg('%s %s has waited %d minutes for instance %s' % 220 (self.__class__.__name__, self.slavename, duration//60, 221 self.instance.id)) 222 self.instance.update() 223 if self.instance.state == RUNNING: 224 self.output = self.instance.get_console_output() 225 minutes = duration//60 226 seconds = duration%60 227 log.msg('%s %s instance %s started on %s ' 228 'in about %d minutes %d seconds (%s)' % 229 (self.__class__.__name__, self.slavename, 230 self.instance.id, self.dns, minutes, seconds, 231 self.output.output)) 232 if self.elastic_ip is not None: 233 self.instance.use_ip(self.elastic_ip) 234 return [self.instance.id, 235 image.id, 236 '%02d:%02d:%02d' % (minutes//60, minutes%60, seconds)] 237 else: 238 log.msg('%s %s failed to start instance %s (%s)' % 239 (self.__class__.__name__, self.slavename, 240 self.instance.id, self.instance.state)) 241 raise interfaces.LatentBuildSlaveFailedToSubstantiate( 242 self.instance.id, self.instance.state)

243

244 - def stop_instance(self, fast=False):

245 if self.instance is None: 246 # be gentle. Something may just be trying to alert us that an 247 # instance never attached, and it's because, somehow, we never 248 # started. 249 return defer.succeed(None) 250 instance = self.instance 251 self.output = self.instance = None 252 return threads.deferToThread( 253 self._stop_instance, instance, fast)

254

255 - def _stop_instance(self, instance, fast):

256 if self.elastic_ip is not None: 257 self.conn.disassociate_address(self.elastic_ip.public_ip) 258 instance.update() 259 if instance.state not in (SHUTTINGDOWN, TERMINATED): 260 instance.stop() 261 log.msg('%s %s terminating instance %s' % 262 (self.__class__.__name__, self.slavename, instance.id)) 263 duration = 0 264 interval = self._poll_resolution 265 if fast: 266 goal = (SHUTTINGDOWN, TERMINATED) 267 instance.update() 268 else: 269 goal = (TERMINATED,) 270 while instance.state not in goal: 271 time.sleep(interval) 272 duration += interval 273 if duration % 60 == 0: 274 log.msg( 275 '%s %s has waited %d minutes for instance %s to end' % 276 (self.__class__.__name__, self.slavename, duration//60, 277 instance.id)) 278 instance.update() 279 log.msg('%s %s instance %s %s ' 280 'after about %d minutes %d seconds' % 281 (self.__class__.__name__, self.slavename, 282 instance.id, goal, duration//60, duration%60))

Source Code for Module buildbot.ec2buildslave