Package buildbot :: Module ec2buildslave
[frames] | no frames]

Source Code for Module buildbot.ec2buildslave

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Portions Copyright Buildbot Team Members 
 15  # Portions Copyright Canonical Ltd. 2009 
 16   
 17  """A LatentSlave that uses EC2 to instantiate the slaves on demand. 
 18   
 19  Tested with Python boto 1.5c 
 20  """ 
 21   
 22  import os 
 23  import re 
 24  import time 
 25   
 26  import boto 
 27  import boto.ec2 
 28  import boto.exception 
 29  from twisted.internet import defer, threads 
 30  from twisted.python import log 
 31   
 32  from buildbot.buildslave import AbstractLatentBuildSlave 
 33  from buildbot import interfaces 
 34   
 35  PENDING = 'pending' 
 36  RUNNING = 'running' 
 37  SHUTTINGDOWN = 'shutting-down' 
 38  TERMINATED = 'terminated' 
 39   
40 -class EC2LatentBuildSlave(AbstractLatentBuildSlave):
41 42 instance = image = None 43 _poll_resolution = 5 # hook point for tests 44
45 - def __init__(self, name, password, instance_type, ami=None, 46 valid_ami_owners=None, valid_ami_location_regex=None, 47 elastic_ip=None, identifier=None, secret_identifier=None, 48 aws_id_file_path=None, user_data=None, region=None, 49 keypair_name='latent_buildbot_slave', 50 security_name='latent_buildbot_slave', 51 max_builds=None, notify_on_missing=[], missing_timeout=60*20, 52 build_wait_timeout=60*10, properties={}, locks=None):
53 54 AbstractLatentBuildSlave.__init__( 55 self, name, password, max_builds, notify_on_missing, 56 missing_timeout, build_wait_timeout, properties, locks) 57 if not ((ami is not None) ^ 58 (valid_ami_owners is not None or 59 valid_ami_location_regex is not None)): 60 raise ValueError( 61 'You must provide either a specific ami, or one or both of ' 62 'valid_ami_location_regex and valid_ami_owners') 63 self.ami = ami 64 if valid_ami_owners is not None: 65 if isinstance(valid_ami_owners, (int, long)): 66 valid_ami_owners = (valid_ami_owners,) 67 else: 68 for element in valid_ami_owners: 69 if not isinstance(element, (int, long)): 70 raise ValueError( 71 'valid_ami_owners should be int or iterable ' 72 'of ints', element) 73 if valid_ami_location_regex is not None: 74 if not isinstance(valid_ami_location_regex, basestring): 75 raise ValueError( 76 'valid_ami_location_regex should be a string') 77 else: 78 # verify that regex will compile 79 re.compile(valid_ami_location_regex) 80 self.valid_ami_owners = valid_ami_owners 81 self.valid_ami_location_regex = valid_ami_location_regex 82 self.instance_type = instance_type 83 self.keypair_name = keypair_name 84 self.security_name = security_name 85 self.user_data = user_data 86 if identifier is None: 87 assert secret_identifier is None, ( 88 'supply both or neither of identifier, secret_identifier') 89 if aws_id_file_path is None: 90 home = os.environ['HOME'] 91 aws_id_file_path = os.path.join(home, '.ec2', 'aws_id') 92 if not os.path.exists(aws_id_file_path): 93 raise ValueError( 94 "Please supply your AWS access key identifier and secret " 95 "access key identifier either when instantiating this %s " 96 "or in the %s file (on two lines).\n" % 97 (self.__class__.__name__, aws_id_file_path)) 98 aws_file = open(aws_id_file_path, 'r') 99 try: 100 identifier = aws_file.readline().strip() 101 secret_identifier = aws_file.readline().strip() 102 finally: 103 aws_file.close() 104 else: 105 assert aws_id_file_path is None, \ 106 'if you supply the identifier and secret_identifier, ' \ 107 'do not specify the aws_id_file_path' 108 assert secret_identifier is not None, \ 109 'supply both or neither of identifier, secret_identifier' 110 111 region_found = None 112 113 # Make the EC2 connection. 114 if region is not None: 115 for r in boto.ec2.regions(aws_access_key_id=identifier, 116 aws_secret_access_key=secret_identifier): 117 118 if r.name == region: 119 region_found = r 120 121 122 if region_found is not None: 123 self.conn = boto.ec2.connect_to_region(region, 124 aws_access_key_id=identifier, 125 aws_secret_access_key=secret_identifier) 126 else: 127 raise ValueError('The specified region does not exist: {0}'.format(region)) 128 129 else: 130 self.conn = boto.connect_ec2(identifier, secret_identifier) 131 132 # Make a keypair 133 # 134 # We currently discard the keypair data because we don't need it. 135 # If we do need it in the future, we will always recreate the keypairs 136 # because there is no way to 137 # programmatically retrieve the private key component, unless we 138 # generate it and store it on the filesystem, which is an unnecessary 139 # usage requirement. 140 try: 141 key_pair = self.conn.get_all_key_pairs(keypair_name)[0] 142 assert key_pair 143 # key_pair.delete() # would be used to recreate 144 except boto.exception.EC2ResponseError, e: 145 if 'InvalidKeyPair.NotFound' not in e.body: 146 if 'AuthFailure' in e.body: 147 print ('POSSIBLE CAUSES OF ERROR:\n' 148 ' Did you sign up for EC2?\n' 149 ' Did you put a credit card number in your AWS ' 150 'account?\n' 151 'Please doublecheck before reporting a problem.\n') 152 raise 153 # make one; we would always do this, and stash the result, if we 154 # needed the key (for instance, to SSH to the box). We'd then 155 # use paramiko to use the key to connect. 156 self.conn.create_key_pair(keypair_name) 157 158 # create security group 159 try: 160 group = self.conn.get_all_security_groups(security_name)[0] 161 assert group 162 except boto.exception.EC2ResponseError, e: 163 if 'InvalidGroup.NotFound' in e.body: 164 self.security_group = self.conn.create_security_group( 165 security_name, 166 'Authorization to access the buildbot instance.') 167 # Authorize the master as necessary 168 # TODO this is where we'd open the hole to do the reverse pb 169 # connect to the buildbot 170 # ip = urllib.urlopen( 171 # 'http://checkip.amazonaws.com').read().strip() 172 # self.security_group.authorize('tcp', 22, 22, '%s/32' % ip) 173 # self.security_group.authorize('tcp', 80, 80, '%s/32' % ip) 174 else: 175 raise 176 177 # get the image 178 if self.ami is not None: 179 self.image = self.conn.get_image(self.ami) 180 else: 181 # verify we have access to at least one acceptable image 182 discard = self.get_image() 183 assert discard 184 185 # get the specified elastic IP, if any 186 if elastic_ip is not None: 187 elastic_ip = self.conn.get_all_addresses([elastic_ip])[0] 188 self.elastic_ip = elastic_ip
189
190 - def get_image(self):
191 if self.image is not None: 192 return self.image 193 if self.valid_ami_location_regex: 194 level = 0 195 options = [] 196 get_match = re.compile(self.valid_ami_location_regex).match 197 for image in self.conn.get_all_images( 198 owners=self.valid_ami_owners): 199 # gather sorting data 200 match = get_match(image.location) 201 if match: 202 alpha_sort = int_sort = None 203 if level < 2: 204 try: 205 alpha_sort = match.group(1) 206 except IndexError: 207 level = 2 208 else: 209 if level == 0: 210 try: 211 int_sort = int(alpha_sort) 212 except ValueError: 213 level = 1 214 options.append([int_sort, alpha_sort, 215 image.location, image.id, image]) 216 if level: 217 log.msg('sorting images at level %d' % level) 218 options = [candidate[level:] for candidate in options] 219 else: 220 options = [(image.location, image.id, image) for image 221 in self.conn.get_all_images( 222 owners=self.valid_ami_owners)] 223 options.sort() 224 log.msg('sorted images (last is chosen): %s' % 225 (', '.join( 226 ['%s (%s)' % (candidate[-1].id, candidate[-1].location) 227 for candidate in options]))) 228 if not options: 229 raise ValueError('no available images match constraints') 230 return options[-1][-1]
231
232 - def dns(self):
233 if self.instance is None: 234 return None 235 return self.instance.public_dns_name
236 dns = property(dns) 237
238 - def start_instance(self, build):
239 if self.instance is not None: 240 raise ValueError('instance active') 241 return threads.deferToThread(self._start_instance)
242
243 - def _start_instance(self):
244 image = self.get_image() 245 reservation = image.run( 246 key_name=self.keypair_name, security_groups=[self.security_name], 247 instance_type=self.instance_type, user_data=self.user_data) 248 self.instance = reservation.instances[0] 249 log.msg('%s %s starting instance %s' % 250 (self.__class__.__name__, self.slavename, self.instance.id)) 251 duration = 0 252 interval = self._poll_resolution 253 while self.instance.state == PENDING: 254 time.sleep(interval) 255 duration += interval 256 if duration % 60 == 0: 257 log.msg('%s %s has waited %d minutes for instance %s' % 258 (self.__class__.__name__, self.slavename, duration//60, 259 self.instance.id)) 260 self.instance.update() 261 if self.instance.state == RUNNING: 262 self.output = self.instance.get_console_output() 263 minutes = duration//60 264 seconds = duration%60 265 log.msg('%s %s instance %s started on %s ' 266 'in about %d minutes %d seconds (%s)' % 267 (self.__class__.__name__, self.slavename, 268 self.instance.id, self.dns, minutes, seconds, 269 self.output.output)) 270 if self.elastic_ip is not None: 271 self.instance.use_ip(self.elastic_ip) 272 return [self.instance.id, 273 image.id, 274 '%02d:%02d:%02d' % (minutes//60, minutes%60, seconds)] 275 else: 276 log.msg('%s %s failed to start instance %s (%s)' % 277 (self.__class__.__name__, self.slavename, 278 self.instance.id, self.instance.state)) 279 raise interfaces.LatentBuildSlaveFailedToSubstantiate( 280 self.instance.id, self.instance.state)
281
282 - def stop_instance(self, fast=False):
283 if self.instance is None: 284 # be gentle. Something may just be trying to alert us that an 285 # instance never attached, and it's because, somehow, we never 286 # started. 287 return defer.succeed(None) 288 instance = self.instance 289 self.output = self.instance = None 290 return threads.deferToThread( 291 self._stop_instance, instance, fast)
292
293 - def _stop_instance(self, instance, fast):
294 if self.elastic_ip is not None: 295 self.conn.disassociate_address(self.elastic_ip.public_ip) 296 instance.update() 297 if instance.state not in (SHUTTINGDOWN, TERMINATED): 298 instance.stop() 299 log.msg('%s %s terminating instance %s' % 300 (self.__class__.__name__, self.slavename, instance.id)) 301 duration = 0 302 interval = self._poll_resolution 303 if fast: 304 goal = (SHUTTINGDOWN, TERMINATED) 305 instance.update() 306 else: 307 goal = (TERMINATED,) 308 while instance.state not in goal: 309 time.sleep(interval) 310 duration += interval 311 if duration % 60 == 0: 312 log.msg( 313 '%s %s has waited %d minutes for instance %s to end' % 314 (self.__class__.__name__, self.slavename, duration//60, 315 instance.id)) 316 instance.update() 317 log.msg('%s %s instance %s %s ' 318 'after about %d minutes %d seconds' % 319 (self.__class__.__name__, self.slavename, 320 instance.id, goal, duration//60, duration%60))
321