1 """A LatentSlave that uses EC2 to instantiate the slaves on demand.
2
3 Tested with Python boto 1.5c
4 """
5
6
7
8 import os
9 import re
10 import time
11
12 import boto
13 import boto.exception
14 from twisted.internet import defer, threads
15 from twisted.python import log
16
17 from buildbot.buildslave import AbstractLatentBuildSlave
18 from buildbot import interfaces
19
20 PENDING = 'pending'
21 RUNNING = 'running'
22 SHUTTINGDOWN = 'shutting-down'
23 TERMINATED = 'terminated'
24
26
27 instance = image = None
28 _poll_resolution = 5
29
30 - def __init__(self, name, password, instance_type, ami=None,
31 valid_ami_owners=None, valid_ami_location_regex=None,
32 elastic_ip=None, identifier=None, secret_identifier=None,
33 aws_id_file_path=None, user_data=None,
34 keypair_name='latent_buildbot_slave',
35 security_name='latent_buildbot_slave',
36 max_builds=None, notify_on_missing=[], missing_timeout=60*20,
37 build_wait_timeout=60*10, properties={}):
38 AbstractLatentBuildSlave.__init__(
39 self, name, password, max_builds, notify_on_missing,
40 missing_timeout, build_wait_timeout, properties)
41 if not ((ami is not None) ^
42 (valid_ami_owners is not None or
43 valid_ami_location_regex is not None)):
44 raise ValueError(
45 'You must provide either a specific ami, or one or both of '
46 'valid_ami_location_regex and valid_ami_owners')
47 self.ami = ami
48 if valid_ami_owners is not None:
49 if isinstance(valid_ami_owners, (int, long)):
50 valid_ami_owners = (valid_ami_owners,)
51 else:
52 for element in valid_ami_owners:
53 if not isinstance(element, (int, long)):
54 raise ValueError(
55 'valid_ami_owners should be int or iterable '
56 'of ints', element)
57 if valid_ami_location_regex is not None:
58 if not isinstance(valid_ami_location_regex, basestring):
59 raise ValueError(
60 'valid_ami_location_regex should be a string')
61 else:
62
63 re.compile(valid_ami_location_regex)
64 self.valid_ami_owners = valid_ami_owners
65 self.valid_ami_location_regex = valid_ami_location_regex
66 self.instance_type = instance_type
67 self.keypair_name = keypair_name
68 self.security_name = security_name
69 self.user_data = user_data
70 if identifier is None:
71 assert secret_identifier is None, (
72 'supply both or neither of identifier, secret_identifier')
73 if aws_id_file_path is None:
74 home = os.environ['HOME']
75 aws_id_file_path = os.path.join(home, '.ec2', 'aws_id')
76 if not os.path.exists(aws_id_file_path):
77 raise ValueError(
78 "Please supply your AWS access key identifier and secret "
79 "access key identifier either when instantiating this %s "
80 "or in the %s file (on two lines).\n" %
81 (self.__class__.__name__, aws_id_file_path))
82 aws_file = open(aws_id_file_path, 'r')
83 try:
84 identifier = aws_file.readline().strip()
85 secret_identifier = aws_file.readline().strip()
86 finally:
87 aws_file.close()
88 else:
89 assert aws_id_file_path is None, \
90 'if you supply the identifier and secret_identifier, ' \
91 'do not specify the aws_id_file_path'
92 assert secret_identifier is not None, \
93 'supply both or neither of identifier, secret_identifier'
94
95 self.conn = boto.connect_ec2(identifier, secret_identifier)
96
97
98
99
100
101
102
103
104
105 try:
106 key_pair = self.conn.get_all_key_pairs(keypair_name)[0]
107
108 except boto.exception.EC2ResponseError, e:
109 if e.code != 'InvalidKeyPair.NotFound':
110 if e.code == 'AuthFailure':
111 print ('POSSIBLE CAUSES OF ERROR:\n'
112 ' Did you sign up for EC2?\n'
113 ' Did you put a credit card number in your AWS '
114 'account?\n'
115 'Please doublecheck before reporting a problem.\n')
116 raise
117
118
119
120 self.conn.create_key_pair(keypair_name)
121
122
123 try:
124 group = self.conn.get_all_security_groups(security_name)[0]
125 except boto.exception.EC2ResponseError, e:
126 if e.code == 'InvalidGroup.NotFound':
127 self.security_group = self.conn.create_security_group(
128 security_name,
129 'Authorization to access the buildbot instance.')
130
131
132
133
134
135
136
137 else:
138 raise
139
140
141 if self.ami is not None:
142 self.image = self.conn.get_image(self.ami)
143 else:
144
145 discard = self.get_image()
146
147
148 if elastic_ip is not None:
149 elastic_ip = self.conn.get_all_addresses([elastic_ip])[0]
150 self.elastic_ip = elastic_ip
151
153 if self.image is not None:
154 return self.image
155 if self.valid_ami_location_regex:
156 level = 0
157 options = []
158 get_match = re.compile(self.valid_ami_location_regex).match
159 for image in self.conn.get_all_images(
160 owners=self.valid_ami_owners):
161
162 match = get_match(image.location)
163 if match:
164 alpha_sort = int_sort = None
165 if level < 2:
166 try:
167 alpha_sort = match.group(1)
168 except IndexError:
169 level = 2
170 else:
171 if level == 0:
172 try:
173 int_sort = int(alpha_sort)
174 except ValueError:
175 level = 1
176 options.append([int_sort, alpha_sort,
177 image.location, image.id, image])
178 if level:
179 log.msg('sorting images at level %d' % level)
180 options = [candidate[level:] for candidate in options]
181 else:
182 options = [(image.location, image.id, image) for image
183 in self.conn.get_all_images(
184 owners=self.valid_ami_owners)]
185 options.sort()
186 log.msg('sorted images (last is chosen): %s' %
187 (', '.join(
188 ['%s (%s)' % (candidate[-1].id, candidate[-1].location)
189 for candidate in options])))
190 if not options:
191 raise ValueError('no available images match constraints')
192 return options[-1][-1]
193
195 if self.instance is None:
196 return None
197 return self.instance.public_dns_name
198 dns = property(dns)
199
201 if self.instance is not None:
202 raise ValueError('instance active')
203 return threads.deferToThread(self._start_instance)
204
206 image = self.get_image()
207 reservation = image.run(
208 key_name=self.keypair_name, security_groups=[self.security_name],
209 instance_type=self.instance_type, user_data=self.user_data)
210 self.instance = reservation.instances[0]
211 log.msg('%s %s starting instance %s' %
212 (self.__class__.__name__, self.slavename, self.instance.id))
213 duration = 0
214 interval = self._poll_resolution
215 while self.instance.state == PENDING:
216 time.sleep(interval)
217 duration += interval
218 if duration % 60 == 0:
219 log.msg('%s %s has waited %d minutes for instance %s' %
220 (self.__class__.__name__, self.slavename, duration//60,
221 self.instance.id))
222 self.instance.update()
223 if self.instance.state == RUNNING:
224 self.output = self.instance.get_console_output()
225 minutes = duration//60
226 seconds = duration%60
227 log.msg('%s %s instance %s started on %s '
228 'in about %d minutes %d seconds (%s)' %
229 (self.__class__.__name__, self.slavename,
230 self.instance.id, self.dns, minutes, seconds,
231 self.output.output))
232 if self.elastic_ip is not None:
233 self.instance.use_ip(self.elastic_ip)
234 return [self.instance.id,
235 image.id,
236 '%02d:%02d:%02d' % (minutes//60, minutes%60, seconds)]
237 else:
238 log.msg('%s %s failed to start instance %s (%s)' %
239 (self.__class__.__name__, self.slavename,
240 self.instance.id, self.instance.state))
241 raise interfaces.LatentBuildSlaveFailedToSubstantiate(
242 self.instance.id, self.instance.state)
243
245 if self.instance is None:
246
247
248
249 return defer.succeed(None)
250 instance = self.instance
251 self.output = self.instance = None
252 return threads.deferToThread(
253 self._stop_instance, instance, fast)
254
256 if self.elastic_ip is not None:
257 self.conn.disassociate_address(self.elastic_ip.public_ip)
258 instance.update()
259 if instance.state not in (SHUTTINGDOWN, TERMINATED):
260 instance.stop()
261 log.msg('%s %s terminating instance %s' %
262 (self.__class__.__name__, self.slavename, instance.id))
263 duration = 0
264 interval = self._poll_resolution
265 if fast:
266 goal = (SHUTTINGDOWN, TERMINATED)
267 instance.update()
268 else:
269 goal = (TERMINATED,)
270 while instance.state not in goal:
271 time.sleep(interval)
272 duration += interval
273 if duration % 60 == 0:
274 log.msg(
275 '%s %s has waited %d minutes for instance %s to end' %
276 (self.__class__.__name__, self.slavename, duration//60,
277 instance.id))
278 instance.update()
279 log.msg('%s %s instance %s %s '
280 'after about %d minutes %d seconds' %
281 (self.__class__.__name__, self.slavename,
282 instance.id, goal, duration//60, duration%60))
283