1
2
3 import sys, os, re, time, random
4 from twisted.internet import utils, protocol, defer, reactor, task
5 from twisted.spread import pb
6 from twisted.cred import credentials
7 from twisted.python import log
8 from twisted.python.procutils import which
9
10 from buildbot.sourcestamp import SourceStamp
11 from buildbot.util import now
12 from buildbot.status import builder
13
15
20
22 """This accepts the arguments of a command, without the actual
23 command itself."""
24 env = os.environ.copy()
25 env['LC_ALL'] = "C"
26 d = utils.getProcessOutputAndValue(self.exe, cmd, env=env,
27 path=self.treetop)
28 d.addCallback(self._didvc, cmd)
29 return d
31 (stdout, stderr, code) = res
32
33
34 return stdout
35
37 """Return a Deferred that fires with a SourceStamp instance."""
38 d = self.getBaseRevision()
39 d.addCallback(self.getPatch)
40 d.addCallback(self.done)
41 return d
49
51 patchlevel = 0
52 vcexe = "cvs"
54
55
56
57
58 self.baserev = time.strftime("%Y-%m-%d %H:%M:%S +0000",
59 time.gmtime(now()))
60 return defer.succeed(None)
61
63
64 if self.branch is not None:
65
66
67
68
69
70
71 raise RuntimeError("Sorry, CVS 'try' builds don't work with "
72 "branches")
73 args = ['-q', 'diff', '-u', '-D', self.baserev]
74 d = self.dovc(args)
75 d.addCallback(self.readPatch, self.patchlevel)
76 return d
77
79 patchlevel = 0
80 vcexe = "svn"
81
83 d = self.dovc(["status", "-u"])
84 d.addCallback(self.parseStatus)
85 return d
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104 for line in res.split("\n"):
105 m = re.search(r'^Status against revision:\s+(\d+)', line)
106 if m:
107 self.baserev = int(m.group(1))
108 return
109 raise IndexError("Could not find 'Status against revision' in "
110 "SVN output: %s" % res)
115
117 patchlevel = 0
118 vcexe = "bzr"
123
125 revno, revid= out.split()
126 self.baserev = 'revid:' + revid
127 return
128
130 d = self.dovc(["diff","-r%s.." % self.baserev])
131 d.addCallback(self.readPatch, self.patchlevel)
132 return d
133
135 patchlevel = 1
136 vcexe = "hg"
138 d = self.dovc(["identify", "--id", "--debug"])
139 d.addCallback(self.parseStatus)
140 return d
142 m = re.search(r'^(\w+)', output)
143 self.baserev = m.group(0)
148
149
151 patchlevel = 0
152 vcexe = "p4"
154 d = self.dovc(["changes", "-m1", "..."])
155 d.addCallback(self.parseStatus)
156 return d
157
159
160
161
162 m = re.search(r'Change (\d+)',res)
163 if m:
164 self.baserev = m.group(1)
165 return
166
167 raise IndexError("Could not find change number in output: %s" % res)
168
170
171
172
173 assert self.branch, "you must specify a branch"
174 mpatch = ""
175 found = False
176 for line in res.split("\n"):
177 m = re.search('==== //depot/' + self.branch + r'/([\w\/\.\d\-\_]+)#(\d+) -',line)
178 if m:
179 mpatch += "--- %s#%s\n" % (m.group(1), m.group(2) )
180 mpatch += "+++ %s\n" % (m.group(1) )
181 found = True
182 else:
183 mpatch += line
184 mpatch += "\n"
185 assert found, "could not parse patch file"
186 self.patch = (patchlevel, mpatch)
191
192
206
208 patchlevel = 1
209 vcexe = "git"
210
212 d = self.dovc(["branch", "--no-color", "-v", "--no-abbrev"])
213 d.addCallback(self.parseStatus)
214 return d
215
217 d = self.dovc(["config", "-l"])
218 d.addCallback(self.parseConfig)
219 return d
220
222 git_config = {}
223 for l in res.split("\n"):
224 if l.strip():
225 parts = l.strip().split("=", 2)
226 git_config[parts[0]] = parts[1]
227
228
229 remote = git_config.get("branch." + self.branch + ".remote")
230 ref = git_config.get("branch." + self.branch + ".merge")
231 if remote and ref:
232 remote_branch = ref.split("/", 3)[-1]
233 d = self.dovc(["rev-parse", remote + "/" + remote_branch])
234 d.addCallback(self.override_baserev)
235 return d
236
238 self.baserev = res.strip()
239
241
242
243
244
245 m = re.search(r'^\* (\S+)\s+([0-9a-f]{40})', res, re.MULTILINE)
246 if m:
247 self.baserev = m.group(2)
248
249
250
251
252
253 if self.branch:
254 d = self.dovc(["rev-parse", self.branch])
255 if '/' in self.branch:
256 self.branch = self.branch.split('/', 1)[1]
257 d.addCallback(self.override_baserev)
258 return d
259 else:
260 self.branch = m.group(1)
261 return self.readConfig()
262 raise IndexError("Could not find current GIT branch: %s" % res)
263
265 d = self.dovc(["diff", self.baserev])
266 d.addCallback(self.readPatch, self.patchlevel)
267 return d
268
287
288
290 return "%d:%s," % (len(s), s)
291
292 -def createJobfile(bsid, branch, baserev, patchlevel, diff, repository,
293 project, builderNames):
294 job = ""
295 job += ns("2")
296 job += ns(bsid)
297 job += ns(branch)
298 job += ns(str(baserev))
299 job += ns("%d" % patchlevel)
300 job += ns(diff)
301 job += ns(repository)
302 job += ns(project)
303 for bn in builderNames:
304 job += ns(bn)
305 return job
306
308 """walk upwards from the current directory until we find this topfile"""
309 if not start:
310 start = os.getcwd()
311 here = start
312 toomany = 20
313 while toomany > 0:
314 if os.path.exists(os.path.join(here, topfile)):
315 return here
316 next = os.path.dirname(here)
317 if next == here:
318 break
319 here = next
320 toomany -= 1
321 raise ValueError("Unable to find topfile '%s' anywhere from %s upwards"
322 % (topfile, start))
323
326 self.job = job
327 self.d = defer.Deferred()
329 self.transport.write(self.job)
330 self.transport.closeStdin()
332 sys.stdout.write(data)
334 sys.stderr.write(data)
336 sig = status_object.value.signal
337 rc = status_object.value.exitCode
338 if sig != None or rc != 0:
339 self.d.errback(RuntimeError("remote 'buildbot tryserver' failed"
340 ": sig=%s, rc=%s" % (sig, rc)))
341 return
342 self.d.callback((sig, rc))
343
345 retryCount = 5
346 retryDelay = 3
347
351
353
354
355 self.d = defer.Deferred()
356
357
358 reactor.callLater(1, self.go)
359 return self.d
360
361 - def go(self, dummy=None):
362 if self.retryCount == 0:
363 raise RuntimeError("couldn't find matching buildset")
364 self.retryCount -= 1
365 d = self.status.callRemote("getBuildSets")
366 d.addCallback(self._gotSets)
367
369 for bs,bsid in buildsets:
370 if bsid == self.bsid:
371
372 self.d.callback(bs)
373 return
374 d = defer.Deferred()
375 d.addCallback(self.go)
376 reactor.callLater(self.retryDelay, d.callback, None)
377
378
379 -class Try(pb.Referenceable):
380 buildsetStatus = None
381 quiet = False
382
389
390 - def getopt(self, config_name, default=None):
391 value = self.config.get(config_name)
392 if value is None or value == []:
393 value = default
394 return value
395
397
398
399
400
401
402
403 self.bsid = "%d-%s" % (time.time(), random.randint(0, 1000000))
404
405
406 branch = self.getopt("branch")
407
408 difffile = self.config.get("diff")
409 if difffile:
410 baserev = self.config.get("baserev")
411 if difffile == "-":
412 diff = sys.stdin.read()
413 else:
414 diff = open(difffile,"r").read()
415 patch = (self.config['patchlevel'], diff)
416 ss = SourceStamp(branch, baserev, patch)
417 d = defer.succeed(ss)
418 else:
419 vc = self.getopt("vc")
420 if vc in ("cvs", "svn"):
421
422 topdir = self.getopt("try-topdir")
423 if topdir:
424 treedir = os.path.expanduser(topdir)
425 else:
426 topfile = self.getopt("try-topfile")
427 treedir = getTopdir(topfile)
428 else:
429 treedir = os.getcwd()
430 d = getSourceStamp(vc, treedir, branch)
431 d.addCallback(self._createJob_1)
432 return d
433
445
447
448 ss = self.sourcestamp
449 print ("Job:\n\tRepository: %s\n\tProject: %s\n\tBranch: %s\n\t"
450 "Revision: %s\n\tBuilders: %s\n%s"
451 % (ss.repository, self.project, ss.branch,
452 ss.revision,
453 self.builderNames,
454 ss.patch[1]))
455 d = defer.Deferred()
456 d.callback(True)
457 return d
458
460
461
462 if self.connect == "ssh":
463 tryhost = self.getopt("tryhost")
464 tryuser = self.getopt("username")
465 trydir = self.getopt("trydir")
466
467 argv = ["ssh", "-l", tryuser, tryhost,
468 "buildbot", "tryserver", "--jobdir", trydir]
469
470
471 pp = RemoteTryPP(self.jobfile)
472 reactor.spawnProcess(pp, argv[0], argv, os.environ)
473 d = pp.d
474 return d
475 if self.connect == "pb":
476 user = self.getopt("username")
477 passwd = self.getopt("passwd")
478 master = self.getopt("master")
479 tryhost, tryport = master.split(":")
480 tryport = int(tryport)
481 f = pb.PBClientFactory()
482 d = f.login(credentials.UsernamePassword(user, passwd))
483 reactor.connectTCP(tryhost, tryport, f)
484 d.addCallback(self._deliverJob_pb)
485 return d
486 raise RuntimeError("unknown connecttype '%s', should be 'ssh' or 'pb'"
487 % self.connect)
488
505
507
508
509 wait = bool(self.getopt("wait", "try_wait"))
510 if not wait:
511
512
513
514
515
516 print "not waiting for builds to finish"
517 return
518 d = self.running = defer.Deferred()
519 if self.buildsetStatus:
520 self._getStatus_1()
521 return self.running
522
523
524 master = self.getopt("master")
525 host, port = master.split(":")
526 port = int(port)
527 self.announce("contacting the status port at %s:%d" % (host, port))
528 f = pb.PBClientFactory()
529 creds = credentials.UsernamePassword("statusClient", "clientpw")
530 d = f.login(creds)
531 reactor.connectTCP(host, port, f)
532 d.addCallback(self._getStatus_ssh_1)
533 return self.running
534
542
544 if res:
545 self.buildsetStatus = res
546
547 d = self.buildsetStatus.callRemote("getBuildRequests")
548 d.addCallback(self._getStatus_2)
549
551 self.builderNames = []
552 self.buildRequests = {}
553
554
555 self.builds = {}
556
557
558
559 self.outstanding = []
560
561
562
563 self.results = {}
564
565
566
567 self.currentStep = {}
568
569
570
571 self.ETA = {}
572
573 for n,br in brs:
574 self.builderNames.append(n)
575 self.buildRequests[n] = br
576 self.builds[n] = None
577 self.outstanding.append(n)
578 self.results[n] = [None,None]
579 self.currentStep[n] = None
580 self.ETA[n] = None
581
582
583 br.callRemote("subscribe", self)
584
585
586
587 self.printloop = task.LoopingCall(self.printStatus)
588 self.printloop.start(3, now=False)
589
590
591
592
594 if self.builds[builderName]:
595 self.builds[builderName].callRemote("unsubscribe", self)
596 self.builds[builderName] = bs
597 bs.callRemote("subscribe", self, 20)
598 d = bs.callRemote("waitUntilFinished")
599 d.addCallback(self._build_finished, builderName)
600
603
606
608 self.ETA[buildername] = now() + eta
609
611
612
613
614 self.builds[builderName] = None
615 self.ETA[builderName] = None
616 self.currentStep[builderName] = "finished"
617 d = bs.callRemote("getResults")
618 d.addCallback(self._build_finished_2, bs, builderName)
619 return d
621 self.results[builderName][0] = results
622 d = bs.callRemote("getText")
623 d.addCallback(self._build_finished_3, builderName)
624 return d
626 self.results[builderName][1] = text
627
628 self.outstanding.remove(builderName)
629 if not self.outstanding:
630
631 return self.statusDone()
632
634 names = self.buildRequests.keys()
635 names.sort()
636 for n in names:
637 if n not in self.outstanding:
638
639 code,text = self.results[n]
640 t = builder.Results[code]
641 if text:
642 t += " (%s)" % " ".join(text)
643 elif self.builds[n]:
644 t = self.currentStep[n] or "building"
645 if self.ETA[n]:
646 t += " [ETA %ds]" % (self.ETA[n] - now())
647 else:
648 t = "no build"
649 self.announce("%s: %s" % (n, t))
650 self.announce("")
651
653 self.printloop.stop()
654 print "All Builds Complete"
655
656 names = self.buildRequests.keys()
657 names.sort()
658 happy = True
659 for n in names:
660 code,text = self.results[n]
661 t = "%s: %s" % (n, builder.Results[code])
662 if text:
663 t += " (%s)" % " ".join(text)
664 print t
665 if code != builder.SUCCESS:
666 happy = False
667
668 if happy:
669 self.exitcode = 0
670 else:
671 self.exitcode = 1
672 self.running.callback(self.exitcode)
673
675
676
677
678 if self.connect == "pb":
679 user = self.getopt("username", "try_username")
680 passwd = self.getopt("passwd", "try_password")
681 master = self.getopt("master", "try_master")
682 tryhost, tryport = master.split(":")
683 tryport = int(tryport)
684 f = pb.PBClientFactory()
685 d = f.login(credentials.UsernamePassword(user, passwd))
686 reactor.connectTCP(tryhost, tryport, f)
687 d.addCallback(self._getBuilderNames, self._getBuilderNames2)
688 return d
689 if self.connect == "ssh":
690 raise RuntimeError("ssh connection type not supported for this command")
691 raise RuntimeError("unknown connecttype '%s', should be 'pb'" % self.connect)
692
694 d = remote.callRemote("getAvailableBuilderNames")
695 d.addCallback(self._getBuilderNames2)
696 return d
697
699 print "The following builders are available for the try scheduler: "
700 for buildername in buildernames:
701 print buildername
702
704 if not self.quiet:
705 print message
706
708
709
710 print "using '%s' connect method" % self.connect
711 self.exitcode = 0
712 d = defer.Deferred()
713 if bool(self.config.get("get-builder-names")):
714 d.addCallback(lambda res: self.getAvailableBuilderNames())
715 else:
716 d.addCallback(lambda res: self.createJob())
717 d.addCallback(lambda res: self.announce("job created"))
718 deliver = self.deliverJob
719 if bool(self.config.get("dryrun")):
720 deliver = self.fakeDeliverJob
721 d.addCallback(lambda res: deliver())
722 d.addCallback(lambda res: self.announce("job has been delivered"))
723 d.addCallback(lambda res: self.getStatus())
724 d.addErrback(log.err)
725 d.addCallback(self.cleanup)
726 d.addCallback(lambda res: reactor.stop())
727
728 reactor.callLater(0, d.callback, None)
729 reactor.run()
730 sys.exit(self.exitcode)
731
733 log.err(why)
734 print "error during 'try' processing"
735 print why
736
740