1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 import sys, os, re, time, random
18 from twisted.internet import utils, protocol, defer, reactor, task
19 from twisted.spread import pb
20 from twisted.cred import credentials
21 from twisted.python import log
22 from twisted.python.procutils import which
23
24 from buildbot.sourcestamp import SourceStamp
25 from buildbot.util import now
26 from buildbot.status import builder
27
29
34
36 """This accepts the arguments of a command, without the actual
37 command itself."""
38 env = os.environ.copy()
39 env['LC_ALL'] = "C"
40 d = utils.getProcessOutputAndValue(self.exe, cmd, env=env,
41 path=self.treetop)
42 d.addCallback(self._didvc, cmd)
43 return d
45 (stdout, stderr, code) = res
46
47
48 return stdout
49
51 """Return a Deferred that fires with a SourceStamp instance."""
52 d = self.getBaseRevision()
53 d.addCallback(self.getPatch)
54 d.addCallback(self.done)
55 return d
63
65 patchlevel = 0
66 vcexe = "cvs"
68
69
70
71
72 self.baserev = time.strftime("%Y-%m-%d %H:%M:%S +0000",
73 time.gmtime(now()))
74 return defer.succeed(None)
75
77
78 if self.branch is not None:
79
80
81
82
83
84
85 raise RuntimeError("Sorry, CVS 'try' builds don't work with "
86 "branches")
87 args = ['-q', 'diff', '-u', '-D', self.baserev]
88 d = self.dovc(args)
89 d.addCallback(self.readPatch, self.patchlevel)
90 return d
91
93 patchlevel = 0
94 vcexe = "svn"
95
97 d = self.dovc(["status", "-u"])
98 d.addCallback(self.parseStatus)
99 return d
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118 for line in res.split("\n"):
119 m = re.search(r'^Status against revision:\s+(\d+)', line)
120 if m:
121 self.baserev = int(m.group(1))
122 return
123 raise IndexError("Could not find 'Status against revision' in "
124 "SVN output: %s" % res)
129
131 patchlevel = 0
132 vcexe = "bzr"
137
139 revno, revid= out.split()
140 self.baserev = 'revid:' + revid
141 return
142
144 d = self.dovc(["diff","-r%s.." % self.baserev])
145 d.addCallback(self.readPatch, self.patchlevel)
146 return d
147
149 patchlevel = 1
150 vcexe = "hg"
152 d = self.dovc(["identify", "--id", "--debug"])
153 d.addCallback(self.parseStatus)
154 return d
156 m = re.search(r'^(\w+)', output)
157 self.baserev = m.group(0)
162
163
165 patchlevel = 0
166 vcexe = "p4"
168 d = self.dovc(["changes", "-m1", "..."])
169 d.addCallback(self.parseStatus)
170 return d
171
173
174
175
176 m = re.search(r'Change (\d+)',res)
177 if m:
178 self.baserev = m.group(1)
179 return
180
181 raise IndexError("Could not find change number in output: %s" % res)
182
184
185
186
187 assert self.branch, "you must specify a branch"
188 mpatch = ""
189 found = False
190 for line in res.split("\n"):
191 m = re.search('==== //depot/' + self.branch + r'/([\w\/\.\d\-\_]+)#(\d+) -',line)
192 if m:
193 mpatch += "--- %s#%s\n" % (m.group(1), m.group(2) )
194 mpatch += "+++ %s\n" % (m.group(1) )
195 found = True
196 else:
197 mpatch += line
198 mpatch += "\n"
199 assert found, "could not parse patch file"
200 self.patch = (patchlevel, mpatch)
205
206
220
222 patchlevel = 1
223 vcexe = "git"
224
226 d = self.dovc(["branch", "--no-color", "-v", "--no-abbrev"])
227 d.addCallback(self.parseStatus)
228 return d
229
231 d = self.dovc(["config", "-l"])
232 d.addCallback(self.parseConfig)
233 return d
234
236 git_config = {}
237 for l in res.split("\n"):
238 if l.strip():
239 parts = l.strip().split("=", 2)
240 git_config[parts[0]] = parts[1]
241
242
243 remote = git_config.get("branch." + self.branch + ".remote")
244 ref = git_config.get("branch." + self.branch + ".merge")
245 if remote and ref:
246 remote_branch = ref.split("/", 3)[-1]
247 d = self.dovc(["rev-parse", remote + "/" + remote_branch])
248 d.addCallback(self.override_baserev)
249 return d
250
252 self.baserev = res.strip()
253
255
256
257
258
259 m = re.search(r'^\* (\S+)\s+([0-9a-f]{40})', res, re.MULTILINE)
260 if m:
261 self.baserev = m.group(2)
262
263
264
265
266
267 if self.branch:
268 d = self.dovc(["rev-parse", self.branch])
269 if '/' in self.branch:
270 self.branch = self.branch.split('/', 1)[1]
271 d.addCallback(self.override_baserev)
272 return d
273 else:
274 self.branch = m.group(1)
275 return self.readConfig()
276 raise IndexError("Could not find current GIT branch: %s" % res)
277
279 d = self.dovc(["diff", self.baserev])
280 d.addCallback(self.readPatch, self.patchlevel)
281 return d
282
301
302
304 return "%d:%s," % (len(s), s)
305
306 -def createJobfile(bsid, branch, baserev, patchlevel, diff, repository,
307 project, builderNames):
308 job = ""
309 job += ns("2")
310 job += ns(bsid)
311 job += ns(branch)
312 job += ns(str(baserev))
313 job += ns("%d" % patchlevel)
314 job += ns(diff)
315 job += ns(repository)
316 job += ns(project)
317 for bn in builderNames:
318 job += ns(bn)
319 return job
320
322 """walk upwards from the current directory until we find this topfile"""
323 if not start:
324 start = os.getcwd()
325 here = start
326 toomany = 20
327 while toomany > 0:
328 if os.path.exists(os.path.join(here, topfile)):
329 return here
330 next = os.path.dirname(here)
331 if next == here:
332 break
333 here = next
334 toomany -= 1
335 raise ValueError("Unable to find topfile '%s' anywhere from %s upwards"
336 % (topfile, start))
337
340 self.job = job
341 self.d = defer.Deferred()
346 sys.stdout.write(data)
348 sys.stderr.write(data)
350 sig = status_object.value.signal
351 rc = status_object.value.exitCode
352 if sig != None or rc != 0:
353 self.d.errback(RuntimeError("remote 'buildbot tryserver' failed"
354 ": sig=%s, rc=%s" % (sig, rc)))
355 return
356 self.d.callback((sig, rc))
357
359 retryCount = 5
360 retryDelay = 3
361
365
367
368
369 self.d = defer.Deferred()
370
371
372 reactor.callLater(1, self.go)
373 return self.d
374
375 - def go(self, dummy=None):
376 if self.retryCount == 0:
377 raise RuntimeError("couldn't find matching buildset")
378 self.retryCount -= 1
379 d = self.status.callRemote("getBuildSets")
380 d.addCallback(self._gotSets)
381
383 for bs,bsid in buildsets:
384 if bsid == self.bsid:
385
386 self.d.callback(bs)
387 return
388 d = defer.Deferred()
389 d.addCallback(self.go)
390 reactor.callLater(self.retryDelay, d.callback, None)
391
392
393 -class Try(pb.Referenceable):
394 buildsetStatus = None
395 quiet = False
396
403
404 - def getopt(self, config_name, default=None):
405 value = self.config.get(config_name)
406 if value is None or value == []:
407 value = default
408 return value
409
411
412
413
414
415
416
417 self.bsid = "%d-%s" % (time.time(), random.randint(0, 1000000))
418
419
420 branch = self.getopt("branch")
421
422 difffile = self.config.get("diff")
423 if difffile:
424 baserev = self.config.get("baserev")
425 if difffile == "-":
426 diff = sys.stdin.read()
427 else:
428 diff = open(difffile,"r").read()
429 patch = (self.config['patchlevel'], diff)
430 ss = SourceStamp(branch, baserev, patch)
431 d = defer.succeed(ss)
432 else:
433 vc = self.getopt("vc")
434 if vc in ("cvs", "svn"):
435
436 topdir = self.getopt("try-topdir")
437 if topdir:
438 treedir = os.path.expanduser(topdir)
439 else:
440 topfile = self.getopt("try-topfile")
441 treedir = getTopdir(topfile)
442 else:
443 treedir = os.getcwd()
444 d = getSourceStamp(vc, treedir, branch)
445 d.addCallback(self._createJob_1)
446 return d
447
459
461
462 ss = self.sourcestamp
463 print ("Job:\n\tRepository: %s\n\tProject: %s\n\tBranch: %s\n\t"
464 "Revision: %s\n\tBuilders: %s\n%s"
465 % (ss.repository, self.project, ss.branch,
466 ss.revision,
467 self.builderNames,
468 ss.patch[1]))
469 d = defer.Deferred()
470 d.callback(True)
471 return d
472
474
475
476 if self.connect == "ssh":
477 tryhost = self.getopt("tryhost")
478 tryuser = self.getopt("username")
479 trydir = self.getopt("trydir")
480
481 argv = ["ssh", "-l", tryuser, tryhost,
482 "buildbot", "tryserver", "--jobdir", trydir]
483
484
485 pp = RemoteTryPP(self.jobfile)
486 reactor.spawnProcess(pp, argv[0], argv, os.environ)
487 d = pp.d
488 return d
489 if self.connect == "pb":
490 user = self.getopt("username")
491 passwd = self.getopt("passwd")
492 master = self.getopt("master")
493 tryhost, tryport = master.split(":")
494 tryport = int(tryport)
495 f = pb.PBClientFactory()
496 d = f.login(credentials.UsernamePassword(user, passwd))
497 reactor.connectTCP(tryhost, tryport, f)
498 d.addCallback(self._deliverJob_pb)
499 return d
500 raise RuntimeError("unknown connecttype '%s', should be 'ssh' or 'pb'"
501 % self.connect)
502
519
521
522
523 wait = bool(self.getopt("wait", "try_wait"))
524 if not wait:
525
526
527
528
529
530 print "not waiting for builds to finish"
531 return
532 d = self.running = defer.Deferred()
533 if self.buildsetStatus:
534 self._getStatus_1()
535 return self.running
536
537
538 master = self.getopt("master")
539 host, port = master.split(":")
540 port = int(port)
541 self.announce("contacting the status port at %s:%d" % (host, port))
542 f = pb.PBClientFactory()
543 creds = credentials.UsernamePassword("statusClient", "clientpw")
544 d = f.login(creds)
545 reactor.connectTCP(host, port, f)
546 d.addCallback(self._getStatus_ssh_1)
547 return self.running
548
556
558 if res:
559 self.buildsetStatus = res
560
561 d = self.buildsetStatus.callRemote("getBuildRequests")
562 d.addCallback(self._getStatus_2)
563
565 self.builderNames = []
566 self.buildRequests = {}
567
568
569 self.builds = {}
570
571
572
573 self.outstanding = []
574
575
576
577 self.results = {}
578
579
580
581 self.currentStep = {}
582
583
584
585 self.ETA = {}
586
587 for n,br in brs:
588 self.builderNames.append(n)
589 self.buildRequests[n] = br
590 self.builds[n] = None
591 self.outstanding.append(n)
592 self.results[n] = [None,None]
593 self.currentStep[n] = None
594 self.ETA[n] = None
595
596
597 br.callRemote("subscribe", self)
598
599
600
601 self.printloop = task.LoopingCall(self.printStatus)
602 self.printloop.start(3, now=False)
603
604
605
606
608 if self.builds[builderName]:
609 self.builds[builderName].callRemote("unsubscribe", self)
610 self.builds[builderName] = bs
611 bs.callRemote("subscribe", self, 20)
612 d = bs.callRemote("waitUntilFinished")
613 d.addCallback(self._build_finished, builderName)
614
617
620
622 self.ETA[buildername] = now() + eta
623
625
626
627
628 self.builds[builderName] = None
629 self.ETA[builderName] = None
630 self.currentStep[builderName] = "finished"
631 d = bs.callRemote("getResults")
632 d.addCallback(self._build_finished_2, bs, builderName)
633 return d
635 self.results[builderName][0] = results
636 d = bs.callRemote("getText")
637 d.addCallback(self._build_finished_3, builderName)
638 return d
640 self.results[builderName][1] = text
641
642 self.outstanding.remove(builderName)
643 if not self.outstanding:
644
645 return self.statusDone()
646
648 names = self.buildRequests.keys()
649 names.sort()
650 for n in names:
651 if n not in self.outstanding:
652
653 code,text = self.results[n]
654 t = builder.Results[code]
655 if text:
656 t += " (%s)" % " ".join(text)
657 elif self.builds[n]:
658 t = self.currentStep[n] or "building"
659 if self.ETA[n]:
660 t += " [ETA %ds]" % (self.ETA[n] - now())
661 else:
662 t = "no build"
663 self.announce("%s: %s" % (n, t))
664 self.announce("")
665
667 self.printloop.stop()
668 print "All Builds Complete"
669
670 names = self.buildRequests.keys()
671 names.sort()
672 happy = True
673 for n in names:
674 code,text = self.results[n]
675 t = "%s: %s" % (n, builder.Results[code])
676 if text:
677 t += " (%s)" % " ".join(text)
678 print t
679 if code != builder.SUCCESS:
680 happy = False
681
682 if happy:
683 self.exitcode = 0
684 else:
685 self.exitcode = 1
686 self.running.callback(self.exitcode)
687
689
690
691
692 if self.connect == "pb":
693 user = self.getopt("username", "try_username")
694 passwd = self.getopt("passwd", "try_password")
695 master = self.getopt("master", "try_master")
696 tryhost, tryport = master.split(":")
697 tryport = int(tryport)
698 f = pb.PBClientFactory()
699 d = f.login(credentials.UsernamePassword(user, passwd))
700 reactor.connectTCP(tryhost, tryport, f)
701 d.addCallback(self._getBuilderNames, self._getBuilderNames2)
702 return d
703 if self.connect == "ssh":
704 raise RuntimeError("ssh connection type not supported for this command")
705 raise RuntimeError("unknown connecttype '%s', should be 'pb'" % self.connect)
706
708 d = remote.callRemote("getAvailableBuilderNames")
709 d.addCallback(self._getBuilderNames2)
710 return d
711
713 print "The following builders are available for the try scheduler: "
714 for buildername in buildernames:
715 print buildername
716
718 if not self.quiet:
719 print message
720
722
723
724 print "using '%s' connect method" % self.connect
725 self.exitcode = 0
726 d = defer.Deferred()
727 if bool(self.config.get("get-builder-names")):
728 d.addCallback(lambda res: self.getAvailableBuilderNames())
729 else:
730 d.addCallback(lambda res: self.createJob())
731 d.addCallback(lambda res: self.announce("job created"))
732 deliver = self.deliverJob
733 if bool(self.config.get("dryrun")):
734 deliver = self.fakeDeliverJob
735 d.addCallback(lambda res: deliver())
736 d.addCallback(lambda res: self.announce("job has been delivered"))
737 d.addCallback(lambda res: self.getStatus())
738 d.addErrback(log.err)
739 d.addCallback(self.cleanup)
740 d.addCallback(lambda res: reactor.stop())
741
742 reactor.callLater(0, d.callback, None)
743 reactor.run()
744 sys.exit(self.exitcode)
745
747 log.err(why)
748 print "error during 'try' processing"
749 print why
750
754