From fc0249eefb81546a3dbf1cb8ac7a8a6753dc3511 Mon Sep 17 00:00:00 2001 From: Murali Nandigama Date: Tue, 11 May 2010 10:59:27 -0700 Subject: [PATCH] don't deploy yet...mods are underwaygit add topfails --- topfails/dbschema.pyc | Bin 0 -> 2031 bytes topfails/topfails/__init__.pyc | Bin 0 -> 169 bytes topfails/topfails/settings.pyc | Bin 0 -> 2144 bytes topfails/topfails/urls.pyc | Bin 0 -> 1116 bytes topfails/topfails/viewer/__init__.pyc | Bin 0 -> 176 bytes topfails/topfails/viewer/models.pyc | Bin 0 -> 6399 bytes topfails/topfails/viewer/views.pyc | Bin 0 -> 6493 bytes topfails/unittest-log.py | 65 ++++++++++++++++++++++---- 8 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 topfails/dbschema.pyc create mode 100644 topfails/topfails/__init__.pyc create mode 100644 topfails/topfails/settings.pyc create mode 100644 topfails/topfails/urls.pyc create mode 100644 topfails/topfails/viewer/__init__.pyc create mode 100644 topfails/topfails/viewer/models.pyc create mode 100644 topfails/topfails/viewer/views.pyc diff --git a/topfails/dbschema.pyc b/topfails/dbschema.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b914be4e4d076987f9593629b13e8f7706158ddf GIT binary patch literal 2031 zcmb_dUvJYe5Kr1}-O9!XKs;DHrHIiLI>aPEWNn73wJAzFiA_RLnz&g+{!FnuhJ-)} zBqUz>SUmB@=im##*^L{QEfr!!J=^CyUw?PMi}U^GX6>c_PqsQBr(WIaU>~tbBr<5;G083b+No;4$fSShiE_P~6!Yx+getTotyMu&8Yi@)ZUO zCm@Wfd{h(ZeO>K(1CGuCp#585k_ZD>Yg~M*4nAhoilqKW^xm%3G-zT#?v75u81Ka zbL0Z5fX6blrOf`c6Y8Xt29QIRyEq62IxUV1>Bc@Ln9g&5xSi)Nf`o}b3BtTS_qQb{ zxj-Lg9O=E0VJ%WLs}K0$0B~ginNU1FaL-|lq(0*yytH3 zy5kFAgl7SoSu+bb`U2HmvtzsvZ8?i0UfzMZu%Gudl#L9EqevCD1u~QelgvmHcK#yt z|1PHJl7f(#nMKsf5Rq14fGfd=Ynt*eP&`TaCg!&{*4RgfTihg$HBb~jkjJoUoFox$ zM6nZlf$v04gC?h}eaO}%O`MTs{J@e>ABD1@9$|<|&T-TqsqRZH)vsP!bC`G*U)+(h%DyzlnFvr`zV9$ojjRUvFM~(|dfr z`IWhIakhE!=F8(J+j6`!-ba1AN~v% zJ6GDhJd*`KwgeM`=b5nP=+Ok5BhKYGB_<XdW#k0LkJN0YPPv7_jCp6vy)jzr&eJUb5C?TMYq^eAk}ySS+H ziziqobPv4mH79bLq1%mLxj#ttEtnOw)wUf~0!8&b$7#8*?GS@#VMX0YszXINiqU+t z9USOXW@N`hrrIB@XxDDFTqm|WoybH4t_E>{)5d9@vPL@l&NQZn#nwrYr(2R|c(fnq z7c=}y5V&(Z>kDf**0p$%=5#b*-|F{V*djqm8Ee^WsEI&cf{g)s1nS8vJJoz;1S48z%t+#;5NsZ>oC~?^epiTvI|(Tq*9P7V9k zPEXF~9xIp)awj5|n%iK_t6p|kZ-*Vj8z7*i(YX}yJ42bUBj0$yH4*He3=BnGPkQ9f49pxi&LCLvd;_E SM-;qL-SOPAQ*s(>K7Rq#HVr5M literal 0 HcmV?d00001 diff --git a/topfails/topfails/viewer/__init__.pyc b/topfails/topfails/viewer/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4d680a4b41545b2aa65fe410afcf521ec011463 GIT binary patch literal 176 zcmcckiI*!j{f2ik0~9ajz74F$xb^(Dv2qAEs1S>&W#S+9;LTp9IGKy4IY$2x?#T9bd8g`o97PB*C zdR9ORs+_W4Qz=2TQwUn=#5Qoq<3R%Te)5f#bA7rvsMHh~B%oH6LRdn%K$86CtOJ+9e-vD>*zwxtHFM-UfD7tB6 zVwaQotu#qYE4OJLy_l*>>0sm-2LHE3~n@ zzLllj>p9jn*=D*|v=4~I#R2`)nq8AEd0sB=>}#HypCb>ef&bMbsAzIppJ6eEA}6)l zvvJhO!@TF7XWwNEcs2VXK(k6FQamqWq9u3hWR#YIhZ4>-daagmUi$0C<@a%Q6G_)< zFH3CF&Uxm0do2hWxR4_cSSZw~@^KuypU3COwLTxKfj-MNvW};o_s7%|=1RhrK+#Z%rH>vAgF@fQ_X0ZYRv7SI=OAo<%XZ z%IO1V3j2KrpA$_Crh?g$h(uKFd06}zL;ou*Oih3S-vOYEC^M?QfGIoE#|`|57_=kY zjP=_CWyWP9d<7=LS6~<%Qh~6^W9`6S(GKrX6Xd~l3=08YF(Sk?y%IrrfuIy?LNyW1 zZFN!$trn>yy*Snv(4(hO*iTtp0~iZXQpN&GDo>8yK%iOT)%xfxqf2vOPIaVVq`h}} z*xeCPgUWJo5`?+Fh+-h>Q$;PIDZA56-C)IIwzwX3O0M@85V#)IF5umu@6IX)83N|C}-BgCsX?*^{&+B%M}av;7^~mp45C%P&4N}yUuE$<7PK(~ z7a`PX`U(9_?8l8AIaEb7FcnOIVTcN#z9Z$4(UGzbV&K(U7$QH6;8yXu|3}QI$usYS z>3_v17j}bgQ)EfmdA;fTUUmuOUN_n#Vta_)Ul9gA11>N0uO-~0?|;C<9t-#S=>fBY zUiN}q;1CmD9rQ8cLx|@izN#`KQXQ3e4KD%DWD1D*@IoNT0%sxILv@H@8mkjh^-mC} zPBL&4T^%6fj|DOq#_%iRm)od#WWZix;{!x7DnQ5rBZ}3T$&{J*SOk{R!d?WE$Zarw zNX<7SlU~bd%*Fg6b%yElS+!V!9N<5y68f#Qm*m3F>q1&u5;?Kh#B}os-U^(&A;381 zCKK{$827NwFvf;SlEN4mpD}RByIfNG(hPZZb93LnL(p%uVkZ`6U=~i(BIHDA+vIpg zz@?e-BiC(`1McVO@t7~Qo)g1ObpFp`@Co{w{vj68%PfAxy$D&)VvmRS*h3Ghxtpf- z5f=5yUpbfSWyQ^ zi4bHAt8@WT^APj!3UCah*-bU`Wj#Rz}V%U=2zygEVwIgL1Pi- zbWvKS!FCcMB-o+UzWo+TA-k5gK*(JwBPl!b5|&S+??`EjWUJY1=k~ozSnu;2{|xNk zfK{vha7EcaVX*!D<_LjmjFddvLy&2TxceXT&2%=77I(Sm^ zT6!|z(%WtE`+bT}P|v_=f`L!$p850ysYw>X?sq^5FCX&d+J~m(uW(fkeC%XsSEtD!k=rf*7JZvHb>BrX ehSYE}c(ZiAHWu)As(gOfQ=ltzwYhK4)&2uc&RH}7 literal 0 HcmV?d00001 diff --git a/topfails/topfails/viewer/views.pyc b/topfails/topfails/viewer/views.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c32d1e6a637346e1139983781829af0740632d10 GIT binary patch literal 6493 zcmcgw+in}z72U(TDC+8~fHtvPqoJExPFurh(V$7|#BoqSMn*;hFmVY=GZIG{a_E^+ zBmq+T-~j!C{y;yYkNtz9Fa3_b^s!xQozaj|?ZQA1k(|SGn{(NFt-bd-fB)A)`!9d| z=kKvJ~Sw*RZ&0y{5%2 z>$P0Js_oBNZ%!lCY`<;sy!GbGwvP2W7St_h*f*GBLHVZgi^{i@Us8Tf`DNwX%C9It zul%a=9p$emzo7h8oF%8E- znD+CepN82a8E0X>gzDWe@5fP=_mi~$;JpXkI{P#E?Lia=c@4Q|X&AapnC1CAYmj+7 zh~k4Z%-bkG-RwVl^7PY3A8&4SJ(Tkfk|%jS@z18BRvt%bxRV@hb4d+6tb2qf`v;O7 zKhL6yMOCWo#Iu*?MKvnwrN>R(!ry7dPO!1MMGXsTY)FpH+cm45R_&x>A3gufPHT2j zwPD>(Y8EvuYEe25S(vk^O+}p*<}rl@&9dB4x#h~x?``1<76g(K|oLa!M~wc6XvU%;%8b9uCI4A+)O3cA%X+ zJjyew{;McF4AcAZfP<|M^HE&(fd(^8$fXH3J{9N-2Am?<+6%YyET_USS01K%H%SSm zt>e6k`cFQ7rsWI=S${t~cH?1OWwdw>r^RcWVh0pY9ubvW$x$4Q_a!Rsyxa?GeZth| zT6|(n;0z8%lgvlSx?GrMPs0}nnAE3a{bkltko_-y`W(Q*;`QPlh0;aQlw`k0?*1?t zh4)9}!8nL^2cyCLJeg2_*}a`K8Qm+FSk_N7>+92eA!Yt$>yu;c)QOyH&$xE;y51Ep z^sZJ!0?Lld7YUfWDG9I@z`p<<(GB1M*92adlWq7!#Q(wAYhWRSz zh&su0`Tko>N|9fVWQiu?=7`A1sAUSa6e*h}(w`#(kw&sBFOiXHadrgYfN3d2Sal+- z6(ZyS8~#nC5@DfOG9+{CVFwkRtXT9V4T?#W7{u{wB_#r(gGRU%iTQW1E-J1N*3GG~ zz9s$uJmL@b?;QTbWbV&dyM8}B8o-K>1NWxZj&jS$)ENm&z!IyR$QYI@%bneu!6aSC zn_!aCy&MUX0z%9=IC6(AJc8pNkSXCf^>F~Dd>ot|FbWq(cEf|gtw6fK81|{yRwhMI_mEB!zHGo9Zy^L`;QD8=Qd8%T{LWt3gcD%AEv= z6{7KW?o5D?Uz&iakLqv35oP|{ctOT@>vEKE7xR4&PXY5g-ZoiDjc}n-!UJTW1gxsi zfaf80$N^wj0|XJMr!{~~A$j8ZoM>~RwG`MHtgoT+qLdwusRT*TA=sG_DMn@N z&|L&CHDg-0qrVYsA3gu8oi^+Q;KPv}cEW)YmI7u$$LP~0?E|6d zPUK>6}jdIFG|&V0;*s6I|Q0@dvEq zl#h6lD_O8LoYmKOP-k%w+puVfX|P~7f<>qhBft2KeGX@j+m@jg9W0|e?Ar?du#r{# z!FXLM^i3eR1A~jb9-=IdMC6kNIA))Vgds`H8e|Z%wG}o6VYZz{lRUz?p9%itIDJ$2^YLB>U#@f2k+FPs`d1=BNgWQr~vdv$t~1>}gnAi2sM z&wIKgn%xEhJDqGa@m>;*wR{&L>Ly7I&(ecXfzMzv!NrsRLypIQ!vBDS>Uhdqj{tWP zDn(hclcXbT8tv947H2F0y}MSdzVh>Qw<>X0n%KXGUOMHo%3W(naGY1YqRSx|HAaA1-BP8Z6c(=Un?#EQ9e$kQkcMAVfTFSDA26)wa8X`mgyO%HtK^_aSVoO=~Z+8}s?3?Bap zT$^ogmhCOI?_Ox1E8E*@|KvjZeA(Vn`#Trf7s~cUwKvaJ0Q^Ng7-Z zb9lKtBcf+0W_OT~JgfM+F!j3E_AWe?Mzt$z$%O}d5Jj?{4U;tAKEOGYBPj066|J8| zF-|Azqa+C9%-KEe)~Ee-QNJwKmnb?L?ZzGQVq5Z)mHGjD%f4}??&!vYs7_Uiw0(u4 zzhK=jnLK5(g=8vJ<-W_?NZO?LvD1S%{56r5-9mC5*Jd49piZUJ>eM^6;@d*0HCOFi F`!D(3Ns#~m literal 0 HcmV?d00001 diff --git a/topfails/unittest-log.py b/topfails/unittest-log.py index 714efba..9585968 100755 --- a/topfails/unittest-log.py +++ b/topfails/unittest-log.py @@ -184,6 +184,7 @@ def UpdateLogfile(conn, treeid, os, starttime, logfile): UPDATE viewer_build SET logfile = %s WHERE tree_id = %s AND os = %s AND starttime = %s AND logfile IS NULL """, (logfile, treeid, os, starttime)) connection.commit() + def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset): """Insert a build into the builds table and return the id.""" conn.execute(""" @@ -192,6 +193,13 @@ def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset): connection.commit() return conn.lastrowid +def HaveFailRecord(conn,buildid, result, testnames_id): + """See if we already have this failRecord in our database.""" + conn.execute(""" + SELECT COUNT(*) FROM viewer_testfailure WHERE build_id = %s AND test_id = %s + """, (buildid, testnames_id)) + return conn.fetchone()[0] == 1 + def InsertTest(conn, buildid, result, testnames_id, description): # ToDo: Add column to save result. conn.execute(""" @@ -290,7 +298,12 @@ parser.add_option("-p", "--passwd", action="store", parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default="False", help="Enable verbose logging") - +parser.add_option("--debug", action='store_true', + dest='debug', default=False, + help="enable interactive debugger on exceptions (pdb)") +parser.add_option("--die", action='store_true', + dest='die', default=False, + help="enable application to die on error") (options, args) = parser.parse_args() logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING) @@ -377,7 +390,7 @@ while curtime < endtime and chunk < totalchunks: if (endtime - curtime) < chunksize: chunksize = endtime - curtime - tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1" \ + tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1&noignore=1" \ % {'tree': options.tree, 'maxdate': curtime + chunksize, # tbox wants the end time 'hours': int(chunksize / S_IN_H)} @@ -395,7 +408,7 @@ while curtime < endtime and chunk < totalchunks: continue # we only care about unit test boxes - unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("test|xpc|check", x)] + unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("ref|mochi|test|xpc|check", x)] # read build table # 'TestFailed' expected log format is "result | test | optional text". testfailedRe = re.compile(r"(TEST-UNEXPECTED-.*) \| (.*) \|(.*)") @@ -443,15 +456,40 @@ while curtime < endtime and chunk < totalchunks: try: # Grab the build log. log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile'])) - gz = GzipFile(log) + gz = GzipFile(log).readlines() # I need a list of lines from the build log # Look for test failures. for line in gz: + # Check all lines if they have INFO Running or url=file:/// stuff. + # If it is mochitest, we see the former string pattern + # If it is jsreftest,crashtest we see the later string pattern. + if "INFO Running" in line or "[url = file:///" in line or "INFO | Loading" in line: + potentialTestName=line + if "[url = file:///" in line: + if "?test=" in line: + potentialTestName = potentialTestName.split('?test=')[1][0:-2] + else: + potentialTestName = potentialTestName.split('url = ')[1][0:-1] + elif "INFO Running" in line: + potentialTestName = potentialTestName.split('INFO Running ')[1][0:-4] + elif "INFO | Loading" in line: + potentialTestName = potentialTestName.split('INFO | Loading ')[1] + else : + potentialTestName= "[unittest-log.py: no logged test]" + m = testfailedRe.match(line) if m: test = rawtest = m.group(2).strip() or "[unittest-log.py: no logged test]" + if 'automation.py' in test or 'automationutils.processLeakLog' in test: + if potentialTestName != "": + #import pdb; pdb.set_trace() + test = rawtest = potentialTestName + potentialTestName="" + + # Code bits below try to change back slash to forward slash + # and get rid of varibale prepends to the /test/../.. names + if rawtest.find('\\') != -1: test = rawtest.replace('\\','/') - if test.find('/') != -1: tup=test.partition('build/') if len(tup[2]) > 2: @@ -461,10 +499,19 @@ while curtime < endtime and chunk < totalchunks: text = m.group(3).strip() or "[unittest-log.py: no logged text]" testnames_id=GetOrInsertTest(conn,test) - InsertTest(conn, buildid, m.group(1).rstrip(), testnames_id, text) - except: - logging.error("Unexpected error: %s" % sys.exc_info()[0]) - #XXX: handle me? + if HaveFailRecord(conn,buildid, m.group(1).rstrip(), testnames_id): + logging.info("Skipping already recorded failure '%s' in build with id '%s' with failure record '%s' " % (test, buildid, text)) + else: + InsertTest(conn, buildid, m.group(1).rstrip(), testnames_id, text) + except Exception, e: + errstring = "Unexpected error: %s" % e + if options.debug: + print errstring + import pdb; pdb.set_trace() + elif options.die: + raise + else: + logging.error(errstring) # Ignore 'Burning' builds: tests may have run nontheless, but it's safer to discard them :-| elif status == BuildStatus.Burning: