diff --git a/topfails/dbschema.pyc b/topfails/dbschema.pyc new file mode 100644 index 0000000..b914be4 Binary files /dev/null and b/topfails/dbschema.pyc differ diff --git a/topfails/topfails/__init__.pyc b/topfails/topfails/__init__.pyc new file mode 100644 index 0000000..eb83dc1 Binary files /dev/null and b/topfails/topfails/__init__.pyc differ diff --git a/topfails/topfails/settings.pyc b/topfails/topfails/settings.pyc new file mode 100644 index 0000000..ab3e43e Binary files /dev/null and b/topfails/topfails/settings.pyc differ diff --git a/topfails/topfails/urls.pyc b/topfails/topfails/urls.pyc new file mode 100644 index 0000000..84c5017 Binary files /dev/null and b/topfails/topfails/urls.pyc differ diff --git a/topfails/topfails/viewer/__init__.pyc b/topfails/topfails/viewer/__init__.pyc new file mode 100644 index 0000000..b4d680a Binary files /dev/null and b/topfails/topfails/viewer/__init__.pyc differ diff --git a/topfails/topfails/viewer/models.pyc b/topfails/topfails/viewer/models.pyc new file mode 100644 index 0000000..ad0b9c1 Binary files /dev/null and b/topfails/topfails/viewer/models.pyc differ diff --git a/topfails/topfails/viewer/views.pyc b/topfails/topfails/viewer/views.pyc new file mode 100644 index 0000000..c32d1e6 Binary files /dev/null and b/topfails/topfails/viewer/views.pyc differ diff --git a/topfails/unittest-log.py b/topfails/unittest-log.py index 714efba..9585968 100755 --- a/topfails/unittest-log.py +++ b/topfails/unittest-log.py @@ -184,6 +184,7 @@ def UpdateLogfile(conn, treeid, os, starttime, logfile): UPDATE viewer_build SET logfile = %s WHERE tree_id = %s AND os = %s AND starttime = %s AND logfile IS NULL """, (logfile, treeid, os, starttime)) connection.commit() + def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset): """Insert a build into the builds table and return the id.""" conn.execute(""" @@ -192,6 +193,13 @@ def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset): connection.commit() return conn.lastrowid +def HaveFailRecord(conn,buildid, result, testnames_id): + """See if we already have this failRecord in our database.""" + conn.execute(""" + SELECT COUNT(*) FROM viewer_testfailure WHERE build_id = %s AND test_id = %s + """, (buildid, testnames_id)) + return conn.fetchone()[0] == 1 + def InsertTest(conn, buildid, result, testnames_id, description): # ToDo: Add column to save result. conn.execute(""" @@ -290,7 +298,12 @@ parser.add_option("-p", "--passwd", action="store", parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default="False", help="Enable verbose logging") - +parser.add_option("--debug", action='store_true', + dest='debug', default=False, + help="enable interactive debugger on exceptions (pdb)") +parser.add_option("--die", action='store_true', + dest='die', default=False, + help="enable application to die on error") (options, args) = parser.parse_args() logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING) @@ -377,7 +390,7 @@ while curtime < endtime and chunk < totalchunks: if (endtime - curtime) < chunksize: chunksize = endtime - curtime - tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1" \ + tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1&noignore=1" \ % {'tree': options.tree, 'maxdate': curtime + chunksize, # tbox wants the end time 'hours': int(chunksize / S_IN_H)} @@ -395,7 +408,7 @@ while curtime < endtime and chunk < totalchunks: continue # we only care about unit test boxes - unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("test|xpc|check", x)] + unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("ref|mochi|test|xpc|check", x)] # read build table # 'TestFailed' expected log format is "result | test | optional text". testfailedRe = re.compile(r"(TEST-UNEXPECTED-.*) \| (.*) \|(.*)") @@ -443,15 +456,40 @@ while curtime < endtime and chunk < totalchunks: try: # Grab the build log. log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile'])) - gz = GzipFile(log) + gz = GzipFile(log).readlines() # I need a list of lines from the build log # Look for test failures. for line in gz: + # Check all lines if they have INFO Running or url=file:/// stuff. + # If it is mochitest, we see the former string pattern + # If it is jsreftest,crashtest we see the later string pattern. + if "INFO Running" in line or "[url = file:///" in line or "INFO | Loading" in line: + potentialTestName=line + if "[url = file:///" in line: + if "?test=" in line: + potentialTestName = potentialTestName.split('?test=')[1][0:-2] + else: + potentialTestName = potentialTestName.split('url = ')[1][0:-1] + elif "INFO Running" in line: + potentialTestName = potentialTestName.split('INFO Running ')[1][0:-4] + elif "INFO | Loading" in line: + potentialTestName = potentialTestName.split('INFO | Loading ')[1] + else : + potentialTestName= "[unittest-log.py: no logged test]" + m = testfailedRe.match(line) if m: test = rawtest = m.group(2).strip() or "[unittest-log.py: no logged test]" + if 'automation.py' in test or 'automationutils.processLeakLog' in test: + if potentialTestName != "": + #import pdb; pdb.set_trace() + test = rawtest = potentialTestName + potentialTestName="" + + # Code bits below try to change back slash to forward slash + # and get rid of varibale prepends to the /test/../.. names + if rawtest.find('\\') != -1: test = rawtest.replace('\\','/') - if test.find('/') != -1: tup=test.partition('build/') if len(tup[2]) > 2: @@ -461,10 +499,19 @@ while curtime < endtime and chunk < totalchunks: text = m.group(3).strip() or "[unittest-log.py: no logged text]" testnames_id=GetOrInsertTest(conn,test) - InsertTest(conn, buildid, m.group(1).rstrip(), testnames_id, text) - except: - logging.error("Unexpected error: %s" % sys.exc_info()[0]) - #XXX: handle me? + if HaveFailRecord(conn,buildid, m.group(1).rstrip(), testnames_id): + logging.info("Skipping already recorded failure '%s' in build with id '%s' with failure record '%s' " % (test, buildid, text)) + else: + InsertTest(conn, buildid, m.group(1).rstrip(), testnames_id, text) + except Exception, e: + errstring = "Unexpected error: %s" % e + if options.debug: + print errstring + import pdb; pdb.set_trace() + elif options.die: + raise + else: + logging.error(errstring) # Ignore 'Burning' builds: tests may have run nontheless, but it's safer to discard them :-| elif status == BuildStatus.Burning: