From 095ed3c3b9b6475fcc0ce38fe554328cb911eee8 Mon Sep 17 00:00:00 2001 From: Murali Nandigama Date: Tue, 6 Apr 2010 22:11:58 -0700 Subject: [PATCH] first batch of code --- unittest-logs/.hg_archival.txt | 2 + unittest-logs/.hgignore | 4 + unittest-logs/dbschema.py | 74 +++ unittest-logs/dbschema.pyc | Bin 0 -> 1100 bytes unittest-logs/unittest-log.py | 433 ++++++++++++++++++ unittest-logs/unittestweb/__init__.py | 0 unittest-logs/unittestweb/__init__.pyc | Bin 0 -> 160 bytes unittest-logs/unittestweb/manage.py | 11 + unittest-logs/unittestweb/settings.py | 75 +++ unittest-logs/unittestweb/settings.pyc | Bin 0 -> 1787 bytes unittest-logs/unittestweb/settings.py~ | 75 +++ .../templates/viewer/changeset.html | 6 + .../templates/viewer/changesets.html | 6 + .../unittestweb/templates/viewer/index.html | 8 + .../unittestweb/templates/viewer/test.html | 8 + .../unittestweb/templates/viewer/tests.html | 6 + .../templates/viewer/timeline.html | 72 +++ .../templates/viewer/topfails.html | 7 + .../unittestweb/templates/viewer/tree.html | 6 + .../unittestweb/templates/viewer/trees.html | 7 + unittest-logs/unittestweb/urls.py | 18 + unittest-logs/unittestweb/urls.pyc | Bin 0 -> 983 bytes unittest-logs/unittestweb/urls.py~ | 18 + unittest-logs/unittestweb/viewer/__init__.py | 0 unittest-logs/unittestweb/viewer/__init__.pyc | Bin 0 -> 167 bytes unittest-logs/unittestweb/viewer/models.py | 80 ++++ unittest-logs/unittestweb/viewer/models.pyc | Bin 0 -> 4693 bytes unittest-logs/unittestweb/viewer/models.py~ | 80 ++++ unittest-logs/unittestweb/viewer/views.py | 55 +++ unittest-logs/unittestweb/viewer/views.pyc | Bin 0 -> 3943 bytes unittest-logs/unittestweb/viewer/views.py~ | 55 +++ 31 files changed, 1106 insertions(+) create mode 100644 unittest-logs/.hg_archival.txt create mode 100644 unittest-logs/.hgignore create mode 100755 unittest-logs/dbschema.py create mode 100644 unittest-logs/dbschema.pyc create mode 100755 unittest-logs/unittest-log.py create mode 100755 unittest-logs/unittestweb/__init__.py create mode 100644 unittest-logs/unittestweb/__init__.pyc create mode 100755 unittest-logs/unittestweb/manage.py create mode 100755 unittest-logs/unittestweb/settings.py create mode 100644 unittest-logs/unittestweb/settings.pyc create mode 100755 unittest-logs/unittestweb/settings.py~ create mode 100755 unittest-logs/unittestweb/templates/viewer/changeset.html create mode 100755 unittest-logs/unittestweb/templates/viewer/changesets.html create mode 100755 unittest-logs/unittestweb/templates/viewer/index.html create mode 100755 unittest-logs/unittestweb/templates/viewer/test.html create mode 100755 unittest-logs/unittestweb/templates/viewer/tests.html create mode 100755 unittest-logs/unittestweb/templates/viewer/timeline.html create mode 100755 unittest-logs/unittestweb/templates/viewer/topfails.html create mode 100755 unittest-logs/unittestweb/templates/viewer/tree.html create mode 100755 unittest-logs/unittestweb/templates/viewer/trees.html create mode 100755 unittest-logs/unittestweb/urls.py create mode 100644 unittest-logs/unittestweb/urls.pyc create mode 100755 unittest-logs/unittestweb/urls.py~ create mode 100755 unittest-logs/unittestweb/viewer/__init__.py create mode 100644 unittest-logs/unittestweb/viewer/__init__.pyc create mode 100755 unittest-logs/unittestweb/viewer/models.py create mode 100644 unittest-logs/unittestweb/viewer/models.pyc create mode 100755 unittest-logs/unittestweb/viewer/models.py~ create mode 100755 unittest-logs/unittestweb/viewer/views.py create mode 100644 unittest-logs/unittestweb/viewer/views.pyc create mode 100755 unittest-logs/unittestweb/viewer/views.py~ diff --git a/unittest-logs/.hg_archival.txt b/unittest-logs/.hg_archival.txt new file mode 100644 index 0000000..4cab583 --- /dev/null +++ b/unittest-logs/.hg_archival.txt @@ -0,0 +1,2 @@ +repo: a4d5cb4607eb14581b77cf7b1829b7557a78348f +node: 0b8878aa93a5f872db6255b3369f8ff75d7c063b diff --git a/unittest-logs/.hgignore b/unittest-logs/.hgignore new file mode 100644 index 0000000..eca68f0 --- /dev/null +++ b/unittest-logs/.hgignore @@ -0,0 +1,4 @@ +\.pyc$ + +# SQLite database file. +^unittest\.db$ diff --git a/unittest-logs/dbschema.py b/unittest-logs/dbschema.py new file mode 100755 index 0000000..bc45132 --- /dev/null +++ b/unittest-logs/dbschema.py @@ -0,0 +1,74 @@ + +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is TopFails site code. +# +# The Initial Developer of the Original Code is +# Mozilla foundation +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Serge Gautherie +# Ted Mielczarek . +# Murali Nandigama +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +# +# DB schema maintenance functions. +# + +import logging + +__all__ = \ + [ + "CreateDBSchema" + ] + +def CreateDBSchema(conn): + logging.info("Executing CreateDBSchema()") + + + conn.execute(""" + CREATE TABLE IF NOT EXISTS trees( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name TEXT) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS builds(id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, treeid INT, os INT, starttime INT, status INT, changeset TEXT, logfile TEXT) + """) + conn.execute(""" + CREATE INDEX builds_starttime ON builds (starttime) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS tests (buildid INT, name TEXT, description TEXT) + """) + conn.execute(""" + CREATE INDEX tests_name ON tests (name(1024)) + """) + + diff --git a/unittest-logs/dbschema.pyc b/unittest-logs/dbschema.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f3d84dd4b49ed3fbd933dc11b1cd2a174f16dff3 GIT binary patch literal 1100 zcmbVL!EVz)5MA4$g^F_I(9?+1MsjOFJt9P!8p&$nRJM!KUM$z%IxFmTm0gP>af6>i zd?Uo}mK?&$vz|BOo%d$F`}=x(>&u^CM>&}f8{Z$$Vn8>28lmT zx(|8e&-KX9TV55Cu0yW%&c1h{ky^A1Qe{)6II6mZs-36uNhV4rnKFW1sG9PESO1nT z2(7WK{UBRQZ)-PdFNuY-8w$-~mccEZ1pYlt^8?oXq3|I5b z`cRr7H8rsVgMeyC-@Jai-+j~pb+Dq+#}D_f#ABz-g%B5w2{LmU8P#{dWb#G`ih!6& z?xE*RnwojF$V9=*tj@gnQ|PsVE6u)(Wu;e~OLL$0F2CvvME23-OYK?P&W62dqgji$ o5fvFutt?Hms!CI{a>4z-0@PD1`L1q?wqoyh(U-d8=bdNHU(n6`NB{r; literal 0 HcmV?d00001 diff --git a/unittest-logs/unittest-log.py b/unittest-logs/unittest-log.py new file mode 100755 index 0000000..48b36f0 --- /dev/null +++ b/unittest-logs/unittest-log.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python +#Indentation is 2 spaces ***** DO NOT USE TABS ***** + +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is TopFails site code. +# +# The Initial Developer of the Original Code is +# Mozilla foundation +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Serge Gautherie +# Ted Mielczarek . +# Murali Nandigama +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import re, os, sys, urllib, logging +import MySQLdb # Moved from sqlite3 db to MySQL +from time import ctime, sleep, time +from math import ceil +from optparse import OptionParser +from gzip import GzipFile + +try: + # 'Availability: Unix.' + from time import tzset +except ImportError: + print >>sys.stderr, "WARNING: time.tzset() is not available on non-Unixes!" + + # Define a fake function. (for development use only) + # ToDo: Investigate Windows/etc situation. (Bug 525699) + def tzset(): + pass + +try: + import simplejson as json +except ImportError: + try: + # 'New in version 2.6.' + import json + except ImportError: + print >>sys.stderr, "ERROR: no simplejson nor json package found!" + sys.exit(1) + +from dbschema import CreateDBSchema + +# Number of seconds in a hour: 60 mn * 60 s = 3600 s. +S_IN_H = 3600 +# Download data in 24 hours chunks so as not to overwhelm the tinderbox server. +chunksize = 24 * S_IN_H +# seconds between requests +SLEEP_TIME = 1 + +class OS(): + Windows = 0 + Mac = 1 + Linux = 2 + Unknown = 3 + +class BuildStatus(): + # Unavailable builds to skip. + # Values need (only) to be less than the 'Success' one. + NoBuild = -2 + InProgress = -1 + + # Builds to save in the db. + # Do not change these values (without updating db data). + Success = 0 + TestFailed = 1 + Burning = 2 + Exception = 3 + Unknown = 4 + +csetre = re.compile("rev/([0-9A-Za-z]+)") +def FindChangesetInScrape(scrape): + for line in scrape: + m = csetre.search(line) + if m: + return m.group(1) + return None + +def OSFromBuilderName(name): + if name.startswith("Linux"): + return OS.Linux + if name.startswith("MacOSX") or name.startswith("OS X"): + return OS.Mac + if name.startswith("WINNT"): + return OS.Windows + return OS.Unknown + +buildStatuses = { + # "No build in progress". + "null": BuildStatus.NoBuild, + # "Build in progress". + "building": BuildStatus.InProgress, + # "Successful build". + "success": BuildStatus.Success, + # "Successful build, but tests failed". + "testfailed": BuildStatus.TestFailed, + # "Build failed". + "busted": BuildStatus.Burning, + # "Non-build failure". (i.e. "automation failure") + "exception": BuildStatus.Exception, +} +def BuildStatusFromText(status): + try: + return buildStatuses[status] + except KeyError: + # Log 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does. + logging.info("WARNING: unknown status = '%s'!" % status) + return BuildStatus.Unknown + +def GetOrInsertTree(conn, tree): + """Get an id for a tree named |tree|. If it's not already in the trees + table, insert a new row and return the id.""" + + conn.execute("""SELECT id FROM trees WHERE name = %s""", (tree)) + rows = conn.fetchone() + if len(rows) > 0: + return rows[0] + + # need to insert it + conn.execute("""INSERT INTO trees (name) VALUES (%s)""", (tree,)) + return conn.lastrowid + +def HaveBuild(conn, treeid, os, starttime): + """See if we already have this build in our database.""" + conn.execute("""SELECT COUNT(*) FROM builds WHERE treeid = %s AND os = %s AND starttime = %s""", (treeid, os, starttime)) + return conn.fetchone()[0] == 1 + +def UpdateLogfile(conn, treeid, os, starttime, logfile): + """Update empty 'logfile' for a given build (added in db schema v1).""" + conn.execute("""UPDATE builds SET logfile = %s WHERE treeid = %s AND os = %s AND starttime = %s AND logfile IS NULL""", (logfile, treeid, os, starttime)) + +def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset): + """Insert a build into the builds table and return the id.""" + conn.execute("""INSERT INTO builds (treeid, os, starttime, status, logfile, changeset) VALUES (%s, %s, %s, %s, %s, %s)""", (treeid, os, starttime, status, logfile, changeset)) + return conn.lastrowid + +def InsertTest(conn, buildid, result, name, description): + # ToDo: Add column to save result. + conn.execute("""INSERT INTO tests (buildid, name, description) VALUES (%s, %s, %s)""", (buildid, name, description)) + +def fix_tbox_json(s): # Check :: This seems to be a problem ? Murali + """Fixes up tinderbox json. + + Tinderbox returns strings as single-quoted strings, and occasionally + includes the unquoted substring 'undef' (with quotes) in the output, e.g. + + {'key': 'hello 'undef' world'} + + should return a dictionary + + {'key': 'hello \'undef\' world'} + """ + + json_data = re.sub(r"^tinderbox_data\s*=\s*", "", s) + json_data = re.sub(r";$", "", json_data) + retval = [] + in_str = False + in_esc = False + skip = 0 + for i,c in enumerate(json_data): + if skip > 0: + skip -= 1 + continue + + if in_str: + if in_esc: + if c == "'": + retval.append("'") + else: + retval.append("\\") + retval.append(c) + in_esc = False + elif c == "\\": + in_esc = True + elif c == "\"": + retval.append("\\\"") + elif c == "'": + if json_data[i:i+7] == "'undef'": + retval.append("'undef'") + skip = 7 + else: + retval.append("\"") + in_str = False + else: + retval.append(c) + else: + if c == "'": + retval.append("\"") + in_str = True + else: + retval.append(c) + + return "".join(retval) + +parser = OptionParser() +parser.add_option("-s", "--span", action="store", + dest="timespan", default="15d", + help="Period of time to fetch data for (N[y,m,w,d,h], default=%default)") +parser.add_option("-t", "--tree", action="store", + dest="tree", default="Firefox", + help="Tinderbox tree to fetch data from (default=%default)") +parser.add_option("-d", "--database", action="store", + dest="db", default="topfailsdb", + help="Database filename (default=%default)") +parser.add_option("--host", action="store", + dest="dbhost", default="localhost", + help="Database host name (default=%default)") +parser.add_option( "--port", action="store", + dest="dbport",default="3306", + help="Database port (default=%default)") +parser.add_option("-u", "--user", action="store", + dest="dbuser", default="root", + help="Database username (default=%default)") +parser.add_option("-p", "--passwd", action="store", + dest="dbpasswd", + help="Database user password") +parser.add_option("-v", "--verbose", action="store_true", + dest="verbose", default="False", + help="Enable verbose logging") + +(options, args) = parser.parse_args() + +logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING) + +os.environ['TZ'] = "US/Pacific" +tzset() +# Get current time, in seconds. +endtime = int(time()) + +m = re.match("(\d+)([ymwdh])", options.timespan) +if m is None: + print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan + sys.exit(1) + +timespan = int(m.group(1)) * {'y': 365 * 24 * S_IN_H, + 'm': 30 * 24 * S_IN_H, + 'w': 7 * 24 * S_IN_H, + 'd': 24 * S_IN_H, + 'h': S_IN_H}[m.group(2)] +# Set current time to beginning of requested timespan ending now. +curtime = endtime - timespan + + +createdb=False + + +try: + connection = MySQLdb.connect (host = options.dbhost, + port = int(options.dbport), + db = options.db, + user = options.dbuser, + passwd = options.dbpasswd) + conn=connection.cursor() +except MySQLdb.Error, e: + print "Error %d: %s" % (e.args[0], e.args[1]) + createdb = True + + +if createdb: + connection = MySQLdb.connect (host = options.dbhost, + port = int(options.dbport), + user = options.dbuser, + passwd = options.dbpasswd) + conn = connection.cursor() + try: + createdatabase='create database %s' %(options.db) + conn.execute (createdatabase) + conn.close() + connection.close() + except MySQLdb.Error, e: + print "Error %d: %s" % (e.args[0], e.args[1]) + sys.exit (1) + try: + connection = MySQLdb.connect (host = options.dbhost, + port = int(options.dbport), + db = options.db, + user = options.dbuser, + passwd = options.dbpasswd) + conn=connection.cursor() + except MySQLdb.Error, e: + print "Error %d: %s" % (e.args[0], e.args[1]) + sys.exit(1) + + CreateDBSchema(conn) + + +treeid = GetOrInsertTree(conn, options.tree) +logging.info("Reading tinderbox data...") + +chunk = 0 +# add a fudge factor here, since builds can take up to 3 hours to finish, +# and we can't get the changeset unless we ask for time up to the end of the +# build +endtime += 3 * S_IN_H +timespan += 3 * S_IN_H +totalchunks = int(ceil(float(timespan) / chunksize)) + +while curtime < endtime and chunk < totalchunks: + chunk += 1 + logging.info("Chunk %d/%d" % (chunk, totalchunks)) + + if (endtime - curtime) < chunksize: + chunksize = endtime - curtime + + tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1" \ + % {'tree': options.tree, + 'maxdate': curtime + chunksize, # tbox wants the end time + 'hours': int(chunksize / S_IN_H)} + u = urllib.urlopen(tboxurl) + tboxjson = ''.join(u.readlines()) + u.close() + + tboxjson = fix_tbox_json(tboxjson) + try: + tboxdata = json.loads(tboxjson) + except Exception, inst: + print >>sys.stderr, "Error parsing JSON: %s" % inst + continue + + # we only care about unit test boxes + unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("test|xpc", x)] + # read build table + # 'TestFailed' expected log format is "result | test | optional text". + testfailedRe = re.compile(r"(TEST-UNEXPECTED-.*) \| (.*) \|(.*)") + for timerow in tboxdata['build_table']: + for index in unittest_indices: + if index >= len(timerow) or timerow[index] == -1: + continue + build = timerow[index] + if 'buildname' not in build or \ + 'logfile' not in build: + continue + + status = BuildStatusFromText(build['buildstatus']) + # Skip unavailable "builds". + if status < BuildStatus.Success: + continue + + name = build['buildname'] + os = OSFromBuilderName(name) + starttime = int(build['buildtime']) + # skip builds we've already seen + if HaveBuild(conn, treeid, os, starttime): + logging.info("Skipping already seen build '%s' at %d (%s)" % (name, starttime, ctime(starttime))) + + # Call 'UpdateLogfile()' anyway. + UpdateLogfile(conn, treeid, os, starttime, build['logfile']) + continue + + # must have scrape data for changeset + if build['logfile'] not in tboxdata['scrape']: + continue + changeset = FindChangesetInScrape(tboxdata['scrape'][build['logfile']]) + if changeset is None: + continue + + buildid = InsertBuild(conn, treeid, os, starttime, status, build['logfile'], changeset) + + # 'Success' is fine as is. + if status == BuildStatus.Success: + pass + + # Parse log to save 'TestFailed' results. + elif status == BuildStatus.TestFailed: + logging.info("Checking build log for '%s' at %d (%s)" % (name, starttime, ctime(starttime))) + try: + # Grab the build log. + log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile'])) + gz = GzipFile(log) + # Look for test failures. + for line in gz: + m = testfailedRe.match(line) + if m: + test = m.group(2).strip() or "[unittest-log.py: no logged test]" + text = m.group(3).strip() or "[unittest-log.py: no logged text]" + InsertTest(conn, buildid, m.group(1).rstrip(), test, text) + except: + logging.error("Unexpected error: %s" % sys.exc_info()[0]) + #XXX: handle me? + + # Ignore 'Burning' builds: tests may have run nontheless, but it's safer to discard them :-| + elif status == BuildStatus.Burning: + continue + + # Ignore 'Exception' builds: should only be worse than 'Burning'. + # (Don't know much at time of writing, since this feature is not active yet: see bug 476656 and follow-ups.) + elif status == BuildStatus.Exception: + continue + + # Save 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does. + elif status == BuildStatus.Unknown: + # Add a fake test failure. + InsertTest(conn, buildid, "TEST-UNEXPECTED-FAIL", "unittest-log.py", "Unknown status = '%s'!" % build['buildstatus']) + continue + + + + if chunk < totalchunks: + sleep(SLEEP_TIME) + curtime += chunksize + +conn.close() + +logging.info("Done") diff --git a/unittest-logs/unittestweb/__init__.py b/unittest-logs/unittestweb/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/unittest-logs/unittestweb/__init__.pyc b/unittest-logs/unittestweb/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dcd7f904ef760c1da15495e4050e62bb7170f23e GIT binary patch literal 160 zcmcckiI*$U`;S*L0~9avGgK6h521NDA4v#8QRVUCU(LkL)&3G5wS=;__AX=1@y<`ti;CDmyIPi z-9AKLsE^QJeT6FP3HWVbZ4hl>9pFpAy1;SW1Kt6)47>|$rID+RTm#+%b_Mt{u&cmVfUP&y zt~I;Y8@T~|71$>5HDEV@-vM?L_+4PPfNuc%1o$Sf+rU2sb_e(^2>YMuxHeF@b(Li4 z7hY2Ebd=mb7GxBpP&PC<*mk__qXVllXe|dJ}cA+S z@e~^b^qKv#;e_-spi%sszEZl?=7n*5V(AiSs%u-8Ltl^pgK#h#8mg-ZYezAfKlHt4 zO)7J;Wg}D987mqQ$DvkC27|DV2wVvH9jfMOnF}+Yztv{=qi4<Changeset {{ changeset }} +
    +{% for build in builds %} +
  • {{ build.tree }}: {{ build.get_os_display }}: {{ build.get_status_display }}
    • +{% endfor %} +
diff --git a/unittest-logs/unittestweb/templates/viewer/changesets.html b/unittest-logs/unittestweb/templates/viewer/changesets.html new file mode 100755 index 0000000..51e542a --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/changesets.html @@ -0,0 +1,6 @@ +

Changesets

+
    +{% for c in changesets %} +
  • {{ c }}
    • +{% endfor %} +
diff --git a/unittest-logs/unittestweb/templates/viewer/index.html b/unittest-logs/unittestweb/templates/viewer/index.html new file mode 100755 index 0000000..4ef5f0a --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/index.html @@ -0,0 +1,8 @@ +

Most recent test failures

+
    +{% for f in failures %} +
  • {{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree.name }} {{ f.build.get_os_display }}: {{ f.name }}, + timeline + - {{ f.description }}
    • +{% endfor %} +
diff --git a/unittest-logs/unittestweb/templates/viewer/test.html b/unittest-logs/unittestweb/templates/viewer/test.html new file mode 100755 index 0000000..de4954d --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/test.html @@ -0,0 +1,8 @@ +

Test results for {{ test }}

+
    +
    • +{% for f in failures %}{% ifchanged f.build.id %}
  • +{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree }} {{ f.build.get_os_display }} [{{ f.build.changeset_link|safe }}]: +
      {% endifchanged %} +
    • {{ f.description }}
      • {% endfor %} +
    diff --git a/unittest-logs/unittestweb/templates/viewer/tests.html b/unittest-logs/unittestweb/templates/viewer/tests.html new file mode 100755 index 0000000..92807f4 --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/tests.html @@ -0,0 +1,6 @@ +

    All known failing tests

    +
      +{% for t in tests %} +
    • {{ t }}
      • +{% endfor %} +
    diff --git a/unittest-logs/unittestweb/templates/viewer/timeline.html b/unittest-logs/unittestweb/templates/viewer/timeline.html new file mode 100755 index 0000000..8363867 --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/timeline.html @@ -0,0 +1,72 @@ + + +Timeline for {{ test }} + + + + + + + +

    Timeline for {{ test }}

    + +
    + 
    
+
    
+
    
+
+
    
+
+  
+    
+    
+    
+  
+
    
+{% for desc in descriptions %}
+
    {{desc}}
    
+{% endfor %}
+        		
+      
    
+        		
+      
    
+    
    
+ 
diff --git a/unittest-logs/unittestweb/templates/viewer/topfails.html b/unittest-logs/unittestweb/templates/viewer/topfails.html
new file mode 100755
index 0000000..9f4af33
--- /dev/null
+++ b/unittest-logs/unittestweb/templates/viewer/topfails.html
@@ -0,0 +1,7 @@
+
    Top 25 failing tests
    
+
+
+{% for f in failures %}
+  
+{% endfor %}
+
    CountTest name
    {{ f.0 }}{{ f.1 }}
    
diff --git a/unittest-logs/unittestweb/templates/viewer/tree.html b/unittest-logs/unittestweb/templates/viewer/tree.html
new file mode 100755
index 0000000..c915113
--- /dev/null
+++ b/unittest-logs/unittestweb/templates/viewer/tree.html
@@ -0,0 +1,6 @@
+
    {{ tree }}
    
+
      
+{% for build in newestbuilds %}
+  
    • {{ build.get_os_display }}: {{ build.get_status_display }}
      • 
+{% endfor %}
+
    
diff --git a/unittest-logs/unittestweb/templates/viewer/trees.html b/unittest-logs/unittestweb/templates/viewer/trees.html
new file mode 100755
index 0000000..f68e635
--- /dev/null
+++ b/unittest-logs/unittestweb/templates/viewer/trees.html
@@ -0,0 +1,7 @@
+
    Trees
    
+
+
diff --git a/unittest-logs/unittestweb/urls.py b/unittest-logs/unittestweb/urls.py
new file mode 100755
index 0000000..8e651cd
--- /dev/null
+++ b/unittest-logs/unittestweb/urls.py
@@ -0,0 +1,18 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+urlpatterns = patterns('unittestweb.viewer.views',
+    (r'^$', 'index'),
+    (r'^/$', 'index'),
+    (r'^trees$', 'trees'),
+    (r'^trees/(?P.+)$', 'tree'),
+    (r'^changesets$', 'changesets'),
+    (r'^changesets/(?P[a-f0-9]+)$', 'changeset'),
+    (r'^tests$', 'tests'),
+    (r'^test$', 'test'),
+    (r'^timeline$', 'timeline'),
+    (r'^topfails$', 'topfails'),
+)
diff --git a/unittest-logs/unittestweb/urls.pyc b/unittest-logs/unittestweb/urls.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4d543c69463e1d3ecd2f36fbfe249b88020bd56c
GIT binary patch
literal 983
zcma)4OK;Oa5FXooK+2;BB--*usbn6pxgu483n!!sxrjp%mA08py2OvjvuWYn3xAKl
z!XJQ{T|0@3knm%6X1>{PXTRN_zi#$kPJg@~LG#?^@h$(9u|fUbpx$AC}Zatn^v0q??5{FG_UZ+_x{JuBw9
zch}ifSeTM_*qfKeT21yspC;!tE}zP-j_8#P!p8!MADk;9tkFcvKuL1-a}&$g;uNw0{M
zNr^B1mq^r#x@{os?a`|2u@jzFzOjuWeRCTOHKa<}`B`RQ9aqwn9Ur>Ri|E1ZIlwW?hoqVw8E>
zIOY(RhTd_Qo>hfTi&B+nCaO^B(Wk?-z1@Snn$RNRyPhUVCv;8DzPnWqiVEks9!g)d
Q$EFu;2fc73i2i#20H)&X;{X5v

literal 0
HcmV?d00001

diff --git a/unittest-logs/unittestweb/urls.py~ b/unittest-logs/unittestweb/urls.py~
new file mode 100755
index 0000000..d4303d0
--- /dev/null
+++ b/unittest-logs/unittestweb/urls.py~
@@ -0,0 +1,18 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('unittestweb.viewer.views',
+    (r'^$', 'index'),
+    (r'^/$', 'index'),
+    (r'^trees$', 'trees'),
+    (r'^trees/(?P.+)$', 'tree'),
+    (r'^changesets$', 'changesets'),
+    (r'^changesets/(?P[a-f0-9]+)$', 'changeset'),
+    (r'^tests$', 'tests'),
+    (r'^test$', 'test'),
+    (r'^timeline$', 'timeline'),
+    (r'^topfails$', 'topfails'),
+)
diff --git a/unittest-logs/unittestweb/viewer/__init__.py b/unittest-logs/unittestweb/viewer/__init__.py
new file mode 100755
index 0000000..e69de29
diff --git a/unittest-logs/unittestweb/viewer/__init__.pyc b/unittest-logs/unittestweb/viewer/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d95167dbe1dbdcdf9670098d69a83fd8401665ba
GIT binary patch
literal 167
zcmcckiI*$U`;S*L0~9a%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+    def tinderbox_link(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    class Meta:
+        db_table = 'builds'
+
+class Tests(models.Model):
+    id = models.IntegerField(primary_key=True)
+    build = models.ForeignKey(Builds, db_column="buildid")
+    name = models.TextField(blank=True)
+    description = models.TextField(blank=True)
+    class Meta:
+        db_table = 'tests'
+
+def get_most_failing_tests():
+    cursor = connection.cursor()
+    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) group by name order by count(*) desc limit 250")
+    for row in cursor:
+        yield row
diff --git a/unittest-logs/unittestweb/viewer/models.pyc b/unittest-logs/unittestweb/viewer/models.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..20d7cb41a37c4b29114c2322e31f7275cf20095c
GIT binary patch
literal 4693
zcmc&&>uwWA6rQznac&Tj5JF3(6>U;O!ET{d)k0{w3Iqgc96+^HS(}})Gi1G+?u60)zd)9
z{Ebhw1j&t|>SjtO)=ff;vozJQGg(>%kirqJDstMHu6Fdmqs9<(ns5WfhNv`lz)4Y;
z2z&qxumLGhZjK7ssE%NgK@ozk;)AqWI8`s9<~SxiHL1#W
ztfMC$MzI^k_XCsmc6s!1x|L=-X%(HKsO%b=_06oS>)kX;mFYy?sNQ(8R_~?8Ic?p=
zB1p$3^c%CeezAZuIihq#Z|x
zyuBOiZBjNLxj>Gk!+?({_5q3|VF2G)APFo>6&*J8r!kMW^$JowIH_G=~TY8T--$=?$}YF(h?k&TJK(QzJX
z7rjYA0(UzQ1~Wef!~p+QyfX6
zCEPTsd#TeMo!>D!QGDn1yytzEQOx#rz3X^X&e0MJRM8GcaFOHK<{V1jZOjF8!9*JQ!W>)#=~o7Wn2hQ}?5mqF01Fxf%svd|Mue*__K5FFM(b~cMrb866!nbSOT
zhI*$h&CI}8vBuI~lE@R(IWRM#Fq;}L1XByx;ONqbWi)HebCf1J@)+$r>r%9`
zF6wUc){t^XT|8gTV^X2=L!fQ3DMik2uwCC0hqoew;#-}9;Fajzh_BmavDn~ity%Jx
zSEFz<*X`A_%eJ2D=k;ZK_S&+&T92;TWejgRxBYFszS*gDv!6|pM71pMAZD?AG1jTe
zqr@9FIsJGuN;}$WcT{kYml!>}Ixu;9rY7<~w|^0@-D2@BEj?HYliDev)X6
zM_0PhE~$ETS*>hlz1*%|=5&Yojn&%-zRYi$@14(C>~xg)4^d$JSFyXLU?+mfU@2fP
z7*mrzWK`^e{w+GtgC2y}!h<9_7pBG1A=nom_d2#Z5EjUjzZMqc67^BSE6I#Fue_7x
z+H}(UdQXy5mY)s=9yMAw?>@SB^LB&t+{{YSkt1($IfK2vK(ULc#t>glqAa2KSTj+Y81ZQo9Agn~$Puxw+YsJ6
z+`#Q4i1=*FhnxL9hzLmkk$;U+n0QHVSkn|2I9(-oExXLAbaZf<;fene?X9edwG>8IzO
zL~{YVh7$VADAcf*Q8{+M(CZ8Dulcb*c}PxUV)*(jXA0|G
z9rv8(8_F+f0)drhJB2?y1Xt+D+B@3eW!bt`8*h1dwQDgV4=}=d6v1>b6O6+ttQBXe
z5~F-#_6+GjR#O|ErgS$+YT{Sjx4qfQ&uEmhaC%jeoaYVS)#OF=F{dBWuL|DD63-=(
yOMZ=N-zm@%hXG3k>t6LgAY7x9+7Q)ZFk6}*p9UP$!CZO1;w6$#S5D8LuKWY;zw6Qf

literal 0
HcmV?d00001

diff --git a/unittest-logs/unittestweb/viewer/models.py~ b/unittest-logs/unittestweb/viewer/models.py~
new file mode 100755
index 0000000..2595938
--- /dev/null
+++ b/unittest-logs/unittestweb/viewer/models.py~
@@ -0,0 +1,80 @@
+# This is an auto-generated Django model module.
+# You'll have to do the following manually to clean this up:
+#     * Rearrange models' order
+#     * Make sure each model has one field with primary_key=True
+# Feel free to rename the models, but don't rename db_table values or field names.
+#
+# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
+# into your database.
+
+from django.db import models, connection
+from datetime import datetime
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+class Trees(models.Model):
+    id = models.IntegerField(primary_key=True)
+    name = models.TextField(blank=True)
+    def __unicode__(self):
+        return self.name
+    class Meta:
+        db_table = u'trees'
+
+class Builds(models.Model):
+    id = models.IntegerField(primary_key=True)
+    tree = models.ForeignKey(Trees, db_column="treeid")
+    os = models.IntegerField(choices=OS_CHOICES)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    logfile = models.TextField(blank=True)
+    changeset = models.TextField(blank=True)
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    def changeset_link(self):
+      return '%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+    def tinderbox_link(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    class Meta:
+        db_table = u'builds'
+
+class Tests(models.Model):
+    ROWID = models.IntegerField(primary_key=True)
+    build = models.ForeignKey(Builds, db_column="buildid")
+    name = models.TextField(blank=True)
+    description = models.TextField(blank=True)
+    class Meta:
+        db_table = u'tests'
+
+def get_most_failing_tests():
+    cursor = connection.cursor()
+    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) group by name order by count(*) desc limit 250")
+    for row in cursor:
+        yield row
diff --git a/unittest-logs/unittestweb/viewer/views.py b/unittest-logs/unittestweb/viewer/views.py
new file mode 100755
index 0000000..a1738a6
--- /dev/null
+++ b/unittest-logs/unittestweb/viewer/views.py
@@ -0,0 +1,55 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from unittestweb.viewer.models import Builds, Trees, Tests, OS_CHOICES, get_most_failing_tests
+
+def index(request):
+  failures = get_list_or_404(Tests.objects.all().order_by('-build__starttime')[:10])
+  return render_to_response('viewer/index.html', {'failures': failures})
+
+def trees(request):
+  alltrees = Trees.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees})
+
+def tree(request, tree):
+  newestbuilds = get_list_or_404(Builds.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request):
+  build_csets = Builds.objects.values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', {'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset):
+  builds = get_list_or_404(Builds, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds})
+
+def tests(request):
+    test_names = Tests.objects.values('name').distinct()
+    return render_to_response('viewer/tests.html', {'tests': [t['name'] for t in test_names]})
+
+def test(request):
+  failures = get_list_or_404(Tests.objects.filter(name__exact=request.GET['name']).order_by('-build__starttime'))
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': failures})
+
+def topfails(request):
+  failures = get_most_failing_tests()
+  return render_to_response('viewer/topfails.html', {'failures': failures})
+  
+def timeline(request):
+  name = request.GET['name']
+  builds = get_list_or_404(Builds, tests__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.tests_set.filter(name__exact=name).order_by('id')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist})
diff --git a/unittest-logs/unittestweb/viewer/views.pyc b/unittest-logs/unittestweb/viewer/views.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..acd3677a02d17d873f3f719af70d80204c061504
GIT binary patch
literal 3943
zcmcgv*^V1U6usTvX7g-7l0hsXAY?(t11RzXLI^R507W=bO$3rr)ar5fjC;~{JKg0h
zl82Y!5BM8+;}3Y{Gxz~G=T?sw!Xtz2F5g;Qb?elrd#-=~v$@v%`1gBN3ACOO8&yD<-@YODDrp|
z#d%<}Adic2HY#Gfh33=P25C~*Aj^Y0AKhuyIKGbJ%Sn<(g=PI=9>=AH@4~L3?ZHuS
z`1ONt4(}b=Yp8OXVTNf=!X!;br-9|jzK1S)zzGlW6de>6A5Rirl8Riw&noe|@QBw<
ze4TrORe{UElPOxO(z)=lCwkH?c2Mpg)3+cfY?xb{4C90zY1h*veir8k3HJKDJ+Q;n
zuAqxfO|b12{V;UO@Ca=KMRt4^_pzx8>@aoXvz$;Jzc6SM2O)>%@skO*WoWmwi$(D*
zii1Hmj1PvRa1L?wc^d}MOcYH*St+3;hJe+-#IRCg$f>&M5gCQUco8wnGj@n61z1(c4S_L4
z(X6r{)N@?SI;x5>vC|)+!J+ft17kKZH(e<)x#9K6E_yg)7=c-27`O!%U5H1X`O343
zXdM`dm)$MDWAMj}V#Aais{qh84(`mq|Rk
z*!$>qBrfUedBD;FGhAs$SAa*lE=5PnX=l(}vX$%o6G}zWyvk*Gb4pSw*EBpTp6HO)
zoPQi`-e8+{u(*ON&Qi{}GmC5TEN9H3SXp((2{8HPG}xCtMoE=k$Z)B|byAF!n+-yK
z(xEOd2TIa8wmSyt}
z7AOBpct89XMdk?r-V4z*1}J%#66}|{!X0!AGW=34vN7`$M>V&7-IL}j80HcXjapZq
zd63n%bm%7Ra;vLus|?XZ`H(%k;1*e->F*vxKRz0vsJ9xhpaBT{EUWVTH!8s0N59B<
zO{O*GDpX|Xf>USkuBxO1AX@>L{p&T=kk`0;SQh*Q!rZxoACSLxT|L>5J>=1QNTEA)tjXg`Hd!K>z8*Dka+vwa
zC-OZ~=4ev}>U6Y);n=}8zMyM6<@r*eyGnmeyEPA9${x&3i~vwrBPw`5*Wd=0ii>`p
zjBS#QRE$p2&}t&6vDs-EhS(S582Mf~12S&fJ^ckU;U!|Bj$GWOB1&FOLlzb)TsWs{
zm|HA3nW1ZzN9}xvJR06Ty>q(IS(c2{XQ#9J|3Nz*=95_UBpi?Ncf&A`FbqBB15U>|
zHEW7O8*42ovXd+yhNV1nfFnACu{x*26axNB(Hb|PYnsB>pg?H22@wU8euYvcj_qI=~+aga?ucI^MJXyOKWV!86
zY@t%3Ct)cYmwzJgx1sGKMthh=aa!m(x$InZM{(Amp|CSmb}1wEU!~qKIw=v%yR<$<
fF$YMk|6b4)rO)8