diff --git a/unittest-logs/.hg_archival.txt b/unittest-logs/.hg_archival.txt new file mode 100644 index 0000000..4cab583 --- /dev/null +++ b/unittest-logs/.hg_archival.txt @@ -0,0 +1,2 @@ +repo: a4d5cb4607eb14581b77cf7b1829b7557a78348f +node: 0b8878aa93a5f872db6255b3369f8ff75d7c063b diff --git a/unittest-logs/.hgignore b/unittest-logs/.hgignore new file mode 100644 index 0000000..eca68f0 --- /dev/null +++ b/unittest-logs/.hgignore @@ -0,0 +1,4 @@ +\.pyc$ + +# SQLite database file. +^unittest\.db$ diff --git a/unittest-logs/dbschema.py b/unittest-logs/dbschema.py new file mode 100755 index 0000000..bc45132 --- /dev/null +++ b/unittest-logs/dbschema.py @@ -0,0 +1,74 @@ + +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is TopFails site code. +# +# The Initial Developer of the Original Code is +# Mozilla foundation +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Serge Gautherie +# Ted Mielczarek . +# Murali Nandigama +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +# +# DB schema maintenance functions. +# + +import logging + +__all__ = \ + [ + "CreateDBSchema" + ] + +def CreateDBSchema(conn): + logging.info("Executing CreateDBSchema()") + + + conn.execute(""" + CREATE TABLE IF NOT EXISTS trees( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name TEXT) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS builds(id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, treeid INT, os INT, starttime INT, status INT, changeset TEXT, logfile TEXT) + """) + conn.execute(""" + CREATE INDEX builds_starttime ON builds (starttime) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS tests (buildid INT, name TEXT, description TEXT) + """) + conn.execute(""" + CREATE INDEX tests_name ON tests (name(1024)) + """) + + diff --git a/unittest-logs/dbschema.pyc b/unittest-logs/dbschema.pyc new file mode 100644 index 0000000..f3d84dd Binary files /dev/null and b/unittest-logs/dbschema.pyc differ diff --git a/unittest-logs/unittest-log.py b/unittest-logs/unittest-log.py new file mode 100755 index 0000000..48b36f0 --- /dev/null +++ b/unittest-logs/unittest-log.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python +#Indentation is 2 spaces ***** DO NOT USE TABS ***** + +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is TopFails site code. +# +# The Initial Developer of the Original Code is +# Mozilla foundation +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Serge Gautherie +# Ted Mielczarek . +# Murali Nandigama +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import re, os, sys, urllib, logging +import MySQLdb # Moved from sqlite3 db to MySQL +from time import ctime, sleep, time +from math import ceil +from optparse import OptionParser +from gzip import GzipFile + +try: + # 'Availability: Unix.' + from time import tzset +except ImportError: + print >>sys.stderr, "WARNING: time.tzset() is not available on non-Unixes!" + + # Define a fake function. (for development use only) + # ToDo: Investigate Windows/etc situation. (Bug 525699) + def tzset(): + pass + +try: + import simplejson as json +except ImportError: + try: + # 'New in version 2.6.' + import json + except ImportError: + print >>sys.stderr, "ERROR: no simplejson nor json package found!" + sys.exit(1) + +from dbschema import CreateDBSchema + +# Number of seconds in a hour: 60 mn * 60 s = 3600 s. +S_IN_H = 3600 +# Download data in 24 hours chunks so as not to overwhelm the tinderbox server. +chunksize = 24 * S_IN_H +# seconds between requests +SLEEP_TIME = 1 + +class OS(): + Windows = 0 + Mac = 1 + Linux = 2 + Unknown = 3 + +class BuildStatus(): + # Unavailable builds to skip. + # Values need (only) to be less than the 'Success' one. + NoBuild = -2 + InProgress = -1 + + # Builds to save in the db. + # Do not change these values (without updating db data). + Success = 0 + TestFailed = 1 + Burning = 2 + Exception = 3 + Unknown = 4 + +csetre = re.compile("rev/([0-9A-Za-z]+)") +def FindChangesetInScrape(scrape): + for line in scrape: + m = csetre.search(line) + if m: + return m.group(1) + return None + +def OSFromBuilderName(name): + if name.startswith("Linux"): + return OS.Linux + if name.startswith("MacOSX") or name.startswith("OS X"): + return OS.Mac + if name.startswith("WINNT"): + return OS.Windows + return OS.Unknown + +buildStatuses = { + # "No build in progress". + "null": BuildStatus.NoBuild, + # "Build in progress". + "building": BuildStatus.InProgress, + # "Successful build". + "success": BuildStatus.Success, + # "Successful build, but tests failed". + "testfailed": BuildStatus.TestFailed, + # "Build failed". + "busted": BuildStatus.Burning, + # "Non-build failure". (i.e. "automation failure") + "exception": BuildStatus.Exception, +} +def BuildStatusFromText(status): + try: + return buildStatuses[status] + except KeyError: + # Log 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does. + logging.info("WARNING: unknown status = '%s'!" % status) + return BuildStatus.Unknown + +def GetOrInsertTree(conn, tree): + """Get an id for a tree named |tree|. If it's not already in the trees + table, insert a new row and return the id.""" + + conn.execute("""SELECT id FROM trees WHERE name = %s""", (tree)) + rows = conn.fetchone() + if len(rows) > 0: + return rows[0] + + # need to insert it + conn.execute("""INSERT INTO trees (name) VALUES (%s)""", (tree,)) + return conn.lastrowid + +def HaveBuild(conn, treeid, os, starttime): + """See if we already have this build in our database.""" + conn.execute("""SELECT COUNT(*) FROM builds WHERE treeid = %s AND os = %s AND starttime = %s""", (treeid, os, starttime)) + return conn.fetchone()[0] == 1 + +def UpdateLogfile(conn, treeid, os, starttime, logfile): + """Update empty 'logfile' for a given build (added in db schema v1).""" + conn.execute("""UPDATE builds SET logfile = %s WHERE treeid = %s AND os = %s AND starttime = %s AND logfile IS NULL""", (logfile, treeid, os, starttime)) + +def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset): + """Insert a build into the builds table and return the id.""" + conn.execute("""INSERT INTO builds (treeid, os, starttime, status, logfile, changeset) VALUES (%s, %s, %s, %s, %s, %s)""", (treeid, os, starttime, status, logfile, changeset)) + return conn.lastrowid + +def InsertTest(conn, buildid, result, name, description): + # ToDo: Add column to save result. + conn.execute("""INSERT INTO tests (buildid, name, description) VALUES (%s, %s, %s)""", (buildid, name, description)) + +def fix_tbox_json(s): # Check :: This seems to be a problem ? Murali + """Fixes up tinderbox json. + + Tinderbox returns strings as single-quoted strings, and occasionally + includes the unquoted substring 'undef' (with quotes) in the output, e.g. + + {'key': 'hello 'undef' world'} + + should return a dictionary + + {'key': 'hello \'undef\' world'} + """ + + json_data = re.sub(r"^tinderbox_data\s*=\s*", "", s) + json_data = re.sub(r";$", "", json_data) + retval = [] + in_str = False + in_esc = False + skip = 0 + for i,c in enumerate(json_data): + if skip > 0: + skip -= 1 + continue + + if in_str: + if in_esc: + if c == "'": + retval.append("'") + else: + retval.append("\\") + retval.append(c) + in_esc = False + elif c == "\\": + in_esc = True + elif c == "\"": + retval.append("\\\"") + elif c == "'": + if json_data[i:i+7] == "'undef'": + retval.append("'undef'") + skip = 7 + else: + retval.append("\"") + in_str = False + else: + retval.append(c) + else: + if c == "'": + retval.append("\"") + in_str = True + else: + retval.append(c) + + return "".join(retval) + +parser = OptionParser() +parser.add_option("-s", "--span", action="store", + dest="timespan", default="15d", + help="Period of time to fetch data for (N[y,m,w,d,h], default=%default)") +parser.add_option("-t", "--tree", action="store", + dest="tree", default="Firefox", + help="Tinderbox tree to fetch data from (default=%default)") +parser.add_option("-d", "--database", action="store", + dest="db", default="topfailsdb", + help="Database filename (default=%default)") +parser.add_option("--host", action="store", + dest="dbhost", default="localhost", + help="Database host name (default=%default)") +parser.add_option( "--port", action="store", + dest="dbport",default="3306", + help="Database port (default=%default)") +parser.add_option("-u", "--user", action="store", + dest="dbuser", default="root", + help="Database username (default=%default)") +parser.add_option("-p", "--passwd", action="store", + dest="dbpasswd", + help="Database user password") +parser.add_option("-v", "--verbose", action="store_true", + dest="verbose", default="False", + help="Enable verbose logging") + +(options, args) = parser.parse_args() + +logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING) + +os.environ['TZ'] = "US/Pacific" +tzset() +# Get current time, in seconds. +endtime = int(time()) + +m = re.match("(\d+)([ymwdh])", options.timespan) +if m is None: + print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan + sys.exit(1) + +timespan = int(m.group(1)) * {'y': 365 * 24 * S_IN_H, + 'm': 30 * 24 * S_IN_H, + 'w': 7 * 24 * S_IN_H, + 'd': 24 * S_IN_H, + 'h': S_IN_H}[m.group(2)] +# Set current time to beginning of requested timespan ending now. +curtime = endtime - timespan + + +createdb=False + + +try: + connection = MySQLdb.connect (host = options.dbhost, + port = int(options.dbport), + db = options.db, + user = options.dbuser, + passwd = options.dbpasswd) + conn=connection.cursor() +except MySQLdb.Error, e: + print "Error %d: %s" % (e.args[0], e.args[1]) + createdb = True + + +if createdb: + connection = MySQLdb.connect (host = options.dbhost, + port = int(options.dbport), + user = options.dbuser, + passwd = options.dbpasswd) + conn = connection.cursor() + try: + createdatabase='create database %s' %(options.db) + conn.execute (createdatabase) + conn.close() + connection.close() + except MySQLdb.Error, e: + print "Error %d: %s" % (e.args[0], e.args[1]) + sys.exit (1) + try: + connection = MySQLdb.connect (host = options.dbhost, + port = int(options.dbport), + db = options.db, + user = options.dbuser, + passwd = options.dbpasswd) + conn=connection.cursor() + except MySQLdb.Error, e: + print "Error %d: %s" % (e.args[0], e.args[1]) + sys.exit(1) + + CreateDBSchema(conn) + + +treeid = GetOrInsertTree(conn, options.tree) +logging.info("Reading tinderbox data...") + +chunk = 0 +# add a fudge factor here, since builds can take up to 3 hours to finish, +# and we can't get the changeset unless we ask for time up to the end of the +# build +endtime += 3 * S_IN_H +timespan += 3 * S_IN_H +totalchunks = int(ceil(float(timespan) / chunksize)) + +while curtime < endtime and chunk < totalchunks: + chunk += 1 + logging.info("Chunk %d/%d" % (chunk, totalchunks)) + + if (endtime - curtime) < chunksize: + chunksize = endtime - curtime + + tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1" \ + % {'tree': options.tree, + 'maxdate': curtime + chunksize, # tbox wants the end time + 'hours': int(chunksize / S_IN_H)} + u = urllib.urlopen(tboxurl) + tboxjson = ''.join(u.readlines()) + u.close() + + tboxjson = fix_tbox_json(tboxjson) + try: + tboxdata = json.loads(tboxjson) + except Exception, inst: + print >>sys.stderr, "Error parsing JSON: %s" % inst + continue + + # we only care about unit test boxes + unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("test|xpc", x)] + # read build table + # 'TestFailed' expected log format is "result | test | optional text". + testfailedRe = re.compile(r"(TEST-UNEXPECTED-.*) \| (.*) \|(.*)") + for timerow in tboxdata['build_table']: + for index in unittest_indices: + if index >= len(timerow) or timerow[index] == -1: + continue + build = timerow[index] + if 'buildname' not in build or \ + 'logfile' not in build: + continue + + status = BuildStatusFromText(build['buildstatus']) + # Skip unavailable "builds". + if status < BuildStatus.Success: + continue + + name = build['buildname'] + os = OSFromBuilderName(name) + starttime = int(build['buildtime']) + # skip builds we've already seen + if HaveBuild(conn, treeid, os, starttime): + logging.info("Skipping already seen build '%s' at %d (%s)" % (name, starttime, ctime(starttime))) + + # Call 'UpdateLogfile()' anyway. + UpdateLogfile(conn, treeid, os, starttime, build['logfile']) + continue + + # must have scrape data for changeset + if build['logfile'] not in tboxdata['scrape']: + continue + changeset = FindChangesetInScrape(tboxdata['scrape'][build['logfile']]) + if changeset is None: + continue + + buildid = InsertBuild(conn, treeid, os, starttime, status, build['logfile'], changeset) + + # 'Success' is fine as is. + if status == BuildStatus.Success: + pass + + # Parse log to save 'TestFailed' results. + elif status == BuildStatus.TestFailed: + logging.info("Checking build log for '%s' at %d (%s)" % (name, starttime, ctime(starttime))) + try: + # Grab the build log. + log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile'])) + gz = GzipFile(log) + # Look for test failures. + for line in gz: + m = testfailedRe.match(line) + if m: + test = m.group(2).strip() or "[unittest-log.py: no logged test]" + text = m.group(3).strip() or "[unittest-log.py: no logged text]" + InsertTest(conn, buildid, m.group(1).rstrip(), test, text) + except: + logging.error("Unexpected error: %s" % sys.exc_info()[0]) + #XXX: handle me? + + # Ignore 'Burning' builds: tests may have run nontheless, but it's safer to discard them :-| + elif status == BuildStatus.Burning: + continue + + # Ignore 'Exception' builds: should only be worse than 'Burning'. + # (Don't know much at time of writing, since this feature is not active yet: see bug 476656 and follow-ups.) + elif status == BuildStatus.Exception: + continue + + # Save 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does. + elif status == BuildStatus.Unknown: + # Add a fake test failure. + InsertTest(conn, buildid, "TEST-UNEXPECTED-FAIL", "unittest-log.py", "Unknown status = '%s'!" % build['buildstatus']) + continue + + + + if chunk < totalchunks: + sleep(SLEEP_TIME) + curtime += chunksize + +conn.close() + +logging.info("Done") diff --git a/unittest-logs/unittestweb/__init__.py b/unittest-logs/unittestweb/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/unittest-logs/unittestweb/__init__.pyc b/unittest-logs/unittestweb/__init__.pyc new file mode 100644 index 0000000..dcd7f90 Binary files /dev/null and b/unittest-logs/unittestweb/__init__.pyc differ diff --git a/unittest-logs/unittestweb/manage.py b/unittest-logs/unittestweb/manage.py new file mode 100755 index 0000000..5e78ea9 --- /dev/null +++ b/unittest-logs/unittestweb/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/unittest-logs/unittestweb/settings.py b/unittest-logs/unittestweb/settings.py new file mode 100755 index 0000000..95245b4 --- /dev/null +++ b/unittest-logs/unittestweb/settings.py @@ -0,0 +1,75 @@ +# Django settings for unittestweb project. + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + ('Murali Nandigama', 'murali.nandigama@gmail.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. +DATABASE_NAME = 'topfailsdb' # Or path to database file if using sqlite3. +DATABASE_USER = 'root' # Not used with sqlite3. +DATABASE_PASSWORD = '' # Not used with sqlite3. +DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'America/Los_Angeles' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +# Examples: "http://media.lawrence.com", "http://example.com/media/" +MEDIA_URL = '' + +# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a +# trailing slash. +# Examples: "http://foo.com/media/", "/media/". +ADMIN_MEDIA_PREFIX = '/media/' + +# Make this unique, and don't share it with anybody. +SECRET_KEY = '0_020s+&-*5!y7yn@h9ubk#+4pnzq5$egrx)%5wq!11@)54gtu' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', +) + +ROOT_URLCONF = 'unittestweb.urls' + +TEMPLATE_DIRS = ( + './templates' +) + +INSTALLED_APPS = ( + 'django.contrib.contenttypes', + 'unittestweb.viewer' +) diff --git a/unittest-logs/unittestweb/settings.pyc b/unittest-logs/unittestweb/settings.pyc new file mode 100644 index 0000000..5bfabf9 Binary files /dev/null and b/unittest-logs/unittestweb/settings.pyc differ diff --git a/unittest-logs/unittestweb/settings.py~ b/unittest-logs/unittestweb/settings.py~ new file mode 100755 index 0000000..1fb0629 --- /dev/null +++ b/unittest-logs/unittestweb/settings.py~ @@ -0,0 +1,75 @@ +# Django settings for unittestweb project. + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + ('Murali Nandigama', 'murali.nandigama@gmail.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. +DATABASE_NAME = 'topfailsdb' # Or path to database file if using sqlite3. +DATABASE_USER = 'root' # Not used with sqlite3. +DATABASE_PASSWORD = 'mkngama' # Not used with sqlite3. +DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '3306' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'America/Los_Angeles' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +# Examples: "http://media.lawrence.com", "http://example.com/media/" +MEDIA_URL = '' + +# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a +# trailing slash. +# Examples: "http://foo.com/media/", "/media/". +ADMIN_MEDIA_PREFIX = '/media/' + +# Make this unique, and don't share it with anybody. +SECRET_KEY = '0_020s+&-*5!y7yn@h9ubk#+4pnzq5$egrx)%5wq!11@)54gtu' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', +) + +ROOT_URLCONF = 'unittestweb.urls' + +TEMPLATE_DIRS = ( + './templates' +) + +INSTALLED_APPS = ( + 'django.contrib.contenttypes', + 'unittestweb.viewer' +) diff --git a/unittest-logs/unittestweb/templates/viewer/changeset.html b/unittest-logs/unittestweb/templates/viewer/changeset.html new file mode 100755 index 0000000..969506c --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/changeset.html @@ -0,0 +1,6 @@ +

Changeset {{ changeset }}

+
    +{% for build in builds %} +
  • {{ build.tree }}: {{ build.get_os_display }}: {{ build.get_status_display }}
    • +{% endfor %} +
diff --git a/unittest-logs/unittestweb/templates/viewer/changesets.html b/unittest-logs/unittestweb/templates/viewer/changesets.html new file mode 100755 index 0000000..51e542a --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/changesets.html @@ -0,0 +1,6 @@ +

Changesets

+
    +{% for c in changesets %} +
  • {{ c }}
    • +{% endfor %} +
diff --git a/unittest-logs/unittestweb/templates/viewer/index.html b/unittest-logs/unittestweb/templates/viewer/index.html new file mode 100755 index 0000000..4ef5f0a --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/index.html @@ -0,0 +1,8 @@ +

Most recent test failures

+
    +{% for f in failures %} +
  • {{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree.name }} {{ f.build.get_os_display }}: {{ f.name }}, + timeline + - {{ f.description }}
    • +{% endfor %} +
diff --git a/unittest-logs/unittestweb/templates/viewer/test.html b/unittest-logs/unittestweb/templates/viewer/test.html new file mode 100755 index 0000000..de4954d --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/test.html @@ -0,0 +1,8 @@ +

Test results for {{ test }}

+
    +
    • +{% for f in failures %}{% ifchanged f.build.id %}
  • +{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree }} {{ f.build.get_os_display }} [{{ f.build.changeset_link|safe }}]: +
      {% endifchanged %} +
    • {{ f.description }}
      • {% endfor %} +
    diff --git a/unittest-logs/unittestweb/templates/viewer/tests.html b/unittest-logs/unittestweb/templates/viewer/tests.html new file mode 100755 index 0000000..92807f4 --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/tests.html @@ -0,0 +1,6 @@ +

    All known failing tests

    +
      +{% for t in tests %} +
    • {{ t }}
      • +{% endfor %} +
    diff --git a/unittest-logs/unittestweb/templates/viewer/timeline.html b/unittest-logs/unittestweb/templates/viewer/timeline.html new file mode 100755 index 0000000..8363867 --- /dev/null +++ b/unittest-logs/unittestweb/templates/viewer/timeline.html @@ -0,0 +1,72 @@ + + +Timeline for {{ test }} + + + + + + + +

    Timeline for {{ test }}

    + +
    + 
    
+
    
+
    
+
+
    
+
+  
+    
+    
+    
+  
+
    
+{% for desc in descriptions %}
+
    {{desc}}
    
+{% endfor %}
+        		
+      
    
+        		
+      
    
+    
    
+ 
diff --git a/unittest-logs/unittestweb/templates/viewer/topfails.html b/unittest-logs/unittestweb/templates/viewer/topfails.html
new file mode 100755
index 0000000..9f4af33
--- /dev/null
+++ b/unittest-logs/unittestweb/templates/viewer/topfails.html
@@ -0,0 +1,7 @@
+
    Top 25 failing tests
    
+
+
+{% for f in failures %}
+  
+{% endfor %}
+
    CountTest name
    {{ f.0 }}{{ f.1 }}
    
diff --git a/unittest-logs/unittestweb/templates/viewer/tree.html b/unittest-logs/unittestweb/templates/viewer/tree.html
new file mode 100755
index 0000000..c915113
--- /dev/null
+++ b/unittest-logs/unittestweb/templates/viewer/tree.html
@@ -0,0 +1,6 @@
+
    {{ tree }}
    
+
      
+{% for build in newestbuilds %}
+  
    • {{ build.get_os_display }}: {{ build.get_status_display }}
      • 
+{% endfor %}
+
    
diff --git a/unittest-logs/unittestweb/templates/viewer/trees.html b/unittest-logs/unittestweb/templates/viewer/trees.html
new file mode 100755
index 0000000..f68e635
--- /dev/null
+++ b/unittest-logs/unittestweb/templates/viewer/trees.html
@@ -0,0 +1,7 @@
+
    Trees
    
+
+
diff --git a/unittest-logs/unittestweb/urls.py b/unittest-logs/unittestweb/urls.py
new file mode 100755
index 0000000..8e651cd
--- /dev/null
+++ b/unittest-logs/unittestweb/urls.py
@@ -0,0 +1,18 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+urlpatterns = patterns('unittestweb.viewer.views',
+    (r'^$', 'index'),
+    (r'^/$', 'index'),
+    (r'^trees$', 'trees'),
+    (r'^trees/(?P.+)$', 'tree'),
+    (r'^changesets$', 'changesets'),
+    (r'^changesets/(?P[a-f0-9]+)$', 'changeset'),
+    (r'^tests$', 'tests'),
+    (r'^test$', 'test'),
+    (r'^timeline$', 'timeline'),
+    (r'^topfails$', 'topfails'),
+)
diff --git a/unittest-logs/unittestweb/urls.pyc b/unittest-logs/unittestweb/urls.pyc
new file mode 100644
index 0000000..4d543c6
Binary files /dev/null and b/unittest-logs/unittestweb/urls.pyc differ
diff --git a/unittest-logs/unittestweb/urls.py~ b/unittest-logs/unittestweb/urls.py~
new file mode 100755
index 0000000..d4303d0
--- /dev/null
+++ b/unittest-logs/unittestweb/urls.py~
@@ -0,0 +1,18 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('unittestweb.viewer.views',
+    (r'^$', 'index'),
+    (r'^/$', 'index'),
+    (r'^trees$', 'trees'),
+    (r'^trees/(?P.+)$', 'tree'),
+    (r'^changesets$', 'changesets'),
+    (r'^changesets/(?P[a-f0-9]+)$', 'changeset'),
+    (r'^tests$', 'tests'),
+    (r'^test$', 'test'),
+    (r'^timeline$', 'timeline'),
+    (r'^topfails$', 'topfails'),
+)
diff --git a/unittest-logs/unittestweb/viewer/__init__.py b/unittest-logs/unittestweb/viewer/__init__.py
new file mode 100755
index 0000000..e69de29
diff --git a/unittest-logs/unittestweb/viewer/__init__.pyc b/unittest-logs/unittestweb/viewer/__init__.pyc
new file mode 100644
index 0000000..d95167d
Binary files /dev/null and b/unittest-logs/unittestweb/viewer/__init__.pyc differ
diff --git a/unittest-logs/unittestweb/viewer/models.py b/unittest-logs/unittestweb/viewer/models.py
new file mode 100755
index 0000000..0edcea7
--- /dev/null
+++ b/unittest-logs/unittestweb/viewer/models.py
@@ -0,0 +1,80 @@
+# This is an auto-generated Django model module.
+# You'll have to do the following manually to clean this up:
+#     * Rearrange models' order
+#     * Make sure each model has one field with primary_key=True
+# Feel free to rename the models, but don't rename db_table values or field names.
+#
+# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
+# into your database.
+
+from django.db import models, connection
+from datetime import datetime
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+class Trees(models.Model):
+    id = models.IntegerField(primary_key=True)
+    name = models.TextField(blank=True)
+    def __unicode__(self):
+        return self.name
+    class Meta:
+        db_table = 'trees'
+
+class Builds(models.Model):
+    id = models.IntegerField(primary_key=True)
+    tree = models.ForeignKey(Trees, db_column="treeid")
+    os = models.IntegerField(choices=OS_CHOICES)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    logfile = models.TextField(blank=True)
+    changeset = models.TextField(blank=True)
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    def changeset_link(self):
+      return '%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+    def tinderbox_link(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    class Meta:
+        db_table = 'builds'
+
+class Tests(models.Model):
+    id = models.IntegerField(primary_key=True)
+    build = models.ForeignKey(Builds, db_column="buildid")
+    name = models.TextField(blank=True)
+    description = models.TextField(blank=True)
+    class Meta:
+        db_table = 'tests'
+
+def get_most_failing_tests():
+    cursor = connection.cursor()
+    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) group by name order by count(*) desc limit 250")
+    for row in cursor:
+        yield row
diff --git a/unittest-logs/unittestweb/viewer/models.pyc b/unittest-logs/unittestweb/viewer/models.pyc
new file mode 100644
index 0000000..20d7cb4
Binary files /dev/null and b/unittest-logs/unittestweb/viewer/models.pyc differ
diff --git a/unittest-logs/unittestweb/viewer/models.py~ b/unittest-logs/unittestweb/viewer/models.py~
new file mode 100755
index 0000000..2595938
--- /dev/null
+++ b/unittest-logs/unittestweb/viewer/models.py~
@@ -0,0 +1,80 @@
+# This is an auto-generated Django model module.
+# You'll have to do the following manually to clean this up:
+#     * Rearrange models' order
+#     * Make sure each model has one field with primary_key=True
+# Feel free to rename the models, but don't rename db_table values or field names.
+#
+# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
+# into your database.
+
+from django.db import models, connection
+from datetime import datetime
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+class Trees(models.Model):
+    id = models.IntegerField(primary_key=True)
+    name = models.TextField(blank=True)
+    def __unicode__(self):
+        return self.name
+    class Meta:
+        db_table = u'trees'
+
+class Builds(models.Model):
+    id = models.IntegerField(primary_key=True)
+    tree = models.ForeignKey(Trees, db_column="treeid")
+    os = models.IntegerField(choices=OS_CHOICES)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    logfile = models.TextField(blank=True)
+    changeset = models.TextField(blank=True)
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    def changeset_link(self):
+      return '%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+    def tinderbox_link(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    class Meta:
+        db_table = u'builds'
+
+class Tests(models.Model):
+    ROWID = models.IntegerField(primary_key=True)
+    build = models.ForeignKey(Builds, db_column="buildid")
+    name = models.TextField(blank=True)
+    description = models.TextField(blank=True)
+    class Meta:
+        db_table = u'tests'
+
+def get_most_failing_tests():
+    cursor = connection.cursor()
+    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) group by name order by count(*) desc limit 250")
+    for row in cursor:
+        yield row
diff --git a/unittest-logs/unittestweb/viewer/views.py b/unittest-logs/unittestweb/viewer/views.py
new file mode 100755
index 0000000..a1738a6
--- /dev/null
+++ b/unittest-logs/unittestweb/viewer/views.py
@@ -0,0 +1,55 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from unittestweb.viewer.models import Builds, Trees, Tests, OS_CHOICES, get_most_failing_tests
+
+def index(request):
+  failures = get_list_or_404(Tests.objects.all().order_by('-build__starttime')[:10])
+  return render_to_response('viewer/index.html', {'failures': failures})
+
+def trees(request):
+  alltrees = Trees.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees})
+
+def tree(request, tree):
+  newestbuilds = get_list_or_404(Builds.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request):
+  build_csets = Builds.objects.values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', {'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset):
+  builds = get_list_or_404(Builds, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds})
+
+def tests(request):
+    test_names = Tests.objects.values('name').distinct()
+    return render_to_response('viewer/tests.html', {'tests': [t['name'] for t in test_names]})
+
+def test(request):
+  failures = get_list_or_404(Tests.objects.filter(name__exact=request.GET['name']).order_by('-build__starttime'))
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': failures})
+
+def topfails(request):
+  failures = get_most_failing_tests()
+  return render_to_response('viewer/topfails.html', {'failures': failures})
+  
+def timeline(request):
+  name = request.GET['name']
+  builds = get_list_or_404(Builds, tests__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.tests_set.filter(name__exact=name).order_by('id')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist})
diff --git a/unittest-logs/unittestweb/viewer/views.pyc b/unittest-logs/unittestweb/viewer/views.pyc
new file mode 100644
index 0000000..acd3677
Binary files /dev/null and b/unittest-logs/unittestweb/viewer/views.pyc differ
diff --git a/unittest-logs/unittestweb/viewer/views.py~ b/unittest-logs/unittestweb/viewer/views.py~
new file mode 100755
index 0000000..ed8652b
--- /dev/null
+++ b/unittest-logs/unittestweb/viewer/views.py~
@@ -0,0 +1,55 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from unittestweb.viewer.models import Builds, Trees, Tests, OS_CHOICES, get_most_failing_tests
+
+def index(request):
+  failures = get_list_or_404(Tests.objects.all().order_by('-build__starttime')[:10])
+  return render_to_response('viewer/index.html', {'failures': failures})
+
+def trees(request):
+  alltrees = Trees.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees})
+
+def tree(request, tree):
+  newestbuilds = get_list_or_404(Builds.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request):
+  build_csets = Builds.objects.values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', {'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset):
+  builds = get_list_or_404(Builds, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds})
+
+def tests(request):
+    test_names = Tests.objects.values('name').distinct()
+    return render_to_response('viewer/tests.html', {'tests': [t['name'] for t in test_names]})
+
+def test(request):
+  failures = get_list_or_404(Tests.objects.filter(name__exact=request.GET['name']).order_by('-build__starttime'))
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': failures})
+
+def topfails(request):
+  failures = get_most_failing_tests()
+  return render_to_response('viewer/topfails.html', {'failures': failures})
+  
+def timeline(request):
+  name = request.GET['name']
+  builds = get_list_or_404(Builds, tests__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.tests_set.filter(name__exact=name).order_by('ROWID')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist})