first batch of code

unittest-logs/.hg_archival.txt

@@ -0,0 +1,2 @@
+repo: a4d5cb4607eb14581b77cf7b1829b7557a78348f
+node: 0b8878aa93a5f872db6255b3369f8ff75d7c063b

unittest-logs/.hgignore

@@ -0,0 +1,4 @@
+\.pyc$
+
+# SQLite database file.
+^unittest\.db$

unittest-logs/dbschema.py

@@ -0,0 +1,74 @@
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is TopFails site code.
+#
+# The Initial Developer of the Original Code is
+# Mozilla foundation
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Serge Gautherie <sgautherie.bz@free.fr>
+#   Ted Mielczarek <ted.mielczarek@gmail.com>.
+#   Murali Nandigama <Murali.Nandigama@Gmail.COM>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+#
+# DB schema maintenance functions.
+#
+
+import logging
+
+__all__ = \
+ [
+  "CreateDBSchema"
+ ]
+
+def CreateDBSchema(conn):
+  logging.info("Executing CreateDBSchema()")
+ 
+  
+  conn.execute("""
+               CREATE TABLE IF NOT EXISTS trees( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name TEXT)
+               """)
+
+  conn.execute("""
+               CREATE TABLE IF NOT EXISTS builds(id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, treeid INT, os INT, starttime INT, status INT, changeset TEXT, logfile TEXT)
+              """)
+  conn.execute("""
+               CREATE INDEX builds_starttime ON builds (starttime)
+               """)
+
+  conn.execute("""
+               CREATE TABLE IF NOT EXISTS tests (buildid INT, name TEXT, description TEXT)
+               """)
+  conn.execute("""
+               CREATE INDEX tests_name ON tests (name(1024))
+               """)
+
+

unittest-logs/unittest-log.py

@@ -0,0 +1,433 @@
+#!/usr/bin/env python
+#Indentation is 2 spaces  ***** DO NOT USE TABS *****
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is TopFails site code.
+#    
+# The Initial Developer of the Original Code is
+# Mozilla foundation
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Serge Gautherie <sgautherie.bz@free.fr>
+#   Ted Mielczarek <ted.mielczarek@gmail.com>.
+#   Murali Nandigama <Murali.Nandigama@Gmail.COM>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import re, os, sys, urllib, logging
+import MySQLdb # Moved from  sqlite3 db  to MySQL
+from time import ctime, sleep, time
+from math import ceil
+from optparse import OptionParser
+from gzip import GzipFile
+
+try:
+  # 'Availability: Unix.'
+  from time import tzset
+except ImportError:
+  print >>sys.stderr, "WARNING: time.tzset() is not available on non-Unixes!"
+
+  # Define a fake function. (for development use only)
+  # ToDo: Investigate Windows/etc situation. (Bug 525699)
+  def tzset():
+    pass
+
+try:
+  import simplejson as json
+except ImportError:
+  try:
+    # 'New in version 2.6.'
+    import json
+  except ImportError:
+    print >>sys.stderr, "ERROR: no simplejson nor json package found!"
+    sys.exit(1)
+
+from dbschema import CreateDBSchema  
+
+# Number of seconds in a hour: 60 mn * 60 s = 3600 s.
+S_IN_H = 3600
+# Download data in 24 hours chunks so as not to overwhelm the tinderbox server.
+chunksize = 24 * S_IN_H
+# seconds between requests
+SLEEP_TIME = 1
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+class BuildStatus():
+  # Unavailable builds to skip.
+  # Values need (only) to be less than the 'Success' one.
+  NoBuild = -2
+  InProgress = -1
+
+  # Builds to save in the db.
+  # Do not change these values (without updating db data).
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+csetre = re.compile("rev/([0-9A-Za-z]+)")
+def FindChangesetInScrape(scrape):
+  for line in scrape:
+    m = csetre.search(line)
+    if m:
+      return m.group(1)
+  return None
+
+def OSFromBuilderName(name):
+  if name.startswith("Linux"):
+    return OS.Linux
+  if name.startswith("MacOSX") or name.startswith("OS X"):
+    return OS.Mac
+  if name.startswith("WINNT"):
+    return OS.Windows
+  return OS.Unknown
+
+buildStatuses = {
+  # "No build in progress".
+  "null":       BuildStatus.NoBuild,
+  # "Build in progress".
+  "building":   BuildStatus.InProgress,
+  # "Successful build".
+  "success":    BuildStatus.Success,
+  # "Successful build, but tests failed".
+  "testfailed": BuildStatus.TestFailed,
+  # "Build failed".
+  "busted":     BuildStatus.Burning,
+  # "Non-build failure". (i.e. "automation failure")
+  "exception":  BuildStatus.Exception,
+}
+def BuildStatusFromText(status):
+  try:
+    return buildStatuses[status]
+  except KeyError:
+    # Log 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does.
+    logging.info("WARNING: unknown status = '%s'!" % status)
+    return BuildStatus.Unknown
+
+def GetOrInsertTree(conn, tree):
+  """Get an id for a tree named |tree|. If it's not already in the trees
+  table, insert a new row and return the id."""
+  
+  conn.execute("""SELECT id FROM trees WHERE name = %s""", (tree))
+  rows = conn.fetchone()
+  if len(rows) > 0:
+    return rows[0]
+
+  # need to insert it
+  conn.execute("""INSERT INTO trees (name) VALUES (%s)""", (tree,))
+  return conn.lastrowid
+
+def HaveBuild(conn, treeid, os, starttime):
+  """See if we already have this build in our database."""
+  conn.execute("""SELECT COUNT(*) FROM builds WHERE treeid = %s AND os = %s AND starttime = %s""", (treeid, os, starttime))
+  return conn.fetchone()[0] == 1
+
+def UpdateLogfile(conn, treeid, os, starttime, logfile):
+  """Update empty 'logfile' for a given build (added in db schema v1)."""
+  conn.execute("""UPDATE builds SET logfile = %s WHERE treeid = %s AND os = %s AND starttime = %s AND logfile IS NULL""", (logfile, treeid, os, starttime))
+
+def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset):
+  """Insert a build into the builds table and return the id."""
+  conn.execute("""INSERT INTO builds (treeid, os, starttime, status, logfile, changeset) VALUES (%s, %s, %s, %s, %s, %s)""", (treeid, os, starttime, status, logfile, changeset))
+  return conn.lastrowid
+
+def InsertTest(conn, buildid, result, name, description):
+  # ToDo: Add column to save result.
+  conn.execute("""INSERT INTO tests (buildid, name, description) VALUES (%s, %s, %s)""", (buildid, name, description))
+
+def fix_tbox_json(s): # Check :: This seems to be a problem ? Murali
+  """Fixes up tinderbox json.
+
+  Tinderbox returns strings as single-quoted strings, and occasionally
+  includes the unquoted substring 'undef' (with quotes) in the output, e.g.
+
+  {'key': 'hello 'undef' world'}
+
+  should return a dictionary
+
+  {'key': 'hello \'undef\' world'}
+  """
+
+  json_data = re.sub(r"^tinderbox_data\s*=\s*", "", s)
+  json_data = re.sub(r";$", "", json_data)
+  retval = []
+  in_str = False
+  in_esc = False
+  skip = 0
+  for i,c in enumerate(json_data):
+    if skip > 0:
+      skip -= 1
+      continue
+
+    if in_str:
+      if in_esc:
+        if c == "'":
+          retval.append("'")
+        else:
+          retval.append("\\")
+          retval.append(c)
+        in_esc = False
+      elif c == "\\":
+        in_esc = True
+      elif c == "\"":
+        retval.append("\\\"")
+      elif c == "'":
+        if json_data[i:i+7] == "'undef'":
+          retval.append("'undef'")
+          skip = 7
+        else:
+          retval.append("\"")
+          in_str = False
+      else:
+        retval.append(c)
+    else:
+      if c == "'":
+        retval.append("\"")
+        in_str = True
+      else:
+        retval.append(c)
+
+  return "".join(retval)
+
+parser = OptionParser()
+parser.add_option("-s", "--span", action="store",
+                  dest="timespan", default="15d",
+                  help="Period of time to fetch data for (N[y,m,w,d,h], default=%default)")
+parser.add_option("-t", "--tree", action="store",
+                  dest="tree", default="Firefox",
+                  help="Tinderbox tree to fetch data from (default=%default)")
+parser.add_option("-d", "--database", action="store",
+                  dest="db", default="topfailsdb",
+                  help="Database filename (default=%default)")
+parser.add_option("--host", action="store",
+                  dest="dbhost", default="localhost",
+                  help="Database host name (default=%default)")
+parser.add_option( "--port", action="store",
+                  dest="dbport",default="3306",
+                  help="Database port (default=%default)")
+parser.add_option("-u", "--user", action="store",
+                  dest="dbuser", default="root",
+                  help="Database username (default=%default)")
+parser.add_option("-p", "--passwd", action="store",
+                  dest="dbpasswd",
+                  help="Database user password")
+parser.add_option("-v", "--verbose", action="store_true",
+                  dest="verbose", default="False",
+                  help="Enable verbose logging")
+
+(options, args) = parser.parse_args()
+
+logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING)
+
+os.environ['TZ'] = "US/Pacific"
+tzset()
+# Get current time, in seconds.
+endtime = int(time())
+
+m = re.match("(\d+)([ymwdh])", options.timespan)
+if m is None:
+  print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+  sys.exit(1)
+
+timespan = int(m.group(1)) * {'y': 365 * 24 * S_IN_H,
+                              'm':  30 * 24 * S_IN_H,
+                              'w':   7 * 24 * S_IN_H,
+                              'd':       24 * S_IN_H,
+                              'h':            S_IN_H}[m.group(2)]
+# Set current time to beginning of requested timespan ending now.
+curtime = endtime - timespan
+
+
+createdb=False
+
+
+try:
+  connection  =  MySQLdb.connect (host  =  options.dbhost,
+                                    port  =  int(options.dbport),
+                                    db  =  options.db,
+                                    user  =  options.dbuser,
+                                    passwd  =  options.dbpasswd)
+  conn=connection.cursor()
+except MySQLdb.Error, e:
+  print "Error %d: %s" % (e.args[0], e.args[1])
+  createdb = True
+     
+
+if createdb:
+  connection = MySQLdb.connect (host  =  options.dbhost,
+                                    port  =  int(options.dbport),
+                                    user  =  options.dbuser,
+                                    passwd  =  options.dbpasswd)
+  conn = connection.cursor()
+  try:
+    createdatabase='create database %s' %(options.db)
+    conn.execute (createdatabase)
+    conn.close()
+    connection.close()
+  except  MySQLdb.Error, e:
+    print "Error %d: %s" % (e.args[0], e.args[1])
+    sys.exit (1)
+  try:
+    connection  =  MySQLdb.connect (host  =  options.dbhost,
+                                      port  =  int(options.dbport),
+                                      db  =  options.db,
+                                      user  =  options.dbuser,
+                                      passwd  =  options.dbpasswd)
+    conn=connection.cursor()
+  except MySQLdb.Error, e:
+    print "Error %d: %s" % (e.args[0], e.args[1])
+    sys.exit(1)
+  
+  CreateDBSchema(conn)
+
+
+treeid = GetOrInsertTree(conn, options.tree)
+logging.info("Reading tinderbox data...")
+
+chunk = 0
+# add a fudge factor here, since builds can take up to 3 hours to finish,
+# and we can't get the changeset unless we ask for time up to the end of the
+# build
+endtime += 3 * S_IN_H
+timespan += 3 * S_IN_H
+totalchunks = int(ceil(float(timespan) / chunksize))
+
+while curtime < endtime and chunk < totalchunks:
+  chunk += 1
+  logging.info("Chunk %d/%d" % (chunk, totalchunks))
+
+  if (endtime - curtime) < chunksize:
+    chunksize = endtime - curtime
+
+  tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1" \
+              % {'tree': options.tree,
+                 'maxdate': curtime + chunksize, # tbox wants the end time
+                 'hours': int(chunksize / S_IN_H)}
+  u = urllib.urlopen(tboxurl)
+  tboxjson = ''.join(u.readlines())
+  u.close()
+
+  tboxjson = fix_tbox_json(tboxjson)
+  try:
+    tboxdata = json.loads(tboxjson)
+  except Exception, inst:
+    print >>sys.stderr, "Error parsing JSON: %s" % inst
+    continue
+
+  # we only care about unit test boxes
+  unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("test|xpc", x)]
+  # read build table
+  # 'TestFailed' expected log format is "result | test | optional text".
+  testfailedRe = re.compile(r"(TEST-UNEXPECTED-.*) \| (.*) \|(.*)")
+  for timerow in tboxdata['build_table']:
+    for index in unittest_indices:
+      if index >= len(timerow) or timerow[index] == -1:
+        continue
+      build = timerow[index]
+      if 'buildname' not in build or \
+         'logfile'   not in build:
+        continue
+
+      status = BuildStatusFromText(build['buildstatus'])
+      # Skip unavailable "builds".
+      if status < BuildStatus.Success:
+        continue
+
+      name = build['buildname']
+      os = OSFromBuilderName(name)
+      starttime = int(build['buildtime'])
+      # skip builds we've already seen
+      if HaveBuild(conn, treeid, os, starttime):
+        logging.info("Skipping already seen build '%s' at %d (%s)" % (name, starttime, ctime(starttime)))
+
+        # Call 'UpdateLogfile()' anyway.
+        UpdateLogfile(conn, treeid, os, starttime, build['logfile'])
+        continue
+
+      # must have scrape data for changeset
+      if build['logfile'] not in tboxdata['scrape']:
+        continue
+      changeset = FindChangesetInScrape(tboxdata['scrape'][build['logfile']])
+      if changeset is None:
+        continue
+
+      buildid = InsertBuild(conn, treeid, os, starttime, status, build['logfile'], changeset)
+
+      # 'Success' is fine as is.
+      if status == BuildStatus.Success:
+        pass
+
+      # Parse log to save 'TestFailed' results.
+      elif status == BuildStatus.TestFailed:
+        logging.info("Checking build log for '%s' at %d (%s)" % (name, starttime, ctime(starttime)))
+        try:
+          # Grab the build log.
+          log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile']))
+          gz = GzipFile(log)
+          # Look for test failures.
+          for line in gz:
+            m = testfailedRe.match(line)
+            if m:
+              test = m.group(2).strip() or "[unittest-log.py: no logged test]"
+              text = m.group(3).strip() or "[unittest-log.py: no logged text]"
+              InsertTest(conn, buildid, m.group(1).rstrip(), test, text)
+        except:
+          logging.error("Unexpected error: %s" % sys.exc_info()[0])
+          #XXX: handle me?
+
+      # Ignore 'Burning' builds: tests may have run nontheless, but it's safer to discard them :-|
+      elif status == BuildStatus.Burning:
+        continue
+
+      # Ignore 'Exception' builds: should only be worse than 'Burning'.
+      # (Don't know much at time of writing, since this feature is not active yet: see bug 476656 and follow-ups.)
+      elif status == BuildStatus.Exception:
+        continue
+
+      # Save 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does.
+      elif status == BuildStatus.Unknown:
+        # Add a fake test failure.
+        InsertTest(conn, buildid, "TEST-UNEXPECTED-FAIL", "unittest-log.py", "Unknown status = '%s'!" % build['buildstatus'])
+        continue
+
+
+
+  if chunk < totalchunks:
+    sleep(SLEEP_TIME)
+  curtime += chunksize
+
+conn.close()
+
+logging.info("Done")

unittest-logs/unittestweb/manage.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+from django.core.management import execute_manager
+try:
+    import settings # Assumed to be in the same directory.
+except ImportError:
+    import sys
+    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
+    sys.exit(1)
+
+if __name__ == "__main__":
+    execute_manager(settings)

unittest-logs/unittestweb/settings.py

@@ -0,0 +1,75 @@
+# Django settings for unittestweb project.
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    ('Murali Nandigama', 'murali.nandigama@gmail.com'),
+)
+
+MANAGERS = ADMINS
+
+DATABASE_ENGINE = 'mysql'           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+DATABASE_NAME = 'topfailsdb'             # Or path to database file if using sqlite3.
+DATABASE_USER = 'root'             # Not used with sqlite3.
+DATABASE_PASSWORD = ''         # Not used with sqlite3.
+DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
+DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'America/Los_Angeles'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# Absolute path to the directory that holds media.
+# Example: "/home/media/media.lawrence.com/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+# Examples: "http://media.lawrence.com", "http://example.com/media/"
+MEDIA_URL = ''
+
+# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
+# trailing slash.
+# Examples: "http://foo.com/media/", "/media/".
+ADMIN_MEDIA_PREFIX = '/media/'
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = '0_020s+&-*5!y7yn@h9ubk#+4pnzq5$egrx)%5wq!11@)54gtu'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.load_template_source',
+    'django.template.loaders.app_directories.load_template_source',
+#     'django.template.loaders.eggs.load_template_source',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+)
+
+ROOT_URLCONF = 'unittestweb.urls'
+
+TEMPLATE_DIRS = (
+  './templates'
+)
+
+INSTALLED_APPS = (
+    'django.contrib.contenttypes',
+    'unittestweb.viewer'
+)

unittest-logs/unittestweb/settings.py~

@@ -0,0 +1,75 @@
+# Django settings for unittestweb project.
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    ('Murali Nandigama', 'murali.nandigama@gmail.com'),
+)
+
+MANAGERS = ADMINS
+
+DATABASE_ENGINE = 'mysql'           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+DATABASE_NAME = 'topfailsdb'             # Or path to database file if using sqlite3.
+DATABASE_USER = 'root'             # Not used with sqlite3.
+DATABASE_PASSWORD = 'mkngama'         # Not used with sqlite3.
+DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
+DATABASE_PORT = '3306'             # Set to empty string for default. Not used with sqlite3.
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'America/Los_Angeles'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# Absolute path to the directory that holds media.
+# Example: "/home/media/media.lawrence.com/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+# Examples: "http://media.lawrence.com", "http://example.com/media/"
+MEDIA_URL = ''
+
+# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
+# trailing slash.
+# Examples: "http://foo.com/media/", "/media/".
+ADMIN_MEDIA_PREFIX = '/media/'
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = '0_020s+&-*5!y7yn@h9ubk#+4pnzq5$egrx)%5wq!11@)54gtu'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.load_template_source',
+    'django.template.loaders.app_directories.load_template_source',
+#     'django.template.loaders.eggs.load_template_source',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+)
+
+ROOT_URLCONF = 'unittestweb.urls'
+
+TEMPLATE_DIRS = (
+  './templates'
+)
+
+INSTALLED_APPS = (
+    'django.contrib.contenttypes',
+    'unittestweb.viewer'
+)

unittest-logs/unittestweb/templates/viewer/changeset.html

@@ -0,0 +1,6 @@
+<h1>Changeset {{ changeset }}</h1>
+<ul>
+{% for build in builds %}
+  <li>{{ build.tree }}: {{ build.get_os_display }}: {{ build.get_status_display }}</li>
+{% endfor %}
+</ul>

unittest-logs/unittestweb/templates/viewer/changesets.html

@@ -0,0 +1,6 @@
+<h1>Changesets</h1>
+<ul>
+{% for c in changesets %}
+  <li><a href="{% url viewer.views.changeset c %}">{{ c }}</a></li>
+{% endfor %}
+</ul>

unittest-logs/unittestweb/templates/viewer/index.html

@@ -0,0 +1,8 @@
+<h1>Most recent test failures</h1>
+<ul>
+{% for f in failures %}
+  <li>{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree.name }} {{ f.build.get_os_display }}: <a href="{% url viewer.views.test %}?name={{ f.name }}">{{ f.name }}</a>,
+  <a href="{% url viewer.views.timeline %}?name={{ f.name }}">timeline</a>
+   - {{ f.description }}</li>
+{% endfor %}
+</ul>

unittest-logs/unittestweb/templates/viewer/test.html

@@ -0,0 +1,8 @@
+<h1>Test results for {{ test }}</h1>
+<ul>
+  <li style="display:none"><ul>
+{% for f in failures %}{% ifchanged f.build.id %}</ul></li><li>
+<a href="{{ f.build.tinderbox_link|safe }}">{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree }} {{ f.build.get_os_display }}</a> [{{ f.build.changeset_link|safe }}]:
+  <ul>{% endifchanged %}
+  <li>{{ f.description }}</li>{% endfor %}
+</ul>

unittest-logs/unittestweb/templates/viewer/tests.html

@@ -0,0 +1,6 @@
+<h1>All known failing tests</h1>
+<ul>
+{% for t in tests %}
+  <li><a href="{% url viewer.views.test %}?name={{ t }}">{{ t }}</a></li>
+{% endfor %}
+</ul>

unittest-logs/unittestweb/templates/viewer/timeline.html

@@ -0,0 +1,72 @@
+<html>
+<head>
+<title>Timeline for {{ test }}</title>
+<script src="http://static.simile.mit.edu/exhibit/api-2.0/exhibit-api.js?autoCreate=false"
+        type="text/javascript"></script>
+<script src="http://static.simile.mit.edu/exhibit/extensions-2.0/time/time-extension.js"></script>
+
+<script type="application/javascript">
+var json = {"items": [
+{% for bd in builds %}
+  {
+    "type": "Build",
+    "id": "bld{{ bd.build.id }}",
+    "label": "{{ bd.os }} {{ bd.build.changeset }}",
+    "desc": "{{ bd.description|escapejs|escape }}",
+    "desc_id": "{{ bd.desctype }}",
+    "time": "{{ bd.time }}",
+    "os": "{{ bd.os }}",
+    "rev": "{{ bd.build.changeset }}",
+  },
+{% endfor %}
+  ],
+"properties": {
+  "time": {"valueType": "date"},
+  }
+};
+
+$(document).ready(function() {
+  window.database = Exhibit.Database.create();
+  window.database.loadData(json);
+  window.exhibit = Exhibit.create();
+  window.exhibit.configureFromDOM();
+});
+
+
+</script>
+</head>
+
+<body>
+<h2>Timeline for {{ test }}</h2>
+
+<div ex:role="lens" ex:itemTypes="Build">
+  <div><span ex:content=".os"></span> <a
+  ex:href-subcontent="http://hg.mozilla.org/mozilla-central/rev/&#123;{.rev}&#125;"
+  target="_blank" ex:content=".rev"></a></div> <pre ex:content=".desc"></pre>
+</div>
+<div id="blds" ex:role="exhibit-collection" ex:itemTypes="Build"></div>
+
+<div ex:role="view"
+     ex:viewClass="Timeline" 
+     ex:start=".time"
+     ex:topBandPixelsPerUnit="50",
+     ex:bottomBandPixelsPerUnit="50",
+></div>
+<table border="0">
+  <tr>
+    <td>
+{% for desc in descriptions %}
+<pre>{{desc}}</pre>
+{% endfor %}
+    </td>
+    <td valign="top" style="padding-left: 5em;">
+      <div ex:role="facet" ex:expression=".desc_id"
+          ex:facetLabel="Description"></div>
+    </td>
+    <td valign="top" style="padding-left: 5em;">
+      <div ex:role="facet" ex:expression=".os"
+          ex:facetLabel="OS"></div>
+    </td>
+  </tr>
+</table>
+</body> </html>

unittest-logs/unittestweb/templates/viewer/topfails.html

@@ -0,0 +1,7 @@
+<h1>Top 25 failing tests</h1>
+<table>
+<tr><th>Count</th><th align="left">Test name</th></tr>
+{% for f in failures %}
+  <tr><td>{{ f.0 }}</td><td><a href="{% url viewer.views.test %}?name={{ f.1 }}">{{ f.1 }}</a></td></tr>
+{% endfor %}
+</table>

unittest-logs/unittestweb/templates/viewer/tree.html

@@ -0,0 +1,6 @@
+<h1>{{ tree }}</h1>
+<ul>
+{% for build in newestbuilds %}
+  <li>{{ build.get_os_display }}: {{ build.get_status_display }}</li>
+{% endfor %}
+</ul>

unittest-logs/unittestweb/templates/viewer/trees.html

@@ -0,0 +1,7 @@
+<h1>Trees</h1>
+<ul>
+{% for tree in trees %}
+  <li><a href="{% url viewer.views.tree tree.name %}">{{ tree.name }}</a></li>
+{% endfor %}
+</ul>
+

unittest-logs/unittestweb/urls.py

@@ -0,0 +1,18 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+urlpatterns = patterns('unittestweb.viewer.views',
+    (r'^$', 'index'),
+    (r'^/$', 'index'),
+    (r'^trees$', 'trees'),
+    (r'^trees/(?P<tree>.+)$', 'tree'),
+    (r'^changesets$', 'changesets'),
+    (r'^changesets/(?P<changeset>[a-f0-9]+)$', 'changeset'),
+    (r'^tests$', 'tests'),
+    (r'^test$', 'test'),
+    (r'^timeline$', 'timeline'),
+    (r'^topfails$', 'topfails'),
+)

unittest-logs/unittestweb/urls.py~

@@ -0,0 +1,18 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('unittestweb.viewer.views',
+    (r'^$', 'index'),
+    (r'^/$', 'index'),
+    (r'^trees$', 'trees'),
+    (r'^trees/(?P<tree>.+)$', 'tree'),
+    (r'^changesets$', 'changesets'),
+    (r'^changesets/(?P<changeset>[a-f0-9]+)$', 'changeset'),
+    (r'^tests$', 'tests'),
+    (r'^test$', 'test'),
+    (r'^timeline$', 'timeline'),
+    (r'^topfails$', 'topfails'),
+)

unittest-logs/unittestweb/viewer/models.py

@@ -0,0 +1,80 @@
+# This is an auto-generated Django model module.
+# You'll have to do the following manually to clean this up:
+#     * Rearrange models' order
+#     * Make sure each model has one field with primary_key=True
+# Feel free to rename the models, but don't rename db_table values or field names.
+#
+# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
+# into your database.
+
+from django.db import models, connection
+from datetime import datetime
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+class Trees(models.Model):
+    id = models.IntegerField(primary_key=True)
+    name = models.TextField(blank=True)
+    def __unicode__(self):
+        return self.name
+    class Meta:
+        db_table = 'trees'
+
+class Builds(models.Model):
+    id = models.IntegerField(primary_key=True)
+    tree = models.ForeignKey(Trees, db_column="treeid")
+    os = models.IntegerField(choices=OS_CHOICES)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    logfile = models.TextField(blank=True)
+    changeset = models.TextField(blank=True)
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    def changeset_link(self):
+      return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+    def tinderbox_link(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    class Meta:
+        db_table = 'builds'
+
+class Tests(models.Model):
+    id = models.IntegerField(primary_key=True)
+    build = models.ForeignKey(Builds, db_column="buildid")
+    name = models.TextField(blank=True)
+    description = models.TextField(blank=True)
+    class Meta:
+        db_table = 'tests'
+
+def get_most_failing_tests():
+    cursor = connection.cursor()
+    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) group by name order by count(*) desc limit 250")
+    for row in cursor:
+        yield row

unittest-logs/unittestweb/viewer/models.py~

@@ -0,0 +1,80 @@
+# This is an auto-generated Django model module.
+# You'll have to do the following manually to clean this up:
+#     * Rearrange models' order
+#     * Make sure each model has one field with primary_key=True
+# Feel free to rename the models, but don't rename db_table values or field names.
+#
+# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
+# into your database.
+
+from django.db import models, connection
+from datetime import datetime
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+class Trees(models.Model):
+    id = models.IntegerField(primary_key=True)
+    name = models.TextField(blank=True)
+    def __unicode__(self):
+        return self.name
+    class Meta:
+        db_table = u'trees'
+
+class Builds(models.Model):
+    id = models.IntegerField(primary_key=True)
+    tree = models.ForeignKey(Trees, db_column="treeid")
+    os = models.IntegerField(choices=OS_CHOICES)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    logfile = models.TextField(blank=True)
+    changeset = models.TextField(blank=True)
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    def changeset_link(self):
+      return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+    def tinderbox_link(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    class Meta:
+        db_table = u'builds'
+
+class Tests(models.Model):
+    ROWID = models.IntegerField(primary_key=True)
+    build = models.ForeignKey(Builds, db_column="buildid")
+    name = models.TextField(blank=True)
+    description = models.TextField(blank=True)
+    class Meta:
+        db_table = u'tests'
+
+def get_most_failing_tests():
+    cursor = connection.cursor()
+    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) group by name order by count(*) desc limit 250")
+    for row in cursor:
+        yield row

unittest-logs/unittestweb/viewer/views.py

@@ -0,0 +1,55 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from unittestweb.viewer.models import Builds, Trees, Tests, OS_CHOICES, get_most_failing_tests
+
+def index(request):
+  failures = get_list_or_404(Tests.objects.all().order_by('-build__starttime')[:10])
+  return render_to_response('viewer/index.html', {'failures': failures})
+
+def trees(request):
+  alltrees = Trees.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees})
+
+def tree(request, tree):
+  newestbuilds = get_list_or_404(Builds.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request):
+  build_csets = Builds.objects.values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', {'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset):
+  builds = get_list_or_404(Builds, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds})
+
+def tests(request):
+    test_names = Tests.objects.values('name').distinct()
+    return render_to_response('viewer/tests.html', {'tests': [t['name'] for t in test_names]})
+
+def test(request):
+  failures = get_list_or_404(Tests.objects.filter(name__exact=request.GET['name']).order_by('-build__starttime'))
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': failures})
+
+def topfails(request):
+  failures = get_most_failing_tests()
+  return render_to_response('viewer/topfails.html', {'failures': failures})
+  
+def timeline(request):
+  name = request.GET['name']
+  builds = get_list_or_404(Builds, tests__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.tests_set.filter(name__exact=name).order_by('id')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist})

unittest-logs/unittestweb/viewer/views.py~

@@ -0,0 +1,55 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from unittestweb.viewer.models import Builds, Trees, Tests, OS_CHOICES, get_most_failing_tests
+
+def index(request):
+  failures = get_list_or_404(Tests.objects.all().order_by('-build__starttime')[:10])
+  return render_to_response('viewer/index.html', {'failures': failures})
+
+def trees(request):
+  alltrees = Trees.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees})
+
+def tree(request, tree):
+  newestbuilds = get_list_or_404(Builds.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request):
+  build_csets = Builds.objects.values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', {'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset):
+  builds = get_list_or_404(Builds, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds})
+
+def tests(request):
+    test_names = Tests.objects.values('name').distinct()
+    return render_to_response('viewer/tests.html', {'tests': [t['name'] for t in test_names]})
+
+def test(request):
+  failures = get_list_or_404(Tests.objects.filter(name__exact=request.GET['name']).order_by('-build__starttime'))
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': failures})
+
+def topfails(request):
+  failures = get_most_failing_tests()
+  return render_to_response('viewer/topfails.html', {'failures': failures})
+  
+def timeline(request):
+  name = request.GET['name']
+  builds = get_list_or_404(Builds, tests__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.tests_set.filter(name__exact=name).order_by('ROWID')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist})