diff --git a/topfails/PREREQS b/topfails/PREREQS new file mode 100644 index 0000000..65358f3 --- /dev/null +++ b/topfails/PREREQS @@ -0,0 +1,6 @@ +At the minimum, you should set the following environment variables + +Assuming you are using BASH shell + +export dbuser= +export dbpasswd= diff --git a/topfails/README b/topfails/README new file mode 100644 index 0000000..5302e9b --- /dev/null +++ b/topfails/README @@ -0,0 +1,3 @@ +This is the README for the topfails application. + +Please read the 'PREREQS' document which will explain about the environment variables to be set in order to run the django application. diff --git a/topfails/cleanup.sh b/topfails/cleanup.sh new file mode 100755 index 0000000..fd9a17d --- /dev/null +++ b/topfails/cleanup.sh @@ -0,0 +1,2 @@ +rm `find . -name \*.pyc` +rm `find . -name \*~` diff --git a/topfails/dbschema.py b/topfails/dbschema.py new file mode 100755 index 0000000..868ea13 --- /dev/null +++ b/topfails/dbschema.py @@ -0,0 +1,121 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is TopFails site code. +# +# The Initial Developer of the Original Code is +# Mozilla foundation +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Murali Nandigama +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +# +# DB schema maintenance functions. +# + +import logging + +__all__ = \ + [ + "CreateDBSchema" + ] + +def CreateDBSchema(conn): + logging.info("Executing CreateDBSchema()") + + + conn.execute(""" + CREATE TABLE viewer_tree ( + id integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + name varchar(45) NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE viewer_build ( + id integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + os integer NOT NULL, + tree_id integer NOT NULL, + starttime integer, + status integer NOT NULL, + changeset varchar(80) NOT NULL, + logfile varchar(300) NOT NULL + ) + """) + + conn.execute(""" + ALTER TABLE viewer_build + ADD CONSTRAINT tree_id_refs_id_11e44bee + FOREIGN KEY (tree_id) + REFERENCES viewer_tree (id) + """) + + conn.execute(""" + CREATE TABLE viewer_test ( + id integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + name varchar(300) NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE viewer_testfailure ( + id integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + build_id integer NOT NULL, + test_id integer NOT NULL, + failtext varchar(400) NOT NULL + ) + """) + + conn.execute(""" + ALTER TABLE viewer_testfailure + ADD CONSTRAINT test_id_refs_id_1cc1b9e6 + FOREIGN KEY (test_id) + REFERENCES viewer_test (id) + """) + + conn.execute(""" + ALTER TABLE viewer_testfailure + ADD CONSTRAINT build_id_refs_id_112c09cb + FOREIGN KEY (build_id) + REFERENCES viewer_build (id) + """) + + conn.execute(""" + CREATE INDEX viewer_build_tree_id ON viewer_build (tree_id) + """) + + conn.execute(""" + CREATE INDEX viewer_testfailure_build_id ON viewer_testfailure (build_id) + """) + + conn.execute(""" + CREATE INDEX viewer_testfailure_test_id ON viewer_testfailure (test_id) + """) + + diff --git a/topfails/topfails/__init__.py b/topfails/topfails/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/topfails/topfails/images/img5846075.png b/topfails/topfails/images/img5846075.png new file mode 100755 index 0000000..46f50cc Binary files /dev/null and b/topfails/topfails/images/img5846075.png differ diff --git a/topfails/topfails/images/nav12971480a.gif b/topfails/topfails/images/nav12971480a.gif new file mode 100755 index 0000000..1954c8a Binary files /dev/null and b/topfails/topfails/images/nav12971480a.gif differ diff --git a/topfails/topfails/images/nav12971480i.gif b/topfails/topfails/images/nav12971480i.gif new file mode 100755 index 0000000..6be3081 Binary files /dev/null and b/topfails/topfails/images/nav12971480i.gif differ diff --git a/topfails/topfails/images/nav12971481a.gif b/topfails/topfails/images/nav12971481a.gif new file mode 100755 index 0000000..1097b47 Binary files /dev/null and b/topfails/topfails/images/nav12971481a.gif differ diff --git a/topfails/topfails/images/nav12971481i.gif b/topfails/topfails/images/nav12971481i.gif new file mode 100755 index 0000000..a405a2f Binary files /dev/null and b/topfails/topfails/images/nav12971481i.gif differ diff --git a/topfails/topfails/images/nav12971482a.gif b/topfails/topfails/images/nav12971482a.gif new file mode 100755 index 0000000..4b4fb9a Binary files /dev/null and b/topfails/topfails/images/nav12971482a.gif differ diff --git a/topfails/topfails/images/nav12971482i.gif b/topfails/topfails/images/nav12971482i.gif new file mode 100755 index 0000000..dfefd81 Binary files /dev/null and b/topfails/topfails/images/nav12971482i.gif differ diff --git a/topfails/topfails/images/nav12971483a.gif b/topfails/topfails/images/nav12971483a.gif new file mode 100755 index 0000000..abab0cf Binary files /dev/null and b/topfails/topfails/images/nav12971483a.gif differ diff --git a/topfails/topfails/images/nav12971483i.gif b/topfails/topfails/images/nav12971483i.gif new file mode 100755 index 0000000..9d8e15f Binary files /dev/null and b/topfails/topfails/images/nav12971483i.gif differ diff --git a/topfails/topfails/images/nav12971484a.gif b/topfails/topfails/images/nav12971484a.gif new file mode 100755 index 0000000..5500df3 Binary files /dev/null and b/topfails/topfails/images/nav12971484a.gif differ diff --git a/topfails/topfails/images/nav12971484i.gif b/topfails/topfails/images/nav12971484i.gif new file mode 100755 index 0000000..e95f4fc Binary files /dev/null and b/topfails/topfails/images/nav12971484i.gif differ diff --git a/topfails/topfails/manage.py b/topfails/topfails/manage.py new file mode 100755 index 0000000..bcdd55e --- /dev/null +++ b/topfails/topfails/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/topfails/topfails/settings.py b/topfails/topfails/settings.py new file mode 100644 index 0000000..0e0f594 --- /dev/null +++ b/topfails/topfails/settings.py @@ -0,0 +1,86 @@ +# Django settings for topfails project. +import os + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +DBUSER = DBPASSWD = '' +if os.environ.has_key('dbuser'): + DBUSER=os.environ.get('dbuser') + #print DBUSER +if os.environ.has_key('dbpasswd'): + DBPASSWD=os.environ.get('dbpasswd') + #print DBPASSWD +ADMINS = ( + ('Murali Nandigama', 'murali.nandigama@gmail.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. +DATABASE_NAME = 'topfails' # Or path to database file if using sqlite3. +DATABASE_USER = DBUSER # Not used with sqlite3. +DATABASE_PASSWORD = DBPASSWD # Not used with sqlite3. +DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'America/Chicago' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +# Examples: "http://media.lawrence.com", "http://example.com/media/" +MEDIA_URL = '' + +# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a +# trailing slash. +# Examples: "http://foo.com/media/", "/media/". +ADMIN_MEDIA_PREFIX = '/media/' + +# Make this unique, and don't share it with anybody. +SECRET_KEY = '^_5(02(8sng5+6+x=c$w+3(4xv+x^&b)2ty4y8m01(o7vz$o5_' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', +) + +ROOT_URLCONF = 'topfails.urls' + +TEMPLATE_DIRS = ( + './templates/' +) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'topfails.viewer' +) diff --git a/topfails/topfails/templates/base.html b/topfails/topfails/templates/base.html new file mode 100755 index 0000000..7c26bcb --- /dev/null +++ b/topfails/topfails/templates/base.html @@ -0,0 +1,114 @@ + + + +Topfails + + + + + + + + + + + + + + + + + + + + +
Firefox Logo
+
+
+
Mozilla Tinderbox Topfails Dashboard
+
+ + + + + + + + +
+{% block content %} +{% endblock %} +
+ + diff --git a/topfails/topfails/templates/viewer/Help.html b/topfails/topfails/templates/viewer/Help.html new file mode 100755 index 0000000..75693b5 --- /dev/null +++ b/topfails/topfails/templates/viewer/Help.html @@ -0,0 +1,21 @@ +{% extends "base.html" %} +{% block content %} +
This web site provides the bird eye view of top failures and other details of Firefox unit tests  from the  Tinderbox daily runs.
+

+
Meaning of the Left navigation links
+
    +
  • Latest Failures : This link provides the list of latest failures from the Tinderbox.
    • +
  • Top 25 Failures : This link provides the list of top 25 failures from the entire history of the topfails database. Please note that this data is as complete as the first date we started collecting results. The data we have in the topfails database does not contain all the failures details from the Time ZERO of tinderbox runs.
    • +
  • All Failed Tests: This link displays all the known failures that are stored in the topfails DB.
    • +
  • Failures from X days: This is a link which allows you to query for failures from NOW till a specific period in the past.
    • +
+
      Example: 2d  =   2 days  in the past till  now
+
                          7h  =   7 hours in the past till now
+
                          5w =  5 weeks in the past till now
+
                          1m =  1 month in the past till now
+
                         10y =  10 years in the past till now
+
Example queries for advanced users 2 weeks 'Firefox' failures
+
Example queries for advanced users 2 days 'Firefox' failures
+
Example queries for advanced users 24 hours 'Firefox' failures
+
Example queries for advanced users 1 month 'Firefox' failures
+{% endblock %} diff --git a/topfails/topfails/templates/viewer/README b/topfails/topfails/templates/viewer/README new file mode 100644 index 0000000..1e69967 --- /dev/null +++ b/topfails/topfails/templates/viewer/README @@ -0,0 +1,5 @@ +I have done a very bad and crude hack to add a top level navigation page. Instead of storing images for top navigation page in the correct location , I am jammin gall the top site nav stuff in here. + +Should fix it when time permits. + + diff --git a/topfails/topfails/templates/viewer/changeset.html b/topfails/topfails/templates/viewer/changeset.html new file mode 100755 index 0000000..ca8c307 --- /dev/null +++ b/topfails/topfails/templates/viewer/changeset.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} +{% block content %} +

Changeset {{ changeset }}

+
    +{% for build in builds %} +
  • {{ build.tree }}: {{ build.get_os_display }}: {{ build.get_status_display }}
    • +{% endfor %} +
+{% endblock %} diff --git a/topfails/topfails/templates/viewer/changesets.html b/topfails/topfails/templates/viewer/changesets.html new file mode 100755 index 0000000..f2208fa --- /dev/null +++ b/topfails/topfails/templates/viewer/changesets.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} +{% block content %} +

Changesets

+
    +{% for c in changesets %} +
  • {{ c }}
    • +{% endfor %} +
+{% endblock %} diff --git a/topfails/topfails/templates/viewer/failswindow.html b/topfails/topfails/templates/viewer/failswindow.html new file mode 100644 index 0000000..cf534b5 --- /dev/null +++ b/topfails/topfails/templates/viewer/failswindow.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} +{% block content %} +

Failed tests from last {{n}} {{d}} up to now

+ + +{% for f in failures %} + +{% endfor %} +
CountTest name
{{ f.count }}{{ f.test__name }}
+{% endblock %} diff --git a/topfails/topfails/templates/viewer/index.html b/topfails/topfails/templates/viewer/index.html new file mode 100755 index 0000000..ff2fdd9 --- /dev/null +++ b/topfails/topfails/templates/viewer/index.html @@ -0,0 +1,15 @@ +{% extends "base.html" %} +{% block content %} + +

Most recent test failures

+
    +{% for f in failures %} +
  • {{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree.name }} {{ f.build.get_os_display }}: {{ f.name }}, + timeline + - {{ f.description }}
    • +{% endfor %} +
+{% endblock %} diff --git a/topfails/topfails/templates/viewer/latest.html b/topfails/topfails/templates/viewer/latest.html new file mode 100755 index 0000000..55fd43e --- /dev/null +++ b/topfails/topfails/templates/viewer/latest.html @@ -0,0 +1,15 @@ +{% extends "base.html" %} +{% block content %} + +

Most recent test failures

+
    +{% for f in failures %} +
  • {{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree.name }} {{ f.build.get_os_display }}: {{ f.test.name }} + + - {{ f.failtext }}
    • +{% endfor %} +
+{% endblock %} diff --git a/topfails/topfails/templates/viewer/test.html b/topfails/topfails/templates/viewer/test.html new file mode 100755 index 0000000..9234194 --- /dev/null +++ b/topfails/topfails/templates/viewer/test.html @@ -0,0 +1,11 @@ +{% extends "base.html" %} +{% block content %} +

Logfile links and Changeset links for {{ test }} failures

+
    +
    • +{% for f in failures %}{% ifchanged f.build.id %}
  • +{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree }} {{ f.build.get_os_display }} [{{ f.build.changesetlink|safe }}]: +
      {% endifchanged %} + {% endfor %} +
    +{% endblock %} diff --git a/topfails/topfails/templates/viewer/test.html.orig b/topfails/topfails/templates/viewer/test.html.orig new file mode 100755 index 0000000..2670eb3 --- /dev/null +++ b/topfails/topfails/templates/viewer/test.html.orig @@ -0,0 +1,27 @@ +{% extends "base.html" %} +{% block content %} +

    Logfile links and Changeset links for {{ test }} failures

    +
      +
      • +{% for f in section %}{% ifchanged f.build.id %}
    • +{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree }} {{ f.build.get_os_display }} [{{ f.build.changesetlink|safe }}]: +
        {% endifchanged %} + {% endfor %} +
      +{% endblock %} + + \ No newline at end of file diff --git a/topfails/topfails/templates/viewer/tests.html b/topfails/topfails/templates/viewer/tests.html new file mode 100755 index 0000000..bc2274a --- /dev/null +++ b/topfails/topfails/templates/viewer/tests.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} +{% block content %} + + +

      All known failing tests

      +
        +{% for t in tests %} +
      • {{ t }}
        • +{% endfor %} +
      +{% endblock %} diff --git a/topfails/topfails/templates/viewer/timeline.html b/topfails/topfails/templates/viewer/timeline.html new file mode 100755 index 0000000..8363867 --- /dev/null +++ b/topfails/topfails/templates/viewer/timeline.html @@ -0,0 +1,72 @@ + + +Timeline for {{ test }} + + + + + + + +

      Timeline for {{ test }}

      + +
      + 
      
+
      
+
      
+
+
      
+
+  
+    
+    
+    
+  
+
      
+{% for desc in descriptions %}
+
      {{desc}}
      
+{% endfor %}
+          		
+      
      
+          		
+      
      
+    
      
+ 
diff --git a/topfails/topfails/templates/viewer/topfails.html b/topfails/topfails/templates/viewer/topfails.html
new file mode 100755
index 0000000..6ae619d
--- /dev/null
+++ b/topfails/topfails/templates/viewer/topfails.html
@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{% block content %}
+
      Top 25 failing tests
      
+
+
+{% for f in failures %}
+  
+{% endfor %}
+
      CountTest name
      {{ f.count }}{{ f.test__name}}
      
+{% endblock %}
diff --git a/topfails/topfails/templates/viewer/topfails.rar b/topfails/topfails/templates/viewer/topfails.rar
new file mode 100755
index 0000000..52c1cfb
Binary files /dev/null and b/topfails/topfails/templates/viewer/topfails.rar differ
diff --git a/topfails/topfails/templates/viewer/tree.html b/topfails/topfails/templates/viewer/tree.html
new file mode 100755
index 0000000..7133671
--- /dev/null
+++ b/topfails/topfails/templates/viewer/tree.html
@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block content %}
+
      {{ tree }}
      
+
        
+{% for build in newestbuilds %}
+  
      • {{ build.get_os_display }}: {{ build.get_status_display }}
        • 
+{% endfor %}
+
      
+{% endblock %}
diff --git a/topfails/topfails/templates/viewer/trees.html b/topfails/topfails/templates/viewer/trees.html
new file mode 100755
index 0000000..51e1767
--- /dev/null
+++ b/topfails/topfails/templates/viewer/trees.html
@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{% block content %}
+
+
      Trees
      
+
+{% endblock %}
diff --git a/topfails/topfails/urls.py b/topfails/topfails/urls.py
new file mode 100644
index 0000000..9304501
--- /dev/null
+++ b/topfails/topfails/urls.py
@@ -0,0 +1,20 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+urlpatterns = patterns('topfails.viewer.views',
+    (r'^$', 'index'),
+    url(r'^trees/(?P.+)?$','trees', name='trees'),
+    url(r'^tree/(?P.+)?$','tree', name='tree'),
+    url(r'^changesets/(?P.+)?$','changesets', name='changesets'),
+    url(r'^changesets/(?P.+)/(?P[a-f0-9]+)$', 'changeset', name='changeset'),
+    url(r'^tests/(?P.+)?$','tests', name='tests'),
+    url(r'^test/(?P.+)?$','test', name='test'),
+    url(r'^timeline/(?P.+)?$','timeline', name='timeline'),
+    url(r'^topfails/(?P.+)?$','topfails', name='topfails'),
+    url(r'^failswindow/(?P.+)?$','failswindow', name='failswindow'),
+    url(r'^latest/(?P.+)?$','latest', name='latest'),
+    url(r'^Help/(?P.+)?$','Help', name='Help'),
+)
\ No newline at end of file
diff --git a/topfails/topfails/viewer/__init__.py b/topfails/topfails/viewer/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/topfails/topfails/viewer/models.py b/topfails/topfails/viewer/models.py
new file mode 100644
index 0000000..db49527
--- /dev/null
+++ b/topfails/topfails/viewer/models.py
@@ -0,0 +1,130 @@
+import re
+from django.db import models, connection
+from datetime import datetime
+from time import ctime, sleep, time
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+
+class Tree(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=45, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+    
+class Build(models.Model):
+    id = models.AutoField(primary_key=True)
+    os = models.IntegerField(choices=OS_CHOICES)
+    tree = models.ForeignKey(Tree)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    changeset = models.CharField(max_length=80,blank=True)
+    logfile = models.CharField(max_length=300,blank=True)
+    
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    
+    def changesetlink(self):
+      if str(self.tree)  == 'Firefox':
+        return '%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+      elif str(self.tree) == 'Firefox3.6':
+        return '%s' % ("http://hg.mozilla.org/releases/mozilla-1.9.2", self.changeset, self.changeset)
+      elif str(self.tree) == 'Thunderbird' or str(self.tree) == 'SeaMonkey':  
+        return '%s' % ("http://hg.mozilla.org/comm-central", self.changeset, self.changeset)
+      else :
+        return '%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset) 
+    
+    def jsonchangesetlink(self):
+      return "%s/rev/%s" %  ("http://hg.mozilla.org/mozilla-central", self.changeset)
+    
+    def tinderboxlink(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    
+
+class Test(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=300, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+   
+class TestFailure(models.Model):
+    id = models.AutoField(primary_key=True)
+    build = models.ForeignKey(Build)
+    test = models.ForeignKey(Test)
+    failtext = models.CharField(max_length=400, blank=True)
+    
+    
+    
+def get_most_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')[:25]
+
+def get_time_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')
+
+#This def is never used not intended to be used. I kept it here as a refresher
+def get_latest_fails(tree):
+     cursor = connection.cursor()
+     statement="""SELECT viewer_testfailure.id, viewer_testfailure.build_id, viewer_test.name, viewer_testfailure.failtext
+     FROM viewer_testfailure, viewer_test, viewer_build, viewer_tree
+     WHERE (viewer_testfailure.build_id = viewer_build.id)
+     AND (viewer_build.tree_id = viewer_tree.id)
+     AND ( viewer_testfailure.test_id = viewer_test.id)
+     AND viewer_tree.name ='"""+tree+"""' ORDER BY viewer_build.starttime DESC LIMIT 20"""
+     cursor.execute(statement)
+     for row in cursor:
+       yield row
+       
+       
+def get_fails_in_timerange(self,tree):
+  
+  # Get current time, in seconds.
+  endtime = int(time())
+  
+  #print endtime
+  m = re.match("(\d+)([ymwdh])", self)
+  #print m.group(1), m.group(2)
+  if m is None:
+    print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+    sys.exit(1)
+  
+  timespan = int(m.group(1)) * {'y': 365 * 24 * 3600,
+                                'm':  30 * 24 * 3600,
+                                'w':   7 * 24 * 3600,
+                                'd':       24 * 3600,
+                                'h':            3600}[m.group(2)]
+  # Set current time to beginning of requested timespan ending now.
+  curtime = endtime - timespan
+  qs = get_time_failing_tests(tree)
+  return qs.filter(build__starttime__gt=curtime)
+
+    
diff --git a/topfails/topfails/viewer/models.py~ b/topfails/topfails/viewer/models.py~
new file mode 100644
index 0000000..d302b6d
--- /dev/null
+++ b/topfails/topfails/viewer/models.py~
@@ -0,0 +1,129 @@
+import re
+from django.db import models, connection
+from datetime import datetime
+from time import ctime, sleep, time
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+
+class Tree(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=45, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+    
+class Build(models.Model):
+    id = models.AutoField(primary_key=True)
+    os = models.IntegerField(choices=OS_CHOICES)
+    tree = models.ForeignKey(Tree)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    changeset = models.CharField(max_length=80,blank=True)
+    logfile = models.CharField(max_length=300,blank=True)
+    
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    
+    def changesetlink(self):
+      if str(self.tree)  == 'Firefox':
+        return '%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+      elif str(self.tree) == 'Firefox3.6':
+        return '%s' % ("http://hg.mozilla.org/releases/mozilla-1.9.2", self.changeset, self.changeset)
+      elif str(self.tree) == 'Thunderbird' or str(self.tree) == 'SeaMonkey':  
+        return '%s' % ("http://hg.mozilla.org/comm-central", self.changeset, self.changeset)
+      else :
+        return '%s' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset) 
+    
+    def jsonchangesetlink(self):
+      return "%s/rev/%s" %  ("http://hg.mozilla.org/mozilla-central", self.changeset)
+    
+    def tinderboxlink(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    
+
+class Test(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=300, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+   
+class TestFailure(models.Model):
+    id = models.AutoField(primary_key=True)
+    build = models.ForeignKey(Build)
+    test = models.ForeignKey(Test)
+    failtext = models.CharField(max_length=400, blank=True)
+    
+    
+    
+def get_most_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')[:25]
+
+def get_time_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')
+
+def get_latest_fails(tree):
+     cursor = connection.cursor()
+     statement="""SELECT viewer_testfailure.id, viewer_testfailure.build_id, viewer_test.name, viewer_testfailure.failtext
+     FROM viewer_testfailure, viewer_test, viewer_build, viewer_tree
+     WHERE (viewer_testfailure.build_id = viewer_build.id)
+     AND (viewer_build.tree_id = viewer_tree.id)
+     AND ( viewer_testfailure.test_id = viewer_test.id)
+     AND viewer_tree.name ='"""+tree+"""' ORDER BY viewer_build.starttime DESC LIMIT 20"""
+     cursor.execute(statement)
+     for row in cursor:
+       yield row
+       
+       
+def get_fails_in_timerange(self,tree):
+  
+  # Get current time, in seconds.
+  endtime = int(time())
+  
+  #print endtime
+  m = re.match("(\d+)([ymwdh])", self)
+  #print m.group(1), m.group(2)
+  if m is None:
+    print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+    sys.exit(1)
+  
+  timespan = int(m.group(1)) * {'y': 365 * 24 * 3600,
+                                'm':  30 * 24 * 3600,
+                                'w':   7 * 24 * 3600,
+                                'd':       24 * 3600,
+                                'h':            3600}[m.group(2)]
+  # Set current time to beginning of requested timespan ending now.
+  curtime = endtime - timespan
+  qs = get_time_failing_tests(tree)
+  return qs.filter(build__starttime__gt=curtime)
+
+    
diff --git a/topfails/topfails/viewer/views.py b/topfails/topfails/viewer/views.py
new file mode 100644
index 0000000..7d8be5e
--- /dev/null
+++ b/topfails/topfails/viewer/views.py
@@ -0,0 +1,107 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from topfails.viewer.models import Build, Tree, Test,TestFailure, OS_CHOICES, get_most_failing_tests, get_fails_in_timerange
+import re
+from django.http import HttpResponse
+import json
+
+def latest(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  if request.GET.has_key('json'):
+    jtext = [{"Test_name":f.test.name, "Build_status":f.build.status, "Logfile": f.build.tinderboxlink(),"Changeset":f.build.jsonchangesetlink() , "Failure_description":f.failtext} for f in failures]
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+def index(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+
+def trees(request,tree='Firefox'):
+  alltrees = Tree.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees , 'tree' : tree})
+
+def tree(request, tree='Firefox'):
+  newestbuilds = get_list_or_404(Build.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request,tree='Firefox'):
+  build_csets = Build.objects.filter(tree__name__exact=tree).values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', { 'tree' : tree,'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset,tree='Firefox'):
+  builds = get_list_or_404(Build, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds, 'tree' : tree})
+
+def tests(request,tree='Firefox'):
+    test_names = TestFailure.objects.filter(build__tree__name__exact=tree).values('test__name').distinct()
+    if request.GET.has_key('json'):
+      jtext = list(test_names)
+      return HttpResponse(json.dumps(jtext))
+    else:
+      return render_to_response('viewer/tests.html', { 'tree' : tree, 'tests': [t['test__name'] for t in test_names]})
+
+def test(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name__exact=tree).filter(test__name__exact=request.GET['name']).order_by('-build__starttime'))
+  #if request.GET.has_key('json'):
+    #jtext = list(failures)
+    #return HttpResponse(json.dumps(jtext))
+  #else:
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': failures, 'tree' : tree})
+
+def topfails(request,tree='Firefox'):
+  failures = get_most_failing_tests(tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/topfails.html', {'failures': failures, 'tree' : tree})
+  
+def Help(request,tree):
+  return render_to_response('viewer/Help.html',{'tree':tree})  
+  
+def timeline(request,tree='Firefox'):
+  name = request.GET['name']
+  builds = get_list_or_404(Build.objects.filter(tree__name__exact=tree), test__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.testfailure_set.filter(testfailure__name__exact=name).order_by('id')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist, 'tree' : tree})
+
+def failswindow(request,tree='Firefox'):
+  period=request.GET['window']
+  m = re.match("(\d+)([ymwdh])", period)
+  failures = get_fails_in_timerange(period,tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    if m.group(2) == 'd':
+      prd='days'
+    elif m.group(2) == 'h':
+      prd = 'hours'
+    elif m.group(2) == 'w':
+      prd = 'weeks'
+    elif m.group(2) == 'm':
+      prd = 'months'
+    elif m.group(2) == 'y':
+      prd = 'years'
+    else:
+      prd = 'days'
+    
+      
+    return render_to_response('viewer/failswindow.html', {'failures': failures,'n':m.group(1),'d':prd, 'tree' : tree})
+    
diff --git a/topfails/topfails/viewer/views.py.orig b/topfails/topfails/viewer/views.py.orig
new file mode 100644
index 0000000..278c3dd
--- /dev/null
+++ b/topfails/topfails/viewer/views.py.orig
@@ -0,0 +1,118 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from topfails.viewer.models import Build, Tree, Test,TestFailure, OS_CHOICES, get_most_failing_tests, get_fails_in_timerange
+import re
+from django.http import HttpResponse
+import json
+
+def latest(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  if request.GET.has_key('json'):
+    jtext = [{"Test_name":f.test.name, "Build_status":f.build.status, "Logfile": f.build.tinderboxlink(),"Changeset":f.build.jsonchangesetlink() , "Failure_description":f.failtext} for f in failures]
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+def index(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+
+def trees(request,tree='Firefox'):
+  alltrees = Tree.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees , 'tree' : tree})
+
+def tree(request, tree='Firefox'):
+  newestbuilds = get_list_or_404(Build.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request,tree='Firefox'):
+  build_csets = Build.objects.filter(tree__name__exact=tree).values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', { 'tree' : tree,'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset,tree='Firefox'):
+  builds = get_list_or_404(Build, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds, 'tree' : tree})
+
+def tests(request,tree='Firefox'):
+    test_names = TestFailure.objects.filter(build__tree__name__exact=tree).values('test__name').distinct()
+    if request.GET.has_key('json'):
+      jtext = list(test_names)
+      return HttpResponse(json.dumps(jtext))
+    else:
+      return render_to_response('viewer/tests.html', { 'tree' : tree, 'tests': [t['test__name'] for t in test_names]})
+
+from django.core.paginator import Paginator, InvalidPage, EmptyPage
+def test(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name__exact=tree).filter(test__name__exact=request.GET['name']).order_by('-build__starttime'))
+  paginator = Paginator(failures, 25) # Show 25 failures per page
+
+  # Make sure page request is an int. If not, deliver first page.
+  try:
+      page = int(request.GET.get('page', '1'))
+  except ValueError:
+      page = 1
+
+  # If page request (9999) is out of range, deliver last page of results.
+  try:
+     section = paginator.page(page)
+  except (EmptyPage, InvalidPage):
+     section = paginator.page(paginator.num_pages)
+
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': section, 'tree' : tree})
+
+def topfails(request,tree='Firefox'):
+  failures = get_most_failing_tests(tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/topfails.html', {'failures': failures, 'tree' : tree})
+  
+def Help(request,tree):
+  return render_to_response('viewer/Help.html',{'tree':tree})  
+  
+def timeline(request,tree='Firefox'):
+  name = request.GET['name']
+  builds = get_list_or_404(Build.objects.filter(tree__name__exact=tree), test__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.testfailure_set.filter(testfailure__name__exact=name).order_by('id')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist, 'tree' : tree})
+
+def failswindow(request,tree='Firefox'):
+  period=request.GET['window']
+  m = re.match("(\d+)([ymwdh])", period)
+  failures = get_fails_in_timerange(period,tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    if m.group(2) == 'd':
+      prd='days'
+    elif m.group(2) == 'h':
+      prd = 'hours'
+    elif m.group(2) == 'w':
+      prd = 'weeks'
+    elif m.group(2) == 'm':
+      prd = 'months'
+    elif m.group(2) == 'y':
+      prd = 'years'
+    else:
+      prd = 'days'
+    
+      
+    return render_to_response('viewer/failswindow.html', {'failures': failures,'n':m.group(1),'d':prd, 'tree' : tree})
+    
diff --git a/topfails/unittest-log.py b/topfails/unittest-log.py
new file mode 100755
index 0000000..714efba
--- /dev/null
+++ b/topfails/unittest-log.py
@@ -0,0 +1,493 @@
+#!/usr/bin/env python
+#Indentation is 2 spaces  ***** DO NOT USE TABS *****
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is TopFails site code.
+#    
+# The Initial Developer of the Original Code is
+# Mozilla foundation
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Serge Gautherie 
+#   Ted Mielczarek .
+#   Murali Nandigama 
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import re, os, sys, urllib, logging
+import MySQLdb # Moved from  sqlite3 db  to MySQL
+from time import ctime, sleep, time
+from math import ceil
+from optparse import OptionParser
+from gzip import GzipFile
+import binascii
+
+try:
+  # 'Availability: Unix.'
+  from time import tzset
+except ImportError:
+  print >>sys.stderr, "WARNING: time.tzset() is not available on non-Unixes!"
+
+  # Define a fake function. (for development use only)
+  # ToDo: Investigate Windows/etc situation. (Bug 525699)
+  def tzset():
+    pass
+
+try:
+  import simplejson as json
+except ImportError:
+  try:
+    # 'New in version 2.6.'
+    import json
+  except ImportError:
+    print >>sys.stderr, "ERROR: no simplejson nor json package found!"
+    sys.exit(1)
+
+from dbschema import CreateDBSchema  
+
+# Number of seconds in a hour: 60 mn * 60 s = 3600 s.
+S_IN_H = 3600
+# Download data in 24 hours chunks so as not to overwhelm the tinderbox server.
+chunksize = 24 * S_IN_H
+# seconds between requests
+SLEEP_TIME = 1
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+class BuildStatus():
+  # Unavailable builds to skip.
+  # Values need (only) to be less than the 'Success' one.
+  NoBuild = -2
+  InProgress = -1
+
+  # Builds to save in the db.
+  # Do not change these values (without updating db data).
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+csetre = re.compile("rev/([0-9A-Za-z]+)")
+def FindChangesetInScrape(scrape):
+  for line in scrape:
+    m = csetre.search(line)
+    if m:
+      return m.group(1)
+  return None
+
+def OSFromBuilderName(name):
+  if name.startswith("Linux") or name.startswith("Rev3 Fedora") or name.startswith("Fedora"):
+    return OS.Linux
+  if name.startswith("MacOSX") or name.startswith("OS X") or name.startswith("Rev3 MacOSX"):
+    return OS.Mac
+  if name.startswith("WINNT"):
+    return OS.Windows
+  return OS.Unknown
+
+buildStatuses = {
+  # "No build in progress".
+  "null":       BuildStatus.NoBuild,
+  # "Build in progress".
+  "building":   BuildStatus.InProgress,
+  # "Successful build".
+  "success":    BuildStatus.Success,
+  # "Successful build, but tests failed".
+  "testfailed": BuildStatus.TestFailed,
+  # "Build failed".
+  "busted":     BuildStatus.Burning,
+  # "Non-build failure". (i.e. "automation failure")
+  "exception":  BuildStatus.Exception,
+}
+def BuildStatusFromText(status):
+  try:
+    return buildStatuses[status]
+  except KeyError:
+    # Log 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does.
+    logging.info("WARNING: unknown status = '%s'!" % status)
+    return BuildStatus.Unknown
+
+def GetOrInsertTree(conn, tree):
+  """Get an id for a tree named |tree|. If it's not already in the trees
+  table, insert a new row and return the id."""
+  
+  conn.execute("""
+               SELECT id FROM viewer_tree WHERE name = %s
+               """, (tree))
+  if conn.rowcount > 0:
+    return conn.fetchone()[0]
+
+  # need to insert it
+  conn.execute("""
+        INSERT INTO viewer_tree (name) VALUES (%s)
+        """, (tree))
+  connection.commit()
+  return conn.lastrowid
+
+def GetOrInsertTest(conn, testname):
+  """Get an id for a test named |testname|. If it's not already in the testnames
+  table, insert a new row and return the id."""
+  
+  conn.execute("""
+               SELECT id FROM viewer_test WHERE name = %s
+               """, (testname))
+  if conn.rowcount > 0:
+    return conn.fetchone()[0]
+
+  # need to insert it
+  conn.execute("""
+               INSERT INTO viewer_test (name) VALUES (%s)
+               """, (testname))
+  connection.commit()
+  return conn.lastrowid
+
+def HaveBuild(conn, treeid, os, starttime):
+  """See if we already have this build in our database."""
+  conn.execute("""
+               SELECT COUNT(*) FROM viewer_build WHERE tree_id = %s AND os = %s AND starttime = %s
+               """, (treeid, os, starttime))
+  return conn.fetchone()[0] == 1
+
+def UpdateLogfile(conn, treeid, os, starttime, logfile):
+  """Update empty 'logfile' for a given build (added in db schema v1)."""
+  conn.execute("""
+               UPDATE viewer_build SET logfile = %s WHERE tree_id = %s AND os = %s AND starttime = %s AND logfile IS NULL
+               """, (logfile, treeid, os, starttime))
+  connection.commit()
+def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset):
+  """Insert a build into the builds table and return the id."""
+  conn.execute("""
+               INSERT INTO viewer_build (tree_id, os, starttime, status, logfile, changeset) VALUES (%s, %s, %s, %s, %s, %s)
+               """, (treeid, os, starttime, status, logfile, changeset))
+  connection.commit()
+  return conn.lastrowid
+
+def InsertTest(conn, buildid, result, testnames_id, description):
+  # ToDo: Add column to save result.
+  conn.execute("""
+               INSERT INTO viewer_testfailure (build_id, test_id, failtext) VALUES (%s, %s, %s)
+               """, (buildid, testnames_id, description))
+  connection.commit()
+def asciirepl(match):
+  # replace the hexadecimal characters with ascii characters
+  s = match.group()  
+  return binascii.unhexlify(s)  
+
+def reformat_content(data):
+  p = re.compile(r'\\x(\w{2})')
+  return p.sub(asciirepl, data)
+  
+  
+def fix_tbox_json(s): 
+  """Fixes up tinderbox json.
+
+  Tinderbox returns strings as single-quoted strings, and occasionally
+  includes the unquoted substring 'undef' (with quotes) in the output, e.g.
+
+  {'key': 'hello 'undef' world'}
+
+  should return a dictionary
+
+  {'key': 'hello \'undef\' world'}
+  """
+
+  json_data = re.sub(r"^tinderbox_data\s*=\s*", "", s)
+  json_data = re.sub(r";$", "", json_data)
+  retval = []
+  in_str = False
+  in_esc = False
+  skip = 0
+  for i,c in enumerate(json_data):
+    # The tinderbox data is a fracked json. and it some times contains
+    # Control characters. that would totally fail the json.loads step.
+    # So, eliminate them .. all of them .. here -- Murali
+    if (c < '\xFD' and c > '\x1F') or c == '\n' or c == '\r' :
+      if skip > 0:
+        skip -= 1
+        continue
+  
+      if in_str:
+        if in_esc:
+          if c == "'":
+            retval.append("'")
+          else:
+            retval.append("\\")
+            retval.append(c)
+          in_esc = False
+        elif c == "\\":
+          in_esc = True
+        elif c == "\"":
+          retval.append("\\\"")
+        elif c == "'":
+          if json_data[i:i+7] == "'undef'":
+            retval.append("'undef'")
+            skip = 7
+          else:
+            retval.append("\"")
+            in_str = False
+        else:
+          retval.append(c)
+      else:
+        if c == "'":
+          retval.append("\"")
+          in_str = True
+        else:
+          retval.append(c)
+  return "".join(retval)
+
+parser = OptionParser()
+parser.add_option("-s", "--span", action="store",
+                  dest="timespan", default="20d",
+                  help="Period of time to fetch data for (N[y,m,w,d,h], default=%default)")
+parser.add_option("-t", "--tree", action="store",
+                  dest="tree", default="Firefox",
+                  help="Tinderbox tree to fetch data from (default=%default)")
+parser.add_option("-d", "--database", action="store",
+                  dest="db", default="topfails",
+                  help="Database filename (default=%default)")
+parser.add_option("--host", action="store",
+                  dest="dbhost", default="localhost",
+                  help="Database host name (default=%default)")
+parser.add_option( "--port", action="store",
+                  dest="dbport",default="3306",
+                  help="Database port (default=%default)")
+parser.add_option("-u", "--user", action="store",
+                  dest="dbuser", default="root",
+                  help="Database username (default=%default)")
+parser.add_option("-p", "--passwd", action="store",
+                  dest="dbpasswd",
+                  help="Database user password")
+parser.add_option("-v", "--verbose", action="store_true",
+                  dest="verbose", default="False",
+                  help="Enable verbose logging")
+
+(options, args) = parser.parse_args()
+
+logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING)
+
+os.environ['TZ'] = "US/Pacific"
+tzset()
+# Get current time, in seconds.
+endtime = int(time())
+
+m = re.match("(\d+)([ymwdh])", options.timespan)
+if m is None:
+  print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+  sys.exit(1)
+
+timespan = int(m.group(1)) * {'y': 365 * 24 * S_IN_H,
+                              'm':  30 * 24 * S_IN_H,
+                              'w':   7 * 24 * S_IN_H,
+                              'd':       24 * S_IN_H,
+                              'h':            S_IN_H}[m.group(2)]
+# Set current time to beginning of requested timespan ending now.
+curtime = endtime - timespan
+
+
+createdb=False
+
+
+try:
+  connection  =  MySQLdb.connect (host  =  options.dbhost,
+                                    port  =  int(options.dbport),
+                                    db  =  options.db,
+                                    user  =  options.dbuser,
+                                    passwd  =  options.dbpasswd)
+  conn=connection.cursor()
+except MySQLdb.Error, e:
+  print "Error %d: %s" % (e.args[0], e.args[1])
+  createdb = True
+     
+
+if createdb:
+  connection = MySQLdb.connect (host  =  options.dbhost,
+                                    port  =  int(options.dbport),
+                                    user  =  options.dbuser,
+                                    passwd  =  options.dbpasswd)
+  conn = connection.cursor()
+  try:
+    createdatabase='create database %s' %(options.db)
+    conn.execute (createdatabase)
+    conn.close()
+    connection.commit()
+    connection.close()
+  except  MySQLdb.Error, e:
+    print "Error %d: %s" % (e.args[0], e.args[1])
+    sys.exit (1)
+  try:
+    connection  =  MySQLdb.connect (host  =  options.dbhost,
+                                      port  =  int(options.dbport),
+                                      db  =  options.db,
+                                      user  =  options.dbuser,
+                                      passwd  =  options.dbpasswd)
+    conn=connection.cursor()
+  except MySQLdb.Error, e:
+    print "Error %d: %s" % (e.args[0], e.args[1])
+    sys.exit(1)
+  
+  CreateDBSchema(conn)
+
+
+treeid = GetOrInsertTree(conn, options.tree)
+
+logging.info("Reading tinderbox data...")
+
+chunk = 0
+# add a fudge factor here, since builds can take up to 3 hours to finish,
+# and we can't get the changeset unless we ask for time up to the end of the
+# build
+endtime += 3 * S_IN_H
+timespan += 3 * S_IN_H
+totalchunks = int(ceil(float(timespan) / chunksize))
+
+while curtime < endtime and chunk < totalchunks:
+  chunk += 1
+  logging.info("Chunk %d/%d" % (chunk, totalchunks))
+
+  if (endtime - curtime) < chunksize:
+    chunksize = endtime - curtime
+
+  tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1" \
+              % {'tree': options.tree,
+                 'maxdate': curtime + chunksize, # tbox wants the end time
+                 'hours': int(chunksize / S_IN_H)}
+  u = urllib.urlopen(tboxurl)
+  tboxjson = u.read()
+  #tboxjson = tboxjson.encode('utf-8').decode('string_escape').decode('utf-8')
+  #tboxjson = ''.join(u.readlines())
+  u.close()
+  
+  tboxjson = fix_tbox_json(tboxjson)
+  try:
+    tboxdata = json.loads(tboxjson)
+  except Exception, inst:
+    print >>sys.stderr, "Error parsing JSON: %s" % inst
+    continue
+
+  # we only care about unit test boxes
+  unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("test|xpc|check", x)]
+  # read build table
+  # 'TestFailed' expected log format is "result | test | optional text".
+  testfailedRe = re.compile(r"(TEST-UNEXPECTED-.*) \| (.*) \|(.*)")
+  for timerow in tboxdata['build_table']:
+    for index in unittest_indices:
+      if index >= len(timerow) or timerow[index] == -1:
+        continue
+      build = timerow[index]
+      if 'buildname' not in build or \
+         'logfile'   not in build:
+        continue
+
+      status = BuildStatusFromText(build['buildstatus'])
+      # Skip unavailable "builds".
+      if status < BuildStatus.Success:
+        continue
+
+      name = build['buildname']
+      os = OSFromBuilderName(name)
+      starttime = int(build['buildtime'])
+      # skip builds we've already seen
+      if HaveBuild(conn, treeid, os, starttime):
+        logging.info("Skipping already seen build '%s' at %d (%s)" % (name, starttime, ctime(starttime)))
+
+        # Call 'UpdateLogfile()' anyway.
+        UpdateLogfile(conn, treeid, os, starttime, build['logfile'])
+        continue
+
+      # must have scrape data for changeset
+      if build['logfile'] not in tboxdata['scrape']:
+        continue
+      changeset = FindChangesetInScrape(tboxdata['scrape'][build['logfile']])
+      if changeset is None:
+        continue
+
+      buildid = InsertBuild(conn, treeid, os, starttime, status, build['logfile'], changeset)
+
+      # 'Success' is fine as is.
+      if status == BuildStatus.Success:
+        pass
+
+      # Parse log to save 'TestFailed' results.
+      elif status == BuildStatus.TestFailed :
+        logging.info("Checking build log for '%s' at %d (%s)" % (name, starttime, ctime(starttime)))
+        try:
+          # Grab the build log.
+          log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile']))
+          gz = GzipFile(log)
+          # Look for test failures.
+          for line in gz:
+            m = testfailedRe.match(line)
+            if m:
+              test = rawtest = m.group(2).strip() or "[unittest-log.py: no logged test]"
+              if rawtest.find('\\') != -1:
+                test = rawtest.replace('\\','/')
+                
+              if test.find('/') != -1:
+                tup=test.partition('build/')
+                if len(tup[2]) > 2:
+                  test=tup[2]
+                else :
+                  test=tup[0]
+                
+              text = m.group(3).strip() or "[unittest-log.py: no logged text]"
+              testnames_id=GetOrInsertTest(conn,test)
+              InsertTest(conn, buildid, m.group(1).rstrip(), testnames_id, text)
+        except:
+          logging.error("Unexpected error: %s" % sys.exc_info()[0])
+          #XXX: handle me?
+
+      # Ignore 'Burning' builds: tests may have run nontheless, but it's safer to discard them :-|
+      elif status == BuildStatus.Burning:
+        continue
+      
+      # Ignore 'Exception' builds: should only be worse than 'Burning'.
+      # (Don't know much at time of writing, since this feature is not active yet: see bug 476656 and follow-ups.)
+      elif status == BuildStatus.Exception:
+        continue
+
+      # Save 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does.
+      elif status == BuildStatus.Unknown:
+        # Add a fake test failure.
+        InsertTest(conn, buildid, "TEST-UNEXPECTED-FAIL", "99999999999", "Unknown status = '%s'!" % build['buildstatus'])
+        continue
+
+
+
+  if chunk < totalchunks:
+    sleep(SLEEP_TIME)
+  curtime += chunksize
+  
+conn.close()
+connection.commit()
+connection.close()
+logging.info("Done")