re-written application

topfails/PREREQS

@@ -0,0 +1,6 @@
+At the minimum, you should set the following environment variables
+
+Assuming you are using BASH shell
+
+export dbuser=<your mysql database username>
+export dbpasswd=<your mysql passwd>

topfails/README

@@ -0,0 +1,3 @@
+This is the README for the topfails application.
+
+Please read the 'PREREQS' document which will explain about the environment variables to be set in order to run the django application.

topfails/cleanup.sh

@@ -0,0 +1,2 @@
+rm `find . -name \*.pyc`
+rm `find . -name \*~`

topfails/dbschema.py

@@ -0,0 +1,121 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is TopFails site code.
+#
+# The Initial Developer of the Original Code is
+# Mozilla foundation
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Murali Nandigama <Murali.Nandigama@Gmail.COM>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+#
+# DB schema maintenance functions.
+#
+
+import logging
+
+__all__ = \
+ [
+  "CreateDBSchema"
+ ]
+
+def CreateDBSchema(conn):
+  logging.info("Executing CreateDBSchema()")
+ 
+  
+  conn.execute("""
+  CREATE TABLE viewer_tree (
+      id integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+      name varchar(45) NOT NULL
+      )
+  """)
+  
+  conn.execute("""
+  CREATE TABLE viewer_build (
+      id integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+      os integer NOT NULL,
+      tree_id integer NOT NULL,
+      starttime integer,
+      status integer NOT NULL,
+      changeset varchar(80) NOT NULL,
+      logfile varchar(300) NOT NULL
+      )
+  """)
+ 
+  conn.execute("""
+  ALTER TABLE viewer_build 
+  ADD CONSTRAINT tree_id_refs_id_11e44bee 
+  FOREIGN KEY (tree_id) 
+  REFERENCES viewer_tree (id)
+  """)
+
+  conn.execute("""
+  CREATE TABLE viewer_test (
+      id integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+      name varchar(300) NOT NULL
+      )
+  """)
+ 
+  conn.execute("""
+  CREATE TABLE viewer_testfailure (
+      id integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+      build_id integer NOT NULL,
+      test_id integer NOT NULL,
+      failtext varchar(400) NOT NULL
+      )
+  """)
+  
+  conn.execute("""
+  ALTER TABLE viewer_testfailure 
+  ADD CONSTRAINT test_id_refs_id_1cc1b9e6 
+  FOREIGN KEY (test_id) 
+  REFERENCES viewer_test (id)
+  """)
+  
+  conn.execute("""
+  ALTER TABLE viewer_testfailure 
+  ADD CONSTRAINT build_id_refs_id_112c09cb 
+  FOREIGN KEY (build_id) 
+  REFERENCES viewer_build (id)
+  """)
+  
+  conn.execute("""
+  CREATE INDEX viewer_build_tree_id ON viewer_build (tree_id)
+  """)
+  
+  conn.execute("""
+  CREATE INDEX viewer_testfailure_build_id ON viewer_testfailure (build_id)
+  """)
+  
+  conn.execute("""
+  CREATE INDEX viewer_testfailure_test_id ON viewer_testfailure (test_id)
+  """)
+
+

topfails/topfails/manage.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+from django.core.management import execute_manager
+try:
+    import settings # Assumed to be in the same directory.
+except ImportError:
+    import sys
+    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
+    sys.exit(1)
+
+if __name__ == "__main__":
+    execute_manager(settings)

topfails/topfails/settings.py

@@ -0,0 +1,86 @@
+# Django settings for topfails project.
+import os
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+DBUSER = DBPASSWD = ''
+if os.environ.has_key('dbuser'):
+   DBUSER=os.environ.get('dbuser')
+   #print DBUSER
+if os.environ.has_key('dbpasswd'):
+    DBPASSWD=os.environ.get('dbpasswd')
+    #print DBPASSWD
+ADMINS = (
+     ('Murali Nandigama', 'murali.nandigama@gmail.com'),
+)
+
+MANAGERS = ADMINS
+
+DATABASE_ENGINE = 'mysql'           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+DATABASE_NAME = 'topfails'             # Or path to database file if using sqlite3.
+DATABASE_USER = DBUSER            # Not used with sqlite3.
+DATABASE_PASSWORD = DBPASSWD         # Not used with sqlite3.
+DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
+DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'America/Chicago'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# Absolute path to the directory that holds media.
+# Example: "/home/media/media.lawrence.com/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+# Examples: "http://media.lawrence.com", "http://example.com/media/"
+MEDIA_URL = ''
+
+# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
+# trailing slash.
+# Examples: "http://foo.com/media/", "/media/".
+ADMIN_MEDIA_PREFIX = '/media/'
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = '^_5(02(8sng5+6+x=c$w+3(4xv+x^&b)2ty4y8m01(o7vz$o5_'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.load_template_source',
+    'django.template.loaders.app_directories.load_template_source',
+#     'django.template.loaders.eggs.load_template_source',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+)
+
+ROOT_URLCONF = 'topfails.urls'
+
+TEMPLATE_DIRS = (
+    './templates/'
+)
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'topfails.viewer'
+)

topfails/topfails/templates/base.html

@@ -0,0 +1,114 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<title>Topfails</title>
+<meta http-equiv="content-type" content="text/html; charset=iso-8859-1">
+
+
+<style type="text/css">
+/*----------Text Styles----------*/
+.ws6 {font-size: 8px;}
+.ws7 {font-size: 9.3px;}
+.ws8 {font-size: 11px;}
+.ws9 {font-size: 12px;}
+.ws10 {font-size: 13px;}
+.ws11 {font-size: 15px;}
+.ws12 {font-size: 16px;}
+.ws14 {font-size: 19px;}
+.ws16 {font-size: 21px;}
+.ws18 {font-size: 24px;}
+.ws20 {font-size: 27px;}
+.ws22 {font-size: 29px;}
+.ws24 {font-size: 32px;}
+.ws26 {font-size: 35px;}
+.ws28 {font-size: 37px;}
+.ws36 {font-size: 48px;}
+.ws48 {font-size: 64px;}
+.ws72 {font-size: 96px;}
+.wpmd {font-size: 13px;font-family: 'Arial';font-style: normal;font-weight: normal;}
+/*----------Para Styles----------*/
+DIV,UL,OL /* Left */
+{
+ margin-top: 0px;
+ margin-bottom: 0px;
+}
+</style>
+
+
+<script language="javascript" type="text/javascript">
+<!--
+function SwapImgRestore() {
+  var i,x,a=document.MM_sr; for(i=0;a&&i<a.length&&(x=a[i])&&x.oSrc;i++) x.src=x.oSrc;
+}
+
+function PreloadImages() {
+  var d=document; if(d.images){ if(!d.MM_p) d.MM_p=new Array();
+    var i,j=d.MM_p.length,a=PreloadImages.arguments; for(i=0; i<a.length; i++)
+    if (a[i].indexOf("#")!=0){ d.MM_p[j]=new Image; d.MM_p[j++].src=a[i];}}
+}
+
+function FindObject(n, d) {
+  var p,i,x;  if(!d) d=document; if((p=n.indexOf("?"))>0&&parent.frames.length) {
+    d=parent.frames[n.substring(p+1)].document; n=n.substring(0,p);}
+  if(!(x=d[n])&&d.all) x=d.all[n]; for (i=0;!x&&i<d.forms.length;i++) x=d.forms[i][n];
+  for(i=0;!x&&d.layers&&i<d.layers.length;i++) x=FindObject(n,d.layers[i].document);
+  if(!x && d.getElementById) x=d.getElementById(n); return x;
+}
+
+function SwapImage() {
+  var i,j=0,x,a=SwapImage.arguments; document.MM_sr=new Array; for(i=0;i<(a.length-2);i+=3)
+   if ((x=FindObject(a[i]))!=null){document.MM_sr[j++]=x; if(!x.oSrc) x.oSrc=x.src; x.src=a[i+2];}
+}
+
+//-->
+</script>
+<script type="text/javascript">
+<!--
+function MyTip(arg)
+{
+    switch(arg)
+    {
+    case "trunk":
+        alert("Press this button to get the Tinderbox Topfails for Firefox Trunk");
+    case "3.6":
+        alert("Press this button to get the Tinderbox Topfails for Firefox 3.6");
+    case "thunderbird":
+        alert("Press this button to get the Tinderbox Topfails for Thunderbird Trunk");
+    case "sea":
+        alert("Press this button to get the Tinderbox Topfails for SeaMonkey");
+    }
+}
+
+//-->
+</script>
+
+
+</head>
+
+<body>
+
+<div id="nav10d" style="position:absolute; left:3px; top:102px; z-index:0"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav10','','http://people.mozilla.com/~mnandigama/images/nav11363110a.gif',1)" href="{% url latest tree=tree %}" <img name="nav10" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav11363110a.gif')" alt="Latest Failures" border=0 src="http://people.mozilla.com/~mnandigama/images/nav11363110i.gif"></a></div>
+<div id="nav11d" style="position:absolute; left:3px; top:135px; z-index:0"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav11','','http://people.mozilla.com/~mnandigama/images/nav11363111a.gif',1)" href="{% url topfails tree=tree %}"><img name="nav11" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav11363111a.gif')" alt="Top 25 Failures" border=0 src="http://people.mozilla.com/~mnandigama/images/nav11363111i.gif"></a></div>
+<div id="nav12d" style="position:absolute; left:3px; top:168px; z-index:0"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav12','','http://people.mozilla.com/~mnandigama/images/nav11363112a.gif',1)" href="{% url tests tree=tree %}"><img name="nav12" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav11363112a.gif')" alt="All Failed Tests" border=0 src="http://people.mozilla.com/~mnandigama/images/nav11363112i.gif"></a></div>
+<div id="nav13d" style="position:absolute; left:3px; top:201px; z-index:0"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav13','','http://people.mozilla.com/~mnandigama/images/nav11363113a.gif',1)" href="{% url failswindow tree=tree %}?window=7d"<img name="nav13" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav11363113a.gif')" alt="Failures from last X days" border=0 src="http://people.mozilla.com/~mnandigama/images/nav11363113i.gif"></a></div>
+<div id="nav14d" style="position:absolute; left:3px; top:234px; z-index:0"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav14','','http://people.mozilla.com/~mnandigama/images/nav11363114a.gif',1)" href="{% url Help tree=tree %}"><img name="nav14" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav11363114a.gif')" alt="Help" border=0 src="http://people.mozilla.com/~mnandigama/images/nav11363114i.gif"></a></div>
+
+<div id="image1" style="position:absolute; overflow:hidden; left:3px; top:1px; width:179px; height:68px; z-index:1"><img src="http://people.mozilla.com/~mnandigama/images/logo-wordmark.png" alt="Firefox Logo" title="http://www.mozilla.com" border=0 width=179 height=68></div>
+<div id="text1" style="position:absolute; overflow:hidden; left:351px; top:2px; width:491px; height:41px; z-index:3">
+<div class="wpmd">
+<div><font color="#CC4200" face="Tahoma" class="ws24"><B>Mozilla Tinderbox Topfails Dashboard</B></font></div>
+</div></div>
+
+<div id="nav20d" style="position:absolute; left:311px; top:45px; z-index:2"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav20','','http://people.mozilla.com/~mnandigama/images/nav14052410a.gif',1)" title="Click for Firefox Trunk Topfails Dashboard" href="{% url latest tree='Firefox' %}" ><img name="nav20" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav14052410a.gif')" alt="Firefox Trunk" border=0 src="http://people.mozilla.com/~mnandigama/images/nav14052410i.gif"></a></div>
+<div id="nav21d" style="position:absolute; left:439px; top:45px; z-index:2"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav21','','http://people.mozilla.com/~mnandigama/images/nav14052411a.gif',1)" href="{% url latest tree='Firefox3.6' %}"><img name="nav21" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav14052411a.gif')" title="Click for Firefox 3.6 Topfails Dashboard" alt="Firefox3.6" border=0 src="http://people.mozilla.com/~mnandigama/images/nav14052411i.gif"></a></div>
+<div id="nav22d" style="position:absolute; left:567px; top:45px; z-index:2"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav22','','http://people.mozilla.com/~mnandigama/images/nav14052412a.gif',1)"  title="Click for Thunderbird Trunk Topfails Dashboard" href="{% url latest tree='Thunderbird' %}"><img name="nav22" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav14052412a.gif')" alt="Thunderbird" border=0 src="http://people.mozilla.com/~mnandigama/images/nav14052412i.gif"></a></div>
+<div id="nav23d" style="position:absolute; left:695px; top:45px; z-index:2"><a onMouseOut="SwapImgRestore()" onMouseOver="SwapImage('nav23','','http://people.mozilla.com/~mnandigama/images/nav14052413a.gif',1)"  title="Click for SeaMonkey  Topfails Dashboard" href="{% url latest tree='SeaMonkey' %}"><img name="nav23" onLoad="PreloadImages('http://people.mozilla.com/~mnandigama/images/nav14052413a.gif')" alt="SeaMonkey" border=0 src="http://people.mozilla.com/~mnandigama/images/nav14052413i.gif"></a></div>
+
+
+
+<div id="text2" style="position:absolute; overflow:hidden; left:190px; top:102px; width:800px;  z-index:4">
+{% block content %}
+{% endblock %}
+</div>
+</body>
+</html>

topfails/topfails/templates/viewer/Help.html

@@ -0,0 +1,21 @@
+{% extends "base.html" %}
+{% block content %}
+<div><font face="Calibri" class="ws11">This web site provides the bird eye view of top failures and other details of Firefox unit tests&nbsp; from the&nbsp; Tinderbox daily runs.</font></div>
+<div><font face="Calibri" class="ws11"><BR></font></div>
+<div><font face="Calibri" class="ws11">Meaning of the Left navigation links</font></div>
+<UL>
+<li><font face="Calibri" class="ws11">Latest Failures : This link provides the list of latest failures from the Tinderbox.</font></li>
+<li><font face="Calibri" class="ws11">Top 25 Failures : This link provides the list of top 25 failures from the entire history of the topfails database. Please note that this data is as complete as the first date we started collecting results. The data we have in the topfails database does not contain all the failures details from the Time ZERO of tinderbox runs.</font></li>
+<li><font face="Calibri" class="ws11">All Failed Tests: This link displays all the known failures that are stored in the topfails DB.</font></li>
+<li><font face="Calibri" class="ws11"><b>Failures from X days</b>: This is a link which allows you to query for failures from NOW till a specific period in the past.</font></li>
+</UL>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Example:   2d&nbsp; =&nbsp;&nbsp; 2 days&nbsp; in the past till&nbsp; now </font></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 7h&nbsp; =&nbsp;&nbsp; 7 hours in the past till now</font></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 5w =&nbsp; 5 weeks in the past till now</font></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1m =&nbsp; 1 month in the past till now</font></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 10y =&nbsp; 10 years in the past till now</font></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">Example queries for advanced users <a href="../failswindow/Firefox?window=2w">2 weeks 'Firefox' failures</a></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">Example queries for advanced users <a href="../failswindow/Firefox?window=2d">2 days 'Firefox' failures</a></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">Example queries for advanced users <a href="../failswindow/Firefox?window=24h">24 hours 'Firefox' failures</a></div>
+<div style="margin-left:40px;"><font face="Calibri" class="ws11">Example queries for advanced users <a href="../failswindow/Firefox?window=1m">1 month 'Firefox' failures</a></div>
+{% endblock %}

topfails/topfails/templates/viewer/README

@@ -0,0 +1,5 @@
+I have done a very bad and crude hack to add a top level navigation page. Instead of storing images for top navigation page in the correct location , I am jammin gall the top site nav stuff in here.
+
+Should fix it when time permits.
+
+

topfails/topfails/templates/viewer/changeset.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block content %}
+<h3>Changeset {{ changeset }}</h3>
+<ul>
+{% for build in builds %}
+  <li>{{ build.tree }}: {{ build.get_os_display }}: {{ build.get_status_display }}</li>
+{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/templates/viewer/changesets.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block content %}
+<h3>Changesets</h3>
+<ul>
+{% for c in changesets %}
+  <li><a href="{% url viewer.views.changeset c %}">{{ c }}</a></li>
+{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/templates/viewer/failswindow.html

@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{% block content %}
+<h3>Failed tests from last {{n}} {{d}} up to now</h3>
+<table>
+<tr><th>Count</th><th align="left">Test name</th></tr>
+{% for f in failures %}
+  <tr><td>{{ f.count }}</td><td><a href="{% url viewer.views.test tree=tree %}?name={{ f.test__name }}">{{ f.test__name }}</a></td></tr>
+{% endfor %}
+</table>
+{% endblock %}

topfails/topfails/templates/viewer/index.html

@@ -0,0 +1,15 @@
+{% extends "base.html" %}
+{% block content %}
+<style type="text/css">
+  li {
+    padding-bottom: 10px;  }
+  </style>
+<h3>Most recent test failures</h3>
+<ul>
+{% for f in failures %}
+  <li>{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree.name }} {{ f.build.get_os_display }}: <a href="{% url viewer.views.test %}?name={{ f.name }}">{{ f.name }}</a>,
+  <a href="{% url viewer.views.timeline %}?name={{ f.name }}">timeline</a>
+   - {{ f.description }}</li>
+{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/templates/viewer/latest.html

@@ -0,0 +1,15 @@
+{% extends "base.html" %}
+{% block content %}
+<style type="text/css">
+  li {
+    padding-bottom: 10px;  }
+  </style>
+<h3>Most recent test failures</h3>
+<ul>
+{% for f in failures %}
+  <li>{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree.name }} {{ f.build.get_os_display }}: <a href="{% url viewer.views.test tree=tree %}?name={{ f.test.name }}">{{ f.test.name }}</a>
+  <!--<a href="{% url viewer.views.timeline tree=tree %}?name={{ f.test.name }}">timeline</a> -->
+   - {{ f.failtext }}</li>
+{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/templates/viewer/test.html

@@ -0,0 +1,11 @@
+{% extends "base.html" %}
+{% block content %}
+<h3>Logfile links and Changeset links for {{ test }} failures</h3>
+<ul>
+  <li style="display:none"><ul>
+{% for f in failures %}{% ifchanged f.build.id %}</ul></li><li>
+<a href="{{ f.build.tinderboxlink|safe }}">{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree }} {{ f.build.get_os_display }}</a> [{{ f.build.changesetlink|safe }}]:
+  <ul>{% endifchanged %}
+  <!--<li>{{ f.description }}</li>-->{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/templates/viewer/test.html.orig

@@ -0,0 +1,27 @@
+{% extends "base.html" %}
+{% block content %}
+<h3>Logfile links and Changeset links for {{ test }} failures</h3>
+<ul>
+  <li style="display:none"><ul>
+{% for f in section %}{% ifchanged f.build.id %}</ul></li><li>
+<a href="{{ f.build.tinderboxlink|safe }}">{{ f.build.startdate|date:"Y-m-d H:i" }} {{ f.build.tree }} {{ f.build.get_os_display }}</a> [{{ f.build.changesetlink|safe }}]:
+  <ul>{% endifchanged %}
+  <!--<li>{{ f.description }}</li>-->{% endfor %}
+</ul>
+{% endblock %}
+
+<div class="pagination">
+    <span class="step-links">
+        {% if section.has_previous %}
+            <a href="?page={{ section.previous_page_number }}">previous</a>
+        {% endif %}
+
+        <span class="current">
+            Page {{ section.number }} of {{ section.paginator.num_pages }}.
+        </span>
+
+        {% if section.has_next %}
+            <a href="?page={{ section.next_page_number }}">next</a>
+        {% endif %}
+    </span>
+</div>

topfails/topfails/templates/viewer/tests.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block content %}
+<style type="text/css">
+  li {
+    padding-bottom: 10px;  }
+  </style>
+
+<h3>All known failing tests</h3>
+<ul>
+{% for t in tests %}
+  <li><a href="{% url viewer.views.test tree=tree %}?name={{ t }}">{{ t }}</a></li>
+{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/templates/viewer/timeline.html

@@ -0,0 +1,72 @@
+<html>
+<head>
+<title>Timeline for {{ test }}</title>
+<script src="http://static.simile.mit.edu/exhibit/api-2.0/exhibit-api.js?autoCreate=false"
+        type="text/javascript"></script>
+<script src="http://static.simile.mit.edu/exhibit/extensions-2.0/time/time-extension.js"></script>
+
+<script type="application/javascript">
+var json = {"items": [
+{% for bd in builds %}
+  {
+    "type": "Build",
+    "id": "bld{{ bd.build.id }}",
+    "label": "{{ bd.os }} {{ bd.build.changeset }}",
+    "desc": "{{ bd.description|escapejs|escape }}",
+    "desc_id": "{{ bd.desctype }}",
+    "time": "{{ bd.time }}",
+    "os": "{{ bd.os }}",
+    "rev": "{{ bd.build.changeset }}",
+  },
+{% endfor %}
+  ],
+"properties": {
+  "time": {"valueType": "date"},
+  }
+};
+
+$(document).ready(function() {
+  window.database = Exhibit.Database.create();
+  window.database.loadData(json);
+  window.exhibit = Exhibit.create();
+  window.exhibit.configureFromDOM();
+});
+
+
+</script>
+</head>
+
+<body>
+<h2>Timeline for {{ test }}</h2>
+
+<div ex:role="lens" ex:itemTypes="Build">
+  <div><span ex:content=".os"></span> <a
+  ex:href-subcontent="http://hg.mozilla.org/mozilla-central/rev/&#123;{.rev}&#125;"
+  target="_blank" ex:content=".rev"></a></div> <pre ex:content=".desc"></pre>
+</div>
+<div id="blds" ex:role="exhibit-collection" ex:itemTypes="Build"></div>
+
+<div ex:role="view"
+     ex:viewClass="Timeline" 
+     ex:start=".time"
+     ex:topBandPixelsPerUnit="50",
+     ex:bottomBandPixelsPerUnit="50",
+></div>
+<table border="0">
+  <tr>
+    <td>
+{% for desc in descriptions %}
+<pre>{{desc}}</pre>
+{% endfor %}
+    </td>
+    <td valign="top" style="padding-left: 5em;">
+      <div ex:role="facet" ex:expression=".desc_id"
+          ex:facetLabel="Description"></div>
+    </td>
+    <td valign="top" style="padding-left: 5em;">
+      <div ex:role="facet" ex:expression=".os"
+          ex:facetLabel="OS"></div>
+    </td>
+  </tr>
+</table>
+</body> </html>

topfails/topfails/templates/viewer/topfails.html

@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{% block content %}
+<h3>Top 25 failing tests</h3>
+<table>
+<tr><th>Count</th><th align="left">Test name</th></tr>
+{% for f in failures %}
+  <tr><td>{{ f.count }}</td><td><a href="{% url  viewer.views.test tree=tree %}?name={{ f.test__name }}">{{ f.test__name}}</a></td></tr>
+{% endfor %}
+</table>
+{% endblock %}

topfails/topfails/templates/viewer/tree.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block content %}
+<h3>{{ tree }}</h3>
+<ul>
+{% for build in newestbuilds %}
+  <li>{{ build.get_os_display }}: {{ build.get_status_display }}</li>
+{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/templates/viewer/trees.html

@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{% block content %}
+
+<h3>Trees</h3>
+<ul>
+{% for tree in trees %}
+  <li><a href="{% url viewer.views.tree tree.name %}">{{ tree.name }}</a></li>
+{% endfor %}
+</ul>
+{% endblock %}

topfails/topfails/urls.py

@@ -0,0 +1,20 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+urlpatterns = patterns('topfails.viewer.views',
+    (r'^$', 'index'),
+    url(r'^trees/(?P<tree>.+)?$','trees', name='trees'),
+    url(r'^tree/(?P<tree>.+)?$','tree', name='tree'),
+    url(r'^changesets/(?P<tree>.+)?$','changesets', name='changesets'),
+    url(r'^changesets/(?P<tree>.+)/(?P<changeset>[a-f0-9]+)$', 'changeset', name='changeset'),
+    url(r'^tests/(?P<tree>.+)?$','tests', name='tests'),
+    url(r'^test/(?P<tree>.+)?$','test', name='test'),
+    url(r'^timeline/(?P<tree>.+)?$','timeline', name='timeline'),
+    url(r'^topfails/(?P<tree>.+)?$','topfails', name='topfails'),
+    url(r'^failswindow/(?P<tree>.+)?$','failswindow', name='failswindow'),
+    url(r'^latest/(?P<tree>.+)?$','latest', name='latest'),
+    url(r'^Help/(?P<tree>.+)?$','Help', name='Help'),
+)

topfails/topfails/viewer/models.py

@@ -0,0 +1,130 @@
+import re
+from django.db import models, connection
+from datetime import datetime
+from time import ctime, sleep, time
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+
+class Tree(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=45, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+    
+class Build(models.Model):
+    id = models.AutoField(primary_key=True)
+    os = models.IntegerField(choices=OS_CHOICES)
+    tree = models.ForeignKey(Tree)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    changeset = models.CharField(max_length=80,blank=True)
+    logfile = models.CharField(max_length=300,blank=True)
+    
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    
+    def changesetlink(self):
+      if str(self.tree)  == 'Firefox':
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+      elif str(self.tree) == 'Firefox3.6':
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/releases/mozilla-1.9.2", self.changeset, self.changeset)
+      elif str(self.tree) == 'Thunderbird' or str(self.tree) == 'SeaMonkey':  
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/comm-central", self.changeset, self.changeset)
+      else :
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset) 
+    
+    def jsonchangesetlink(self):
+      return "%s/rev/%s" %  ("http://hg.mozilla.org/mozilla-central", self.changeset)
+    
+    def tinderboxlink(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    
+
+class Test(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=300, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+   
+class TestFailure(models.Model):
+    id = models.AutoField(primary_key=True)
+    build = models.ForeignKey(Build)
+    test = models.ForeignKey(Test)
+    failtext = models.CharField(max_length=400, blank=True)
+    
+    
+    
+def get_most_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')[:25]
+
+def get_time_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')
+
+#This def is never used not intended to be used. I kept it here as a refresher
+def get_latest_fails(tree):
+     cursor = connection.cursor()
+     statement="""SELECT viewer_testfailure.id, viewer_testfailure.build_id, viewer_test.name, viewer_testfailure.failtext
+     FROM viewer_testfailure, viewer_test, viewer_build, viewer_tree
+     WHERE (viewer_testfailure.build_id = viewer_build.id)
+     AND (viewer_build.tree_id = viewer_tree.id)
+     AND ( viewer_testfailure.test_id = viewer_test.id)
+     AND viewer_tree.name ='"""+tree+"""' ORDER BY viewer_build.starttime DESC LIMIT 20"""
+     cursor.execute(statement)
+     for row in cursor:
+       yield row
+       
+       
+def get_fails_in_timerange(self,tree):
+  
+  # Get current time, in seconds.
+  endtime = int(time())
+  
+  #print endtime
+  m = re.match("(\d+)([ymwdh])", self)
+  #print m.group(1), m.group(2)
+  if m is None:
+    print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+    sys.exit(1)
+  
+  timespan = int(m.group(1)) * {'y': 365 * 24 * 3600,
+                                'm':  30 * 24 * 3600,
+                                'w':   7 * 24 * 3600,
+                                'd':       24 * 3600,
+                                'h':            3600}[m.group(2)]
+  # Set current time to beginning of requested timespan ending now.
+  curtime = endtime - timespan
+  qs = get_time_failing_tests(tree)
+  return qs.filter(build__starttime__gt=curtime)
+
+    

topfails/topfails/viewer/models.py~

@@ -0,0 +1,129 @@
+import re
+from django.db import models, connection
+from datetime import datetime
+from time import ctime, sleep, time
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+OS_CHOICES = (
+    (OS.Windows, 'Windows'),
+    (OS.Mac, 'Mac'),
+    (OS.Linux, 'Linux'),
+    (OS.Unknown, 'Unknown')
+)
+
+class BuildStatus():
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+BUILDSTATUS_CHOICES = (
+    (BuildStatus.Success, 'Success'),
+    (BuildStatus.TestFailed, 'Test Failed'),
+    (BuildStatus.Burning, 'Burning'),
+    (BuildStatus.Exception, 'Exception'),
+    (BuildStatus.Unknown, 'Unknown')
+)
+
+
+class Tree(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=45, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+    
+class Build(models.Model):
+    id = models.AutoField(primary_key=True)
+    os = models.IntegerField(choices=OS_CHOICES)
+    tree = models.ForeignKey(Tree)
+    starttime = models.IntegerField(null=True, blank=True)
+    status = models.IntegerField(choices=BUILDSTATUS_CHOICES)
+    changeset = models.CharField(max_length=80,blank=True)
+    logfile = models.CharField(max_length=300,blank=True)
+    
+    def startdate(self):
+        return datetime.fromtimestamp(self.starttime)
+    
+    def changesetlink(self):
+      if str(self.tree)  == 'Firefox':
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset)
+      elif str(self.tree) == 'Firefox3.6':
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/releases/mozilla-1.9.2", self.changeset, self.changeset)
+      elif str(self.tree) == 'Thunderbird' or str(self.tree) == 'SeaMonkey':  
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/comm-central", self.changeset, self.changeset)
+      else :
+        return '<a href="%s/rev/%s">%s</a>' % ("http://hg.mozilla.org/mozilla-central", self.changeset, self.changeset) 
+    
+    def jsonchangesetlink(self):
+      return "%s/rev/%s" %  ("http://hg.mozilla.org/mozilla-central", self.changeset)
+    
+    def tinderboxlink(self):
+      if self.logfile:
+        return "http://tinderbox.mozilla.org/showlog.cgi?log=%s/%s" % (self.tree.name, self.logfile)
+      return "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%s&maxdate=%d&hours=3" % (self.tree.name, self.starttime)
+    
+
+class Test(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=300, blank=True)
+    
+    def __unicode__(self):
+      return self.name
+   
+class TestFailure(models.Model):
+    id = models.AutoField(primary_key=True)
+    build = models.ForeignKey(Build)
+    test = models.ForeignKey(Test)
+    failtext = models.CharField(max_length=400, blank=True)
+    
+    
+    
+def get_most_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')[:25]
+
+def get_time_failing_tests(tree):
+    return TestFailure.objects.filter(build__tree__name=tree).values('test__name').annotate(count=models.Count('test__name')).order_by('-count')
+
+def get_latest_fails(tree):
+     cursor = connection.cursor()
+     statement="""SELECT viewer_testfailure.id, viewer_testfailure.build_id, viewer_test.name, viewer_testfailure.failtext
+     FROM viewer_testfailure, viewer_test, viewer_build, viewer_tree
+     WHERE (viewer_testfailure.build_id = viewer_build.id)
+     AND (viewer_build.tree_id = viewer_tree.id)
+     AND ( viewer_testfailure.test_id = viewer_test.id)
+     AND viewer_tree.name ='"""+tree+"""' ORDER BY viewer_build.starttime DESC LIMIT 20"""
+     cursor.execute(statement)
+     for row in cursor:
+       yield row
+       
+       
+def get_fails_in_timerange(self,tree):
+  
+  # Get current time, in seconds.
+  endtime = int(time())
+  
+  #print endtime
+  m = re.match("(\d+)([ymwdh])", self)
+  #print m.group(1), m.group(2)
+  if m is None:
+    print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+    sys.exit(1)
+  
+  timespan = int(m.group(1)) * {'y': 365 * 24 * 3600,
+                                'm':  30 * 24 * 3600,
+                                'w':   7 * 24 * 3600,
+                                'd':       24 * 3600,
+                                'h':            3600}[m.group(2)]
+  # Set current time to beginning of requested timespan ending now.
+  curtime = endtime - timespan
+  qs = get_time_failing_tests(tree)
+  return qs.filter(build__starttime__gt=curtime)
+
+    

topfails/topfails/viewer/views.py

@@ -0,0 +1,107 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from topfails.viewer.models import Build, Tree, Test,TestFailure, OS_CHOICES, get_most_failing_tests, get_fails_in_timerange
+import re
+from django.http import HttpResponse
+import json
+
+def latest(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  if request.GET.has_key('json'):
+    jtext = [{"Test_name":f.test.name, "Build_status":f.build.status, "Logfile": f.build.tinderboxlink(),"Changeset":f.build.jsonchangesetlink() , "Failure_description":f.failtext} for f in failures]
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+def index(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+
+def trees(request,tree='Firefox'):
+  alltrees = Tree.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees , 'tree' : tree})
+
+def tree(request, tree='Firefox'):
+  newestbuilds = get_list_or_404(Build.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request,tree='Firefox'):
+  build_csets = Build.objects.filter(tree__name__exact=tree).values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', { 'tree' : tree,'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset,tree='Firefox'):
+  builds = get_list_or_404(Build, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds, 'tree' : tree})
+
+def tests(request,tree='Firefox'):
+    test_names = TestFailure.objects.filter(build__tree__name__exact=tree).values('test__name').distinct()
+    if request.GET.has_key('json'):
+      jtext = list(test_names)
+      return HttpResponse(json.dumps(jtext))
+    else:
+      return render_to_response('viewer/tests.html', { 'tree' : tree, 'tests': [t['test__name'] for t in test_names]})
+
+def test(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name__exact=tree).filter(test__name__exact=request.GET['name']).order_by('-build__starttime'))
+  #if request.GET.has_key('json'):
+    #jtext = list(failures)
+    #return HttpResponse(json.dumps(jtext))
+  #else:
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': failures, 'tree' : tree})
+
+def topfails(request,tree='Firefox'):
+  failures = get_most_failing_tests(tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/topfails.html', {'failures': failures, 'tree' : tree})
+  
+def Help(request,tree):
+  return render_to_response('viewer/Help.html',{'tree':tree})  
+  
+def timeline(request,tree='Firefox'):
+  name = request.GET['name']
+  builds = get_list_or_404(Build.objects.filter(tree__name__exact=tree), test__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.testfailure_set.filter(testfailure__name__exact=name).order_by('id')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist, 'tree' : tree})
+
+def failswindow(request,tree='Firefox'):
+  period=request.GET['window']
+  m = re.match("(\d+)([ymwdh])", period)
+  failures = get_fails_in_timerange(period,tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    if m.group(2) == 'd':
+      prd='days'
+    elif m.group(2) == 'h':
+      prd = 'hours'
+    elif m.group(2) == 'w':
+      prd = 'weeks'
+    elif m.group(2) == 'm':
+      prd = 'months'
+    elif m.group(2) == 'y':
+      prd = 'years'
+    else:
+      prd = 'days'
+    
+      
+    return render_to_response('viewer/failswindow.html', {'failures': failures,'n':m.group(1),'d':prd, 'tree' : tree})
+    

topfails/topfails/viewer/views.py.orig

@@ -0,0 +1,118 @@
+from django.shortcuts import render_to_response, get_list_or_404
+from topfails.viewer.models import Build, Tree, Test,TestFailure, OS_CHOICES, get_most_failing_tests, get_fails_in_timerange
+import re
+from django.http import HttpResponse
+import json
+
+def latest(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  if request.GET.has_key('json'):
+    jtext = [{"Test_name":f.test.name, "Build_status":f.build.status, "Logfile": f.build.tinderboxlink(),"Changeset":f.build.jsonchangesetlink() , "Failure_description":f.failtext} for f in failures]
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+def index(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name=tree).order_by('-build__starttime')[:20])
+  return render_to_response('viewer/latest.html', {'failures': failures, 'tree' : tree})
+
+
+def trees(request,tree='Firefox'):
+  alltrees = Tree.objects.all().order_by('name')
+  return render_to_response('viewer/trees.html', {'trees': alltrees , 'tree' : tree})
+
+def tree(request, tree='Firefox'):
+  newestbuilds = get_list_or_404(Build.objects.filter(tree__name__exact=tree).order_by('-starttime')[:5])
+  return render_to_response('viewer/tree.html', {'tree': tree, 'newestbuilds': newestbuilds})
+
+def changesets(request,tree='Firefox'):
+  build_csets = Build.objects.filter(tree__name__exact=tree).values('changeset').distinct()
+  return render_to_response('viewer/changesets.html', { 'tree' : tree,'changesets': [b['changeset'] for b in build_csets]})
+
+def changeset(request, changeset,tree='Firefox'):
+  builds = get_list_or_404(Build, changeset__exact=changeset)
+  return render_to_response('viewer/changeset.html', {'changeset': changeset, 'builds': builds, 'tree' : tree})
+
+def tests(request,tree='Firefox'):
+    test_names = TestFailure.objects.filter(build__tree__name__exact=tree).values('test__name').distinct()
+    if request.GET.has_key('json'):
+      jtext = list(test_names)
+      return HttpResponse(json.dumps(jtext))
+    else:
+      return render_to_response('viewer/tests.html', { 'tree' : tree, 'tests': [t['test__name'] for t in test_names]})
+
+from django.core.paginator import Paginator, InvalidPage, EmptyPage
+def test(request,tree='Firefox'):
+  failures = get_list_or_404(TestFailure.objects.filter(build__tree__name__exact=tree).filter(test__name__exact=request.GET['name']).order_by('-build__starttime'))
+  paginator = Paginator(failures, 25) # Show 25 failures per page
+
+  # Make sure page request is an int. If not, deliver first page.
+  try:
+      page = int(request.GET.get('page', '1'))
+  except ValueError:
+      page = 1
+
+  # If page request (9999) is out of range, deliver last page of results.
+  try:
+     section = paginator.page(page)
+  except (EmptyPage, InvalidPage):
+     section = paginator.page(paginator.num_pages)
+
+  return render_to_response('viewer/test.html', {'test': request.GET['name'], 'failures': section, 'tree' : tree})
+
+def topfails(request,tree='Firefox'):
+  failures = get_most_failing_tests(tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    return render_to_response('viewer/topfails.html', {'failures': failures, 'tree' : tree})
+  
+def Help(request,tree):
+  return render_to_response('viewer/Help.html',{'tree':tree})  
+  
+def timeline(request,tree='Firefox'):
+  name = request.GET['name']
+  builds = get_list_or_404(Build.objects.filter(tree__name__exact=tree), test__name__exact=name)
+  buildlist = []
+  desc_list = []
+  for b in builds:
+    descs = b.testfailure_set.filter(testfailure__name__exact=name).order_by('id')
+    desc = '\n'.join(descs.values_list('description', flat=True))
+    if desc not in desc_list:
+      desc_list.append(desc)
+    desc_i = desc_list.index(desc)
+    buildlist.append({'build': b,
+                      'desctype': desc_i,
+                      'description': desc,
+                      'os': OS_CHOICES[b.os][1],
+                      'time': b.startdate().isoformat() + "Z",
+                      })
+  return render_to_response('viewer/timeline.html', {'test': name,
+                                                     'descriptions': desc_list,
+                                                     'builds': buildlist, 'tree' : tree})
+
+def failswindow(request,tree='Firefox'):
+  period=request.GET['window']
+  m = re.match("(\d+)([ymwdh])", period)
+  failures = get_fails_in_timerange(period,tree)
+  if request.GET.has_key('json'):
+    jtext = list(failures)
+    return HttpResponse(json.dumps(jtext))
+  else:
+    if m.group(2) == 'd':
+      prd='days'
+    elif m.group(2) == 'h':
+      prd = 'hours'
+    elif m.group(2) == 'w':
+      prd = 'weeks'
+    elif m.group(2) == 'm':
+      prd = 'months'
+    elif m.group(2) == 'y':
+      prd = 'years'
+    else:
+      prd = 'days'
+    
+      
+    return render_to_response('viewer/failswindow.html', {'failures': failures,'n':m.group(1),'d':prd, 'tree' : tree})
+    

topfails/unittest-log.py

@@ -0,0 +1,493 @@
+#!/usr/bin/env python
+#Indentation is 2 spaces  ***** DO NOT USE TABS *****
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is TopFails site code.
+#    
+# The Initial Developer of the Original Code is
+# Mozilla foundation
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Serge Gautherie <sgautherie.bz@free.fr>
+#   Ted Mielczarek <ted.mielczarek@gmail.com>.
+#   Murali Nandigama <Murali.Nandigama@Gmail.COM>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import re, os, sys, urllib, logging
+import MySQLdb # Moved from  sqlite3 db  to MySQL
+from time import ctime, sleep, time
+from math import ceil
+from optparse import OptionParser
+from gzip import GzipFile
+import binascii
+
+try:
+  # 'Availability: Unix.'
+  from time import tzset
+except ImportError:
+  print >>sys.stderr, "WARNING: time.tzset() is not available on non-Unixes!"
+
+  # Define a fake function. (for development use only)
+  # ToDo: Investigate Windows/etc situation. (Bug 525699)
+  def tzset():
+    pass
+
+try:
+  import simplejson as json
+except ImportError:
+  try:
+    # 'New in version 2.6.'
+    import json
+  except ImportError:
+    print >>sys.stderr, "ERROR: no simplejson nor json package found!"
+    sys.exit(1)
+
+from dbschema import CreateDBSchema  
+
+# Number of seconds in a hour: 60 mn * 60 s = 3600 s.
+S_IN_H = 3600
+# Download data in 24 hours chunks so as not to overwhelm the tinderbox server.
+chunksize = 24 * S_IN_H
+# seconds between requests
+SLEEP_TIME = 1
+
+class OS():
+  Windows = 0
+  Mac = 1
+  Linux = 2
+  Unknown = 3
+
+class BuildStatus():
+  # Unavailable builds to skip.
+  # Values need (only) to be less than the 'Success' one.
+  NoBuild = -2
+  InProgress = -1
+
+  # Builds to save in the db.
+  # Do not change these values (without updating db data).
+  Success = 0
+  TestFailed = 1
+  Burning = 2
+  Exception = 3
+  Unknown = 4
+
+csetre = re.compile("rev/([0-9A-Za-z]+)")
+def FindChangesetInScrape(scrape):
+  for line in scrape:
+    m = csetre.search(line)
+    if m:
+      return m.group(1)
+  return None
+
+def OSFromBuilderName(name):
+  if name.startswith("Linux") or name.startswith("Rev3 Fedora") or name.startswith("Fedora"):
+    return OS.Linux
+  if name.startswith("MacOSX") or name.startswith("OS X") or name.startswith("Rev3 MacOSX"):
+    return OS.Mac
+  if name.startswith("WINNT"):
+    return OS.Windows
+  return OS.Unknown
+
+buildStatuses = {
+  # "No build in progress".
+  "null":       BuildStatus.NoBuild,
+  # "Build in progress".
+  "building":   BuildStatus.InProgress,
+  # "Successful build".
+  "success":    BuildStatus.Success,
+  # "Successful build, but tests failed".
+  "testfailed": BuildStatus.TestFailed,
+  # "Build failed".
+  "busted":     BuildStatus.Burning,
+  # "Non-build failure". (i.e. "automation failure")
+  "exception":  BuildStatus.Exception,
+}
+def BuildStatusFromText(status):
+  try:
+    return buildStatuses[status]
+  except KeyError:
+    # Log 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does.
+    logging.info("WARNING: unknown status = '%s'!" % status)
+    return BuildStatus.Unknown
+
+def GetOrInsertTree(conn, tree):
+  """Get an id for a tree named |tree|. If it's not already in the trees
+  table, insert a new row and return the id."""
+  
+  conn.execute("""
+               SELECT id FROM viewer_tree WHERE name = %s
+               """, (tree))
+  if conn.rowcount > 0:
+    return conn.fetchone()[0]
+
+  # need to insert it
+  conn.execute("""
+        INSERT INTO viewer_tree (name) VALUES (%s)
+        """, (tree))
+  connection.commit()
+  return conn.lastrowid
+
+def GetOrInsertTest(conn, testname):
+  """Get an id for a test named |testname|. If it's not already in the testnames
+  table, insert a new row and return the id."""
+  
+  conn.execute("""
+               SELECT id FROM viewer_test WHERE name = %s
+               """, (testname))
+  if conn.rowcount > 0:
+    return conn.fetchone()[0]
+
+  # need to insert it
+  conn.execute("""
+               INSERT INTO viewer_test (name) VALUES (%s)
+               """, (testname))
+  connection.commit()
+  return conn.lastrowid
+
+def HaveBuild(conn, treeid, os, starttime):
+  """See if we already have this build in our database."""
+  conn.execute("""
+               SELECT COUNT(*) FROM viewer_build WHERE tree_id = %s AND os = %s AND starttime = %s
+               """, (treeid, os, starttime))
+  return conn.fetchone()[0] == 1
+
+def UpdateLogfile(conn, treeid, os, starttime, logfile):
+  """Update empty 'logfile' for a given build (added in db schema v1)."""
+  conn.execute("""
+               UPDATE viewer_build SET logfile = %s WHERE tree_id = %s AND os = %s AND starttime = %s AND logfile IS NULL
+               """, (logfile, treeid, os, starttime))
+  connection.commit()
+def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset):
+  """Insert a build into the builds table and return the id."""
+  conn.execute("""
+               INSERT INTO viewer_build (tree_id, os, starttime, status, logfile, changeset) VALUES (%s, %s, %s, %s, %s, %s)
+               """, (treeid, os, starttime, status, logfile, changeset))
+  connection.commit()
+  return conn.lastrowid
+
+def InsertTest(conn, buildid, result, testnames_id, description):
+  # ToDo: Add column to save result.
+  conn.execute("""
+               INSERT INTO viewer_testfailure (build_id, test_id, failtext) VALUES (%s, %s, %s)
+               """, (buildid, testnames_id, description))
+  connection.commit()
+def asciirepl(match):
+  # replace the hexadecimal characters with ascii characters
+  s = match.group()  
+  return binascii.unhexlify(s)  
+
+def reformat_content(data):
+  p = re.compile(r'\\x(\w{2})')
+  return p.sub(asciirepl, data)
+  
+  
+def fix_tbox_json(s): 
+  """Fixes up tinderbox json.
+
+  Tinderbox returns strings as single-quoted strings, and occasionally
+  includes the unquoted substring 'undef' (with quotes) in the output, e.g.
+
+  {'key': 'hello 'undef' world'}
+
+  should return a dictionary
+
+  {'key': 'hello \'undef\' world'}
+  """
+
+  json_data = re.sub(r"^tinderbox_data\s*=\s*", "", s)
+  json_data = re.sub(r";$", "", json_data)
+  retval = []
+  in_str = False
+  in_esc = False
+  skip = 0
+  for i,c in enumerate(json_data):
+    # The tinderbox data is a fracked json. and it some times contains
+    # Control characters. that would totally fail the json.loads step.
+    # So, eliminate them .. all of them .. here -- Murali
+    if (c < '\xFD' and c > '\x1F') or c == '\n' or c == '\r' :
+      if skip > 0:
+        skip -= 1
+        continue
+  
+      if in_str:
+        if in_esc:
+          if c == "'":
+            retval.append("'")
+          else:
+            retval.append("\\")
+            retval.append(c)
+          in_esc = False
+        elif c == "\\":
+          in_esc = True
+        elif c == "\"":
+          retval.append("\\\"")
+        elif c == "'":
+          if json_data[i:i+7] == "'undef'":
+            retval.append("'undef'")
+            skip = 7
+          else:
+            retval.append("\"")
+            in_str = False
+        else:
+          retval.append(c)
+      else:
+        if c == "'":
+          retval.append("\"")
+          in_str = True
+        else:
+          retval.append(c)
+  return "".join(retval)
+
+parser = OptionParser()
+parser.add_option("-s", "--span", action="store",
+                  dest="timespan", default="20d",
+                  help="Period of time to fetch data for (N[y,m,w,d,h], default=%default)")
+parser.add_option("-t", "--tree", action="store",
+                  dest="tree", default="Firefox",
+                  help="Tinderbox tree to fetch data from (default=%default)")
+parser.add_option("-d", "--database", action="store",
+                  dest="db", default="topfails",
+                  help="Database filename (default=%default)")
+parser.add_option("--host", action="store",
+                  dest="dbhost", default="localhost",
+                  help="Database host name (default=%default)")
+parser.add_option( "--port", action="store",
+                  dest="dbport",default="3306",
+                  help="Database port (default=%default)")
+parser.add_option("-u", "--user", action="store",
+                  dest="dbuser", default="root",
+                  help="Database username (default=%default)")
+parser.add_option("-p", "--passwd", action="store",
+                  dest="dbpasswd",
+                  help="Database user password")
+parser.add_option("-v", "--verbose", action="store_true",
+                  dest="verbose", default="False",
+                  help="Enable verbose logging")
+
+(options, args) = parser.parse_args()
+
+logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING)
+
+os.environ['TZ'] = "US/Pacific"
+tzset()
+# Get current time, in seconds.
+endtime = int(time())
+
+m = re.match("(\d+)([ymwdh])", options.timespan)
+if m is None:
+  print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+  sys.exit(1)
+
+timespan = int(m.group(1)) * {'y': 365 * 24 * S_IN_H,
+                              'm':  30 * 24 * S_IN_H,
+                              'w':   7 * 24 * S_IN_H,
+                              'd':       24 * S_IN_H,
+                              'h':            S_IN_H}[m.group(2)]
+# Set current time to beginning of requested timespan ending now.
+curtime = endtime - timespan
+
+
+createdb=False
+
+
+try:
+  connection  =  MySQLdb.connect (host  =  options.dbhost,
+                                    port  =  int(options.dbport),
+                                    db  =  options.db,
+                                    user  =  options.dbuser,
+                                    passwd  =  options.dbpasswd)
+  conn=connection.cursor()
+except MySQLdb.Error, e:
+  print "Error %d: %s" % (e.args[0], e.args[1])
+  createdb = True
+     
+
+if createdb:
+  connection = MySQLdb.connect (host  =  options.dbhost,
+                                    port  =  int(options.dbport),
+                                    user  =  options.dbuser,
+                                    passwd  =  options.dbpasswd)
+  conn = connection.cursor()
+  try:
+    createdatabase='create database %s' %(options.db)
+    conn.execute (createdatabase)
+    conn.close()
+    connection.commit()
+    connection.close()
+  except  MySQLdb.Error, e:
+    print "Error %d: %s" % (e.args[0], e.args[1])
+    sys.exit (1)
+  try:
+    connection  =  MySQLdb.connect (host  =  options.dbhost,
+                                      port  =  int(options.dbport),
+                                      db  =  options.db,
+                                      user  =  options.dbuser,
+                                      passwd  =  options.dbpasswd)
+    conn=connection.cursor()
+  except MySQLdb.Error, e:
+    print "Error %d: %s" % (e.args[0], e.args[1])
+    sys.exit(1)
+  
+  CreateDBSchema(conn)
+
+
+treeid = GetOrInsertTree(conn, options.tree)
+
+logging.info("Reading tinderbox data...")
+
+chunk = 0
+# add a fudge factor here, since builds can take up to 3 hours to finish,
+# and we can't get the changeset unless we ask for time up to the end of the
+# build
+endtime += 3 * S_IN_H
+timespan += 3 * S_IN_H
+totalchunks = int(ceil(float(timespan) / chunksize))
+
+while curtime < endtime and chunk < totalchunks:
+  chunk += 1
+  logging.info("Chunk %d/%d" % (chunk, totalchunks))
+
+  if (endtime - curtime) < chunksize:
+    chunksize = endtime - curtime
+
+  tboxurl = "http://tinderbox.mozilla.org/showbuilds.cgi?tree=%(tree)s&maxdate=%(maxdate)d&noignore=1&hours=%(hours)d&json=1" \
+              % {'tree': options.tree,
+                 'maxdate': curtime + chunksize, # tbox wants the end time
+                 'hours': int(chunksize / S_IN_H)}
+  u = urllib.urlopen(tboxurl)
+  tboxjson = u.read()
+  #tboxjson = tboxjson.encode('utf-8').decode('string_escape').decode('utf-8')
+  #tboxjson = ''.join(u.readlines())
+  u.close()
+  
+  tboxjson = fix_tbox_json(tboxjson)
+  try:
+    tboxdata = json.loads(tboxjson)
+  except Exception, inst:
+    print >>sys.stderr, "Error parsing JSON: %s" % inst
+    continue
+
+  # we only care about unit test boxes
+  unittest_indices = [tboxdata['build_name_index'][x] for x in tboxdata['build_name_index'] if re.search("test|xpc|check", x)]
+  # read build table
+  # 'TestFailed' expected log format is "result | test | optional text".
+  testfailedRe = re.compile(r"(TEST-UNEXPECTED-.*) \| (.*) \|(.*)")
+  for timerow in tboxdata['build_table']:
+    for index in unittest_indices:
+      if index >= len(timerow) or timerow[index] == -1:
+        continue
+      build = timerow[index]
+      if 'buildname' not in build or \
+         'logfile'   not in build:
+        continue
+
+      status = BuildStatusFromText(build['buildstatus'])
+      # Skip unavailable "builds".
+      if status < BuildStatus.Success:
+        continue
+
+      name = build['buildname']
+      os = OSFromBuilderName(name)
+      starttime = int(build['buildtime'])
+      # skip builds we've already seen
+      if HaveBuild(conn, treeid, os, starttime):
+        logging.info("Skipping already seen build '%s' at %d (%s)" % (name, starttime, ctime(starttime)))
+
+        # Call 'UpdateLogfile()' anyway.
+        UpdateLogfile(conn, treeid, os, starttime, build['logfile'])
+        continue
+
+      # must have scrape data for changeset
+      if build['logfile'] not in tboxdata['scrape']:
+        continue
+      changeset = FindChangesetInScrape(tboxdata['scrape'][build['logfile']])
+      if changeset is None:
+        continue
+
+      buildid = InsertBuild(conn, treeid, os, starttime, status, build['logfile'], changeset)
+
+      # 'Success' is fine as is.
+      if status == BuildStatus.Success:
+        pass
+
+      # Parse log to save 'TestFailed' results.
+      elif status == BuildStatus.TestFailed :
+        logging.info("Checking build log for '%s' at %d (%s)" % (name, starttime, ctime(starttime)))
+        try:
+          # Grab the build log.
+          log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile']))
+          gz = GzipFile(log)
+          # Look for test failures.
+          for line in gz:
+            m = testfailedRe.match(line)
+            if m:
+              test = rawtest = m.group(2).strip() or "[unittest-log.py: no logged test]"
+              if rawtest.find('\\') != -1:
+                test = rawtest.replace('\\','/')
+                
+              if test.find('/') != -1:
+                tup=test.partition('build/')
+                if len(tup[2]) > 2:
+                  test=tup[2]
+                else :
+                  test=tup[0]
+                
+              text = m.group(3).strip() or "[unittest-log.py: no logged text]"
+              testnames_id=GetOrInsertTest(conn,test)
+              InsertTest(conn, buildid, m.group(1).rstrip(), testnames_id, text)
+        except:
+          logging.error("Unexpected error: %s" % sys.exc_info()[0])
+          #XXX: handle me?
+
+      # Ignore 'Burning' builds: tests may have run nontheless, but it's safer to discard them :-|
+      elif status == BuildStatus.Burning:
+        continue
+      
+      # Ignore 'Exception' builds: should only be worse than 'Burning'.
+      # (Don't know much at time of writing, since this feature is not active yet: see bug 476656 and follow-ups.)
+      elif status == BuildStatus.Exception:
+        continue
+
+      # Save 'Unknown' status failure: this should not happen (unless new statuses are created), but we want to know if it does.
+      elif status == BuildStatus.Unknown:
+        # Add a fake test failure.
+        InsertTest(conn, buildid, "TEST-UNEXPECTED-FAIL", "99999999999", "Unknown status = '%s'!" % build['buildstatus'])
+        continue
+
+
+
+  if chunk < totalchunks:
+    sleep(SLEEP_TIME)
+  curtime += chunksize
+  
+conn.close()
+connection.commit()
+connection.close()
+logging.info("Done")