enhacements and error checks

unittest-logs/unittest-log.py

@@ -46,6 +46,8 @@ from time import ctime, sleep, time
 from math import ceil
 from optparse import OptionParser
 from gzip import GzipFile
+import curses.ascii
+import binascii
 
 try:
   # 'Availability: Unix.'
@@ -166,7 +168,17 @@ def InsertTest(conn, buildid, result, name, description):
   # ToDo: Add column to save result.
   conn.execute("""INSERT INTO tests (buildid, name, description) VALUES (%s, %s, %s)""", (buildid, name, description))
 
-def fix_tbox_json(s): # Check :: This seems to be a problem ? Murali
+def asciirepl(match):
+  # replace the hexadecimal characters with ascii characters
+  s = match.group()  
+  return binascii.unhexlify(s)  
+
+def reformat_content(data):
+  p = re.compile(r'\\x(\w{2})')
+  return p.sub(asciirepl, data)
+  
+  
+def fix_tbox_json(s): # Check :: This is a bad logic by  :: not checking for CRTL chars in JSON text -- Murali
   """Fixes up tinderbox json.
 
   Tinderbox returns strings as single-quoted strings, and occasionally
@@ -186,43 +198,46 @@ def fix_tbox_json(s): # Check :: This seems to be a problem ? Murali
   in_esc = False
   skip = 0
   for i,c in enumerate(json_data):
-    if skip > 0:
+    # The tinderbox data is a fracked json. and it some times contains
-      skip -= 1
+    # Control characters. that would totally fail the json.loads step.
-      continue
+    # So, eliminate them .. all of them .. here -- Murali
-
+    if (c < '\xFD' and c > '\x1F') or c == '\n' or c == '\r' :
-    if in_str:
+      if skip > 0:
-      if in_esc:
+        skip -= 1
-        if c == "'":
+        continue
-          retval.append("'")
+  
+      if in_str:
+        if in_esc:
+          if c == "'":
+            retval.append("'")
+          else:
+            retval.append("\\")
+            retval.append(c)
+          in_esc = False
+        elif c == "\\":
+          in_esc = True
+        elif c == "\"":
+          retval.append("\\\"")
+        elif c == "'":
+          if json_data[i:i+7] == "'undef'":
+            retval.append("'undef'")
+            skip = 7
+          else:
+            retval.append("\"")
+            in_str = False
         else:
-          retval.append("\\")
           retval.append(c)
-        in_esc = False
+      else:
-      elif c == "\\":
+        if c == "'":
-        in_esc = True
-      elif c == "\"":
-        retval.append("\\\"")
-      elif c == "'":
-        if json_data[i:i+7] == "'undef'":
-          retval.append("'undef'")
-          skip = 7
-        else:
           retval.append("\"")
-          in_str = False
+          in_str = True
-      else:
+        else:
-        retval.append(c)
+          retval.append(c)
-    else:
-      if c == "'":
-        retval.append("\"")
-        in_str = True
-      else:
-        retval.append(c)
-
   return "".join(retval)
 
 parser = OptionParser()
 parser.add_option("-s", "--span", action="store",
-                  dest="timespan", default="15d",
+                  dest="timespan", default="20d",
                   help="Period of time to fetch data for (N[y,m,w,d,h], default=%default)")
 parser.add_option("-t", "--tree", action="store",
                   dest="tree", default="Firefox",
@@ -335,9 +350,11 @@ while curtime < endtime and chunk < totalchunks:
                  'maxdate': curtime + chunksize, # tbox wants the end time
                  'hours': int(chunksize / S_IN_H)}
   u = urllib.urlopen(tboxurl)
-  tboxjson = ''.join(u.readlines())
+  tboxjson = u.read()
+  #tboxjson = tboxjson.encode('utf-8').decode('string_escape').decode('utf-8')
+  #tboxjson = ''.join(u.readlines())
   u.close()
-
+  
   tboxjson = fix_tbox_json(tboxjson)
   try:
     tboxdata = json.loads(tboxjson)
@@ -399,7 +416,17 @@ while curtime < endtime and chunk < totalchunks:
           for line in gz:
             m = testfailedRe.match(line)
             if m:
-              test = m.group(2).strip() or "[unittest-log.py: no logged test]"
+              test = rawtest = m.group(2).strip() or "[unittest-log.py: no logged test]"
+              if rawtest.find('\\') != -1:
+                test = rawtest.replace('\\','/')
+                
+              if test.find('/') != -1:
+                tup=test.partition('build/')
+                if len(tup[2]) > 2:
+                  test=tup[2]
+                else :
+                  test=tup[0]
+                
               text = m.group(3).strip() or "[unittest-log.py: no logged text]"
               InsertTest(conn, buildid, m.group(1).rstrip(), test, text)
         except:

unittest-logs/unittestweb/templates/viewer/failswindow.html

@@ -0,0 +1,7 @@
+<h1>Failed tests from last {{n}} {{d}} up to now</h1>
+<table>
+<tr><th>Count</th><th align="left">Test name</th></tr>
+{% for f in failures %}
+  <tr><td>{{ f.0 }}</td><td><a href="{% url viewer.views.test %}?name={{ f.1 }}">{{ f.1 }}</a></td></tr>
+{% endfor %}
+</table>

unittest-logs/unittestweb/urls.py

@@ -15,4 +15,5 @@ urlpatterns = patterns('unittestweb.viewer.views',
     (r'^test$', 'test'),
     (r'^timeline$', 'timeline'),
     (r'^topfails$', 'topfails'),
+    (r'^failswindow$','failswindow'),
 )

unittest-logs/unittestweb/viewer/models.py

@@ -6,10 +6,10 @@
 #
 # Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
 # into your database.
-
+import re
 from django.db import models, connection
 from datetime import datetime
-
+from time import ctime, sleep, time
 class OS():
   Windows = 0
   Mac = 1
@@ -75,6 +75,34 @@ class Tests(models.Model):
 
 def get_most_failing_tests():
     cursor = connection.cursor()
-    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) aaa group by name order by count(*) desc limit 250")
+    cursor.execute("select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid  group by builds.id, name) aaa group by name order by count(*) desc limit 25")
     for row in cursor:
         yield row
+
+def get_fails_in_timerange(self):
+  
+  # Get current time, in seconds.
+  endtime = int(time())
+  
+  #print endtime
+  m = re.match("(\d+)([ymwdh])", self)
+  #print m.group(1), m.group(2)
+  if m is None:
+    print >>sys.stderr, "ERROR: bad timespan = '%s'!" % options.timespan
+    sys.exit(1)
+  
+  timespan = int(m.group(1)) * {'y': 365 * 24 * 3600,
+                                'm':  30 * 24 * 3600,
+                                'w':   7 * 24 * 3600,
+                                'd':       24 * 3600,
+                                'h':            3600}[m.group(2)]
+  # Set current time to beginning of requested timespan ending now.
+  curtime = endtime - timespan
+  #print curtime, timespan, endtime-curtime
+  cursor = connection.cursor()
+  statement = "select count(*), name from (select builds.id, name from builds inner join tests on builds.id = tests.buildid where builds.starttime >"+str(curtime)+" group by builds.id, name) aaa group by name order by count(*) DESC"
+  cursor.execute(statement)
+  for row in cursor:
+    print row
+    yield row
+    

unittest-logs/unittestweb/viewer/views.py

@@ -1,5 +1,6 @@
 from django.shortcuts import render_to_response, get_list_or_404
-from unittestweb.viewer.models import Builds, Trees, Tests, OS_CHOICES, get_most_failing_tests
+from unittestweb.viewer.models import Builds, Trees, Tests, OS_CHOICES, get_most_failing_tests, get_fails_in_timerange
+import re
 
 def index(request):
   failures = get_list_or_404(Tests.objects.all().order_by('-build__starttime')[:10])
@@ -53,3 +54,24 @@ def timeline(request):
   return render_to_response('viewer/timeline.html', {'test': name,
                                                      'descriptions': desc_list,
                                                      'builds': buildlist})
+
+def failswindow(request):
+  period=request.GET['window']
+  m = re.match("(\d+)([ymwdh])", period)
+  failures = get_fails_in_timerange(period)
+  if m.group(2) == 'd':
+    prd='days'
+  elif m.group(2) == 'h':
+    prd = 'hours'
+  elif m.group(2) == 'w':
+    prd = 'weeks'
+  elif m.group(2) == 'm':
+    prd = 'months'
+  elif m.group(2) == 'y':
+    prd = 'years'
+  else:
+    prd = 'days'
+  
+    
+  return render_to_response('viewer/failswindow.html', {'failures': failures,'n':m.group(1),'d':prd})
+