working code bits ??

topfails/mappings.py

@@ -1,11 +1,15 @@
+#!/usr/bin/env python
 """
+Build name:
 <OS> <branch> [type of build]
 
 Examples:
 
 The builder: <OS> <branch> build (e.g. 'Linux mozilla-central build')
+-- in this case [type of build] is the string 'build'
 
 A debug test: <OS> <branch> debug test <test type>
+-- in this case [type of build] is the string 'debug test <test type>'
 
 An opt test: <OS> <branch> opt test <test type>
 
@@ -15,8 +19,28 @@ Talos:
 <OS> <branch> talos (e.g. 'Rev3 Fedora 12x64 mozilla-central talos')
 -or-
 <OS> <branch> talos <type> (e.g. 'Rev3 Fedora 12x64 mozilla-central talos cold')
+
+Currently, the mappings are coded here (in python);  this has the restriction
+that the mappings cannot be (intelligently) programmatically updated.
+If it is desired that the mappings may be undated programmatically
+(e.g. from the command line), then a [presumedly text-ish] storage method
+should be used for these mappings, e.g. an .ini file
+
+So....
+
+All of this data lives in the buildbot-configs.
+See http://hg.mozilla.org/build/buildbot-configs/file/tip/mozilla2/config.py
+The mappings are duplicated here for expediency.
+
+ - what really should happen is that this config file should be imported and
+   used here.  In order for this to happen:
+   - the config.py file should be refactored so that it is consumable (and probably the entire buildbot-configs as well)
+   - buildbot-configs (or whatever this piece is refactored into) should
+     become a real python package or otherwise installable/depended upon
 """
 
+import re
+
 # OS mappings
 OSes = [ 'Linux',
          'Linux x86-64',
@@ -32,14 +56,40 @@ OSes = [ 'Linux',
 OS_to_index = dict([(OS, index) for index, OS in enumerate(OSes)])
 index_to_OS = dict([(index, OS) for index, OS in enumerate(OSes)])
 OSdata = { 'Linux': {'name': 'Linux', 'bits': 32 },
-           'Rev3 Fedora 12': { 'name': 'Fedora', 'bits': 32}
+           'Rev3 Fedora 12': { 'name': 'Fedora', 'bits': 32},
-           'Rev3 Fedora 12x64': { 'name': 'Fedora', 'bits': 64}
+           'Rev3 Fedora 12x64': { 'name': 'Fedora', 'bits': 64},
-	   'Linux x86-64': { 'name': 'Linux', 'bits': 64}
+	   'Linux x86-64': { 'name': 'Linux', 'bits': 64},
-	   'OS X 10.5.2': { 'name': 'MAC OSX', 'bits': 32}
+	   'OS X 10.5.2': { 'name': 'MAC OSX', 'bits': 32},
-	   'OS X 10.6.2': { 'name': 'MAC OSX', 'bits': 64}
+	   'OS X 10.6.2': { 'name': 'MAC OSX', 'bits': 64},
-	   'Rev3 MacOSX Leopard 10.5.8': { 'name': 'MAC OSX', 'bits': 32}
+	   'Rev3 MacOSX Leopard 10.5.8': { 'name': 'MAC OSX', 'bits': 32},
-	   'Rev3 MacOSX Snow Leopard 10.6.2': { 'name': 'MAC OSX', 'bits': 64}
+	   'Rev3 MacOSX Snow Leopard 10.6.2': { 'name': 'MAC OSX', 'bits': 64},
-	   'Rev3 WINNT 5.1': { 'name': 'Windows', 'bits': 32}
+	   'Rev3 WINNT 5.1': { 'name': 'Windows', 'bits': 32},
-	   'Rev3 WINNT 6.1': { 'name': 'Windows', 'bits': 64}
+	   'Rev3 WINNT 6.1': { 'name': 'Windows', 'bits': 64},
-	   'WINNT 5.2': { 'name': 'Windows', 'bits': 32}
+	   'WINNT 5.2': { 'name': 'Windows', 'bits': 32},
     }
+
+# branch objects
+# branches = [ 'mozilla-central',
+#              'mozilla-1.9.2',
+#              'comm-central',
+#              'comm-central-trunk'
+#     ]
+trees = { 'Firefox': 'mozilla-central',
+          'Firefox3.6': 'mozilla-1.9.2',
+          'Thunderbird': 'comm-central',
+          'SeaMonkey': 'comm-central-trunk',
+          }
+
+build_name_regex = r'(?P<os>%s) (?P<branch>%s) (?P<type>.*)' % ('|'.join(OSes), '|'.join(trees.values()))
+build_name_regex = re.compile(build_name_regex)
+def parse_build_name(name):
+  match = re.match(build_name_regex, name)
+  if match is None:
+    return None
+  return match.groupdict()
+
+if __name__ == '__main__':
+  import sys
+  for arg in sys.argv[1:]:
+    print parse_build_name(arg)
+    

topfails/unittest-log.py

@@ -49,8 +49,9 @@ from optparse import OptionParser
 from gzip import GzipFile
 import binascii
 
-# local import
+# local imports
 import log_parser
+import mappings
 
 try:
   # 'Availability: Unix.'
@@ -82,12 +83,6 @@ chunksize = 24 * S_IN_H
 # seconds between requests
 SLEEP_TIME = 1
 
-class OS():
-  Windows = 0
-  Mac = 1
-  Linux = 2
-  Unknown = 3
-
 class BuildStatus():
   # Unavailable builds to skip.
   # Values need (only) to be less than the 'Success' one.
@@ -110,15 +105,6 @@ def FindChangesetInScrape(scrape):
       return m.group(1)
   return None
 
-def OSFromBuilderName(name):
-  if name.startswith("Linux") or name.startswith("Rev3 Fedora") or name.startswith("Fedora"):
-    return OS.Linux
-  if name.startswith("MacOSX") or name.startswith("OS X") or name.startswith("Rev3 MacOSX"):
-    return OS.Mac
-  if name.startswith("WINNT"):
-    return OS.Windows
-  return OS.Unknown
-
 buildStatuses = {
   # "No build in progress".
   "null":       BuildStatus.NoBuild,
@@ -175,25 +161,25 @@ def GetOrInsertTest(conn, testname):
   connection.commit()
   return conn.lastrowid
 
-def HaveBuild(conn, treeid, os, starttime):
+def HaveBuild(conn, treeid, _os, starttime):
   """See if we already have this build in our database."""
   conn.execute("""
                SELECT COUNT(*) FROM viewer_build WHERE tree_id = %s AND os = %s AND starttime = %s
-               """, (treeid, os, starttime))
+               """, (treeid, _os, starttime))
   return conn.fetchone()[0] == 1
 
-def UpdateLogfile(conn, treeid, os, starttime, logfile):
+def UpdateLogfile(conn, treeid, _os, starttime, logfile):
   """Update empty 'logfile' for a given build (added in db schema v1)."""
   conn.execute("""
                UPDATE viewer_build SET logfile = %s WHERE tree_id = %s AND os = %s AND starttime = %s AND logfile IS NULL
-               """, (logfile, treeid, os, starttime))
+               """, (logfile, treeid, _os, starttime))
   connection.commit()
 
-def InsertBuild(conn, treeid, os, starttime, status, logfile, changeset):
+def InsertBuild(conn, treeid, _os, starttime, status, logfile, changeset):
   """Insert a build into the builds table and return the id."""
   conn.execute("""
                INSERT INTO viewer_build (tree_id, os, starttime, status, logfile, changeset) VALUES (%s, %s, %s, %s, %s, %s)
-               """, (treeid, os, starttime, status, logfile, changeset))
+               """, (treeid, _os, starttime, status, logfile, changeset))
   connection.commit()
   return conn.lastrowid
 
@@ -310,6 +296,10 @@ parser.add_option("--die", action='store_true',
                   help="enable application to die on error")
 (options, args) = parser.parse_args()
 
+# check parsed options
+if options.tree not in mappings.trees:
+  parser.error("Unknown tree: '%s'; should be one of [%s]" % (options.tree, ', '.join(mappings.trees)))
+
 logging.basicConfig(level=options.verbose and logging.DEBUG or logging.WARNING)
 
 os.environ['TZ'] = "US/Pacific"
@@ -450,14 +440,18 @@ while curtime < endtime and chunk < totalchunks:
         continue
 
       name = build['buildname']
-      os = OSFromBuilderName(name)
+      build_name_dict = mappings.parse_build_name(name)
+      if build_name_dict:
+        _os = OS_to_index[build_name_dict['os']]
+      else:
+        _os = -1 # UNKNOWN
       starttime = int(build['buildtime'])
       # skip builds we've already seen
-      if HaveBuild(conn, treeid, os, starttime):
+      if HaveBuild(conn, treeid, _os, starttime):
         logging.info("Skipping already seen build '%s' at %d (%s)" % (name, starttime, ctime(starttime)))
 
         # Call 'UpdateLogfile()' anyway.
-        UpdateLogfile(conn, treeid, os, starttime, build['logfile'])
+        UpdateLogfile(conn, treeid, _os, starttime, build['logfile'])
         continue
 
       # must have scrape data for changeset
@@ -467,7 +461,7 @@ while curtime < endtime and chunk < totalchunks:
       if changeset is None:
         continue
 
-      buildid = InsertBuild(conn, treeid, os, starttime, status, build['logfile'], changeset)
+      buildid = InsertBuild(conn, treeid, _os, starttime, status, build['logfile'], changeset)
 
       # 'Success' is fine as is.
       if status == BuildStatus.Success: