""" Merge all the command-line parameters, see how many systems there are (use UNA :-), make those be columns, then go through all the tests in some list, and see how they compare. TODO: - totals/statistics - better reporting of syntax & network errors during load - do something like IMPORTS so sources can have parts be separate Smelly Bits: - some tedious code around approved/proposed/extracredit """ import time from sys import stderr from sys import stdout from sys import exit ################################################################ # My own stuff, found at from http://www.w3.org/2000/10/swap # we're generating HTML, this exports the HTML element names as functions, # like p(...), a(..., href=), ... from html import * import ArgHandler import LX.kb from LX.namespace import Namespace, ns import LX.reporter maxPasses=10 ################################################################ # pass this to html.py as the str(...) thing to use??? def text(path): """Return something we can add to a flow.... Should follow arcs, use xml:lang, etc, to get the right kind of value we might want. Should be passed to html.py for it to use to str(...) things we pass it? """ try: (value, datatype) = path.data except KeyError, k: raise k #print >> stderr, path, "missing data" #return "<< @@ missing data >>" if (datatype is None) or (datatype == ns.xsd.string): return value if datatype == ns.rdf.XMLLiteral: return Raw(value) print >> stderr, "P", `path.data` print >> stderr, path, "data of unexpected datatype:", type(datatype) print >> stderr, "datatype == None?", str(datatype == None) return "<< @@ data of unexpected datatype >>" def arrayset(a, i, value): try: a[i]=value except IndexError, e: a.extend( (None,) * (1+i-len(a)) ) a[i]=value ################################################################ class TestResultsTable: def __init__(self, reporter=LX.reporter.theNullReporter): self.stylesheet = "http://www.w3.org/2003/08/owl-systems/style.css" self.reporter=reporter self.sources = [] self.pageStarted = 0 self.kb = LX.kb.KB() self.kb.reporter=self.reporter self.kb.ns.sys = Namespace("http://www.w3.org/2003/08/owl-systems/sys-ont", strict=0) self.kb.ns.tres = Namespace("http://www.w3.org/2002/03owlt/resultsOntology#", strict=0) self.kb.ns.otest = Namespace("http://www.w3.org/2002/03owlt/testOntology#", strict=0) self.tests = {} self.systems = {} self.nextCol=0 self.sysByCol = {} syn = (self.kb.ns.otest.SyntacticLevelTest, "Syntactic Level Tests", None) other = (None, "Tests of Unknown Type", None) self.categoriesInOrder = [ (self.kb.ns.otest.NotOwlFeatureTest, "Use of OWL Namespace", "testBadOwl"), (self.kb.ns.otest.PositiveEntailmentTest, "Entailment Tests", "testEntailment"), (self.kb.ns.otest.NegativeEntailmentTest, "Non-Entailment Tests", "testNonEntailment"), (self.kb.ns.otest.TrueTest, "True Tests", "testTrue"), (self.kb.ns.otest.OWLforOWLTest, "OWL for OWL Tests", "testOWLforOWL"), (self.kb.ns.otest.ConsistencyTest, "Consistency Tests", "testConsistency"), (self.kb.ns.otest.InconsistencyTest, "Inconsistency Tests", "testInconsistency"), (self.kb.ns.otest.ImportEntailmentTest, "Import Entailment Tests", "testImportEntailment"), (self.kb.ns.otest.ImportLevelTest, "Import Level Tests", "testImportLevel"), syn, other ] self.indexOfSyntacticLevelTest = self.categoriesInOrder.index(syn) self.indexOfOther = self.categoriesInOrder.index(other) self.passCount = { 'Approved': [ [] for i in range(0, maxPasses+1) ], 'Proposed': [ [] for i in range(0, maxPasses+1) ], 'Extra Credit': [ [] for i in range(0, maxPasses+1) ] } self.neverPassedApproved = [] self.twicePassedProposed = [] self.zoPassedProposed = [] self.bin = [[] for i in self.categoriesInOrder] self.testCount = 0 self.syntacticLevelTestFrom = { } self.tableCount = 1 def load(self, uri): t0 = time.time() self.kb.load(uri) self.sources.append(uri) def buildPage(): pass def buildTable(): pass anonSys=1 def considerSystem(self, sys): if self.systems.has_key(sys): return self.systems[sys] = 1 sys.column = self.nextCol try: sys.label = text(sys.rdfs_label) except: sys.label = "anon_"+str(self.anonSys) self.anonSys += 1 arrayset(self.sysByCol, self.nextCol, sys) self.nextCol += 1 def considerTest(self, test): """Add some properties to the given test object, especially .cell (which is what will be output as a table cell for it) and .sortKey. RETURNs test, which might be different, if some equality processing was done. Some tests are in the spec, some are not. Some are approved, ... Some have URIs, some are based on ones with URIs, ... """ if self.tests.has_key(test): return test self.tests[test] = 1 test.filled = [] test.passedBy = {} test.failedBy = {} test.approved = 0 test.obsoleted = 0 test.proposed = 0 test.extraCredit = 0 try: stf = test.tres_syntacticLevelTestFrom.only() except KeyError, e: pass else: # reuse existing stf (syn test from) test try: test = self.syntacticLevelTestFrom[stf] except: self.syntacticLevelTestFrom[stf] = test else: return test stf=self.considerTest(stf) # make sure it's been considered first test.label = "Syntactic Level Test for "+stf.label test.uri = "lhjfslkjfsjdkfsdfjksd @@@" test.id = "test"+str(self.testCount) self.testCount+=1 test.cell = td("Syntactic Level Test for ",a(stf.label, href="#"+stf.id)) test.category = self.kb.ns.otest.SyntacticLevelTest test.categoryKey = self.indexOfSyntacticLevelTest test.sortKey = test.label test.approved = stf.approved test.proposed = stf.proposed test.extraCredit = stf.extraCredit test.minLevel = stf.minLevel self.bin[self.indexOfSyntacticLevelTest].append(test) return test test.id = "test"+str(self.testCount) self.testCount+=1 try: u = test.fromTerm.uri except AttributeError, e: pass else: test.uri = u href = u if u.endswith("#test"): u = u[:-5] name=u std = "http://www.w3.org/2002/03owlt/" if u.startswith(std): parts=u[len(std):].split("/") if parts[1].startswith("Manifest"): number=parts[1][len("Manifest"):] name = parts[0]+"-"+number useSpec = 1 test.label = name test.cell = td() #test.cell << a(test.label, href=href) test.levels = [] levels = [] for level in test.otest_level: # maybe there should be a label property on these things? test.levels.append(level) level = level.fromTerm if level == ns.otest.DL: levels.append("2DL") elif level == ns.otest.Full: levels.append("3Full") elif level == ns.otest.Lite: levels.append("1Lite") else: # print >>stderr, level levels.append(str(level.uri)) if not levels: levels = ["No Data",] levels.sort() test.sortKey = " ".join(levels)+test.label levels = [x[1:] for x in levels] test.minLevel = levels[0] test.cell << span("(levels:"+", ".join(levels)+") ", class_="links") test.cell << a(test.label, href=href) try: status = test.rtest_status.data[0].encode('utf-8') description = text(test.rtest_description) except KeyError, e: pass else: r = span(class_="links") r << " [" + status if status == "OBSOLETED": test.obsoleted=1 if status == "OBSOLETE": test.obsoleted=1 if status == "APPROVED": test.approved=1 if status == "EXTRACREDIT": test.extraCredit=1 if status == "PROPOSED": test.proposed=1 if (test.approved or test.extraCredit) and useSpec: r << ": " r << a("Med", href="http://www.w3.org/TR/owl-test/M#"+name) r << " " r << a("L", href="http://www.w3.org/TR/owl-test/L#"+name) r << " " r << a("XXL", href="http://www.w3.org/TR/owl-test/XXL#"+name) r << "]" test.cell << r test.cell << span(description, class_="desc") #test.sortKey = test.label # in case we haven't already been put in some category if not hasattr(test, "categoryKey"): test.categoryKey = self.indexOfOther self.bin[self.indexOfOther].append(test) return test raise RuntimeError(test) # how to label it, then? def considerRun(self, run): run.sys = run.tres_system.only() self.considerSystem(run.sys) run.test = run.tres_test.only() run.test = self.considerTest(run.test) arrayset(run.test.filled, run.sys.column, run) def considerPassingRun(self, run): self.considerRun(run) try: run.cell = td(a("Pass", href=run.tres_output.uri), class_="pass") except KeyError: run.cell = td("Pass", class_="pass") run.passes = 1 run.test.passedBy[run.sys] = 1 #@ def considerFailingRun(self, run): self.considerRun(run) run.passes = 0 run.test.failedBy[run.sys] = 1 try: run.cell = td(a("Fail", href=run.tres_output.uri), class_="fail") except KeyError: run.cell = td("Fail", class_="fail") def considerIncompleteRun(self, run): self.considerRun(run) run.passes = 0 try: #un.cell = td(a("Incomplete", href=run.tres_output.uri)) run.cell = td(a("Undecided", href=run.tres_output.uri)) except KeyError: run.cell = td("Undecided") def generateTable(self, tests, section): tab = table() sortList = [(x.sortKey, x) for x in tests] sortList.sort() testsInOrder = [val for (key, val) in sortList] useSystem = [0] * self.nextCol passCount = [0] * self.nextCol count = 0 for test in testsInOrder: count += 1 for col in range(0, self.nextCol): try: run = test.filled[col] except IndexError, e: run = None if run: useSystem[col] = 1 if run.passes: passCount[col] += 1 skip=[] for col in range(0, self.nextCol): if not useSystem[col]: skip.append(a(self.sysByCol[col].label, href="#sys"+str(col))) if skip: s2 = p() s2 << "No results on these tests for: " for x in skip: s2 << x s2 << " " section << s2 row = tr() tab << thead(row) row << th("Test") for col in range(0, self.nextCol): if not useSystem[col]: continue row << td(a(self.sysByCol[col].label, href="#sys"+str(col))) row = tr() tab << row row << td("Percent Passing (of %d tests)" % count) for col in range(0, self.nextCol): if not useSystem[col]: continue row << td("%2.1d%%" % (100 * passCount[col] / count)) for test in testsInOrder: row = tr(id_=test.id) count = len(test.passedBy) flatCount = max(count, maxPasses) if test.proposed: self.passCount['Proposed'][flatCount].append(test) elif test.approved: self.passCount['Approved'][flatCount].append(test) elif test.extraCredit: self.passCount['Extra Credit'][flatCount].append(test) if count < 1: #row << td(str(count), class_="fail") if test.approved: self.neverPassedApproved.append(test) elif count > 1: #row << td(str(count), class_="pass") if test.proposed: self.twicePassedProposed.append(test) else: #row << td(str(count)) pass if count < 2 and test.proposed: self.zoPassedProposed.append(test) row << test.cell for col in range(0, self.nextCol): if not useSystem[col]: continue class_=None output=None try: run = test.filled[col] except IndexError, e: run = None if run: row << run.cell else: row << td("no data", class_="nodata") #print #name = label(system) #s = div(id_=name) #s << h3(name) ## s << p(text(system.sys_description.x_member.x_text)) #d << s if not test.obsoleted: tab << row section << tab def generatePage(self): self.prepareData() return self.pageItself() def prepareData(self): self.reporter.begin("page generation") self.reporter.begin("precomputing attributes") self.kb.fillNodes() self.reporter.end() # redo this at the triple level to maybe pick up more systems # being mention, etc? # for run in self.kb.query(x, ns.tres.test(x, y)) #print self.kb self.reporter.begin("gathering tests of each kind") for index in range(0, len(self.categoriesInOrder)): (term, string, fragment) = self.categoriesInOrder[index] self.reporter.msg(string) n = self.kb.getNode(term) for test in n.is_rdf_type: test.categoryKey = index self.bin[index].append(test) self.considerTest(test) self.reporter.end() self.reporter.begin("gathering runs of each kind") for run in self.kb.tres_PassingRun.is_rdf_type: self.considerPassingRun(run) for run in self.kb.tres_FailingRun.is_rdf_type: self.considerFailingRun(run) for run in self.kb.tres_IncompleteRun.is_rdf_type: self.considerIncompleteRun(run) for run in self.kb.tres_UndecidedRun.is_rdf_type: self.considerIncompleteRun(run) self.reporter.end() def pageItself(self): self.reporter.begin("generating merged HTML") if self.pageStarted: d = div() else: d = Document() d.head << title(self.pageTitle) d.head << stylelink(self.stylesheet) d << h1(self.pageTitle) loc="http://www.w3.org/2003/08/owl-systems/test-results-out" d << p(["Primary version lives at ", a(loc, href=loc)]) try: d << self.extra1 except AttributeError: pass d << h2("Contents") tableOfContents = ol() d << tableOfContents self.group1(d, tableOfContents) self.group2(d, tableOfContents) self.group3(d, tableOfContents) self.group4(d, tableOfContents) self.group5(d, tableOfContents) self.reporter.end() self.reporter.end() return d def group1(self, d, tableOfContents): tableOfContents << li(a("OWL Test Categories", href="#sum")) st = table() # this is built later, as we're generating the tables d << div(h2("OWL Test Categories"), st, id_="sum") tableOfContents << li(a("Results Grouped By Category", href="#res")) d << div(h2("Results Grouped By Category", id_="res")) stHead1 = tr() stHead2 = tr() st << stHead1 st << stHead2 for index in range(0, len(self.categoriesInOrder)): stRow = tr() (term, string, fragment) = self.categoriesInOrder[index] if index == 0: stHead1 << th() stHead2 << th() tag = string if fragment: tag = a(string, href="http://www.w3.org/TR/owl-test/#"+fragment) stRow << th(tag, class_="testTypeName", align="Left") #print >>stderr, "index",index,"len",len(self.bin[index]) someDataInRow = False for (attr, attrLabel) in [ ("approved", "Approved"), ("proposed", "Proposed"), ("extraCredit", "Extra Credit") ]: if index == 0: stHead1 << th(attrLabel, colspan="3", class_="status") for minLevel in ( ("Lite", "DL", "Full") ): label = attrLabel + " " + minLevel if index == 0: stHead2 << th(minLevel, class_="minLevel") tests = [x for x in self.bin[index] if getattr(x, attr) and x.minLevel==minLevel] cell = td(class_="linkToTable", align="center") stRow << cell if tests: id_ = "table_"+str(index)+"_"+label name = self.categoriesInOrder[index][1]+" ("+str(len(tests))+" "+label+")" s = div(id_=id_) s << h3(str(self.tableCount)+". "+name) self.tableCount+=1 self.generateTable(tests, s) d << s cell << a(str(len(tests)), href="#"+id_) someDataInRow = True else: cell << "0" #self.reporter.msg("No test for category: " + (self.categoriesInOrder[index][1]+" ("+label+")")) if someDataInRow: st << stRow def group2(self, d, tableOfContents): # arg, this next big block is very similar to the above block, # but I can't see how to factor it out. I guess some kin of # class, with custom bits overridden.... Hrm. tableOfContents << li(a("Results Grouped By Number of Systems Passing", href="#bycount")) d << h2("Results by Number of Systems Passing", id_="bycount") st = table() d << st stHead1 = tr() stHead2 = tr() st << stHead1 st << stHead2 for index in range(0, maxPasses+1): stRow = tr() if index == 0: stHead1 << th() stHead2 << th() if index == 1: tag = "Tests With %d Pass" % index elif index == maxPasses: tag = "Tests With %d+ Passes" % index else: tag = "Tests With %d Passes" % index stRow << th(tag, class_="testTypeName", align="Left") someDataInRow = False for (attr, attrLabel) in [ ("approved", "Approved"), ("proposed", "Proposed"), ("extraCredit", "Extra Credit") ]: if index == 0: stHead1 << th(attrLabel, colspan="3", class_="status") for minLevel in ( ("Lite", "DL", "Full") ): label = attrLabel + " " + minLevel if index == 0: stHead2 << th(minLevel, class_="minLevel") tests = [x for x in self.tests.keys() if len(x.passedBy) == index and getattr(x,attr) and x.minLevel==minLevel] cell = td(class_="linkToTable", align="center") stRow << cell if tests: id_ = "byPasses_%s_%d_%s" % (attrLabel, index, minLevel) name = "%s %s (%s)" % (attrLabel, tag, minLevel) s = div(id_=id_) s << h3(str(self.tableCount)+". "+name) self.tableCount+=1 self.generateTable(tests, s) d << s cell << a(str(len(tests)), href="#"+id_) someDataInRow = True else: cell << "0" if someDataInRow: st << stRow def group3(self, d, tableOfContents): tableOfContents << li(a("Results Grouped By Number of Systems Failing", href="#byFailcount")) d << h2("Results by Number of Systems Failing", id_="byFailcount") st = table() d << st stHead1 = tr() stHead2 = tr() st << stHead1 st << stHead2 for index in range(0, maxPasses+1): stRow = tr() if index == 0: stHead1 << th() stHead2 << th() if index == 1: tag = "Tests With %d Fail" % index elif index == maxPasses: tag = "Tests With %d+ Fails" % index else: tag = "Tests With %d Fails" % index stRow << th(tag, class_="testTypeName", align="Left") someDataInRow = False for (attr, attrLabel) in [ ("approved", "Approved"), ("proposed", "Proposed"), ("extraCredit", "Extra Credit") ]: if index == 0: stHead1 << th(attrLabel, colspan="3", class_="status") for minLevel in ( ("Lite", "DL", "Full") ): label = attrLabel + " " + minLevel if index == 0: stHead2 << th(minLevel, class_="minLevel") tests = [x for x in self.tests.keys() if len(x.failedBy) == index and getattr(x,attr) and x.minLevel==minLevel] cell = td(class_="linkToTable", align="center") stRow << cell if tests: id_ = "byFails_%s_%d_%s" % (attrLabel, index, minLevel) name = "%s %s (%s)" % (attrLabel, tag, minLevel) s = div(id_=id_) s << h3(str(self.tableCount)+". "+name) self.tableCount+=1 self.generateTable(tests, s) d << s cell << a(str(len(tests)), href="#"+id_) someDataInRow = True else: cell << "0" if someDataInRow: st << stRow ## d << h2("Issues for CR Exit", id_="cr") ## d << h3("Not-Passed Approved Tests", id_="notPassed") ## if self.neverPassedApproved: ## for test in self.neverPassedApproved: ## d << p(a("[Results]", href="#"+test.id), ## " ", ## test.cell.content) # plus more info? ## else: ## d << p("None.") ## d << h3("Twice-Passed Proposed Tests", id_="twicePassedProposed") ## if self.twicePassedProposed: ## for test in self.twicePassedProposed: ## d << p(a("[Results]", href="#"+test.id), ## " ", ## test.cell.content) # plus more info? ## else: ## d << p("None.") ## d << h3("Not-Yet-Twice-Passed Proposed Tests", id_="notyettwicePassedProposed") ## if self.zoPassedProposed: ## for test in self.zoPassedProposed: ## d << p(a("[Results]", href="#"+test.id), ## " ", ## test.cell.content) # plus more info? ## else: ## d << p("None.") def group4(self, d, tableOfContents): tableOfContents << li(a("Tested Systems", href="#systems")) d << h2("Tested Systems", id_="systems") for col in range(0, self.nextCol): d << h3(a("System "+str(col+1)+": "+self.sysByCol[col].label, id_="sys"+str(col))) try: d << p(text(self.sysByCol[col].rdfs_comment)) except KeyError: # phrasing to leave vague whose fault it is! d << p("Unable to display system description") #d << p(str(self.sysByCol[col])) def group5(self, d, tableOfContents): tableOfContents << li(a("(Re)Generation of this Document", href="#about")) d << h2("About This Document / Regeneration", id_="about") d << p("Just starting to get some real data here. If you have some to offer, see if it works here, then send sandro@w3.org a pointer, cc public-webont-comments@w3.org.") d << Raw("""
This is generated from RDF by some python code.
""") f = form(method="GET", action="http://swada.w3.org/test-results-cgi") #f << p("Sources:") #list = ol() #f << list #for uri in sources: # # use default-checked # list << li([input(type="checkbox", name="uri", value="yes", checked="yes"), a(uri, href=uri)]) f << "Current Source URIs (one per line):" f << br() ta = textarea(cols="80", rows="8", name="sources") for uri in self.sources: ta << uri ta << "\n" f << ta f << p("The underlying library now does caching based on HTTP 1.0 features ('Expires' and 'If-Modified-Since'). It greatly speeds regeneration, but if you can't control your own Expires value, you make find debugging difficult. If this turns out to be a problem for anyone, please let me know.") f << p(input(type="submit", value="Regenerate")) d << f d << p("Generated "+time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())) d << p("$Id: TestResultsTable.py,v 1.44 2003/11/24 19:58:55 sandro Exp $") # @@@ Ooops: cvs will intercept this in the OUTPUT d << address("sandro@w3.org") # print tests def startPage(self): self.pageStated = 1 print """Ping! Time=%d
" % time.time()) exit() reporter = LX.reporter.timingHTMLReporter(stdout) reporter.begin("Page generation progress report") trt = TestResultsTable(reporter=reporter) trt.pageTitle = "OWL Test Results (Unofficial Dynamic View)" trt.startPage() sourcesField = fields.getfirst("sources") for arg in sourcesField.splitlines(): arg = arg.strip() if arg == "": continue if arg.startswith("http://"): try: trt.load(arg) except: err(p(["Can't load ", arg])) return else: err(p(["Error: only HTTP URIs are supported. You said: \"", arg, "\""])) return d=trt.generatePage() reporter.end() print "