Commit 21ae34a7 authored by Dr. Martin Goik's avatar Dr. Martin Goik

Better validation error hints.

parent 83d2d7d7
......@@ -3,11 +3,11 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.hdm-stuttgart.mi.sda1</groupId>
<artifactId>exam</artifactId>
<artifactId>sda1_2015winter_solve</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<name>exam</name>
<name>sda1_2015winter_solve</name>
<url>http://freedocs.mi.hdm-stuttgart.de</url>
......@@ -26,7 +26,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.12</version>
<scope>test</scope>
<optional>true</optional>
</dependency>
......@@ -61,6 +61,13 @@
<version>1.1.6</version>
</dependency>
<dependency>
<groupId>de.hdm_stuttgart.mi.exam</groupId>
<artifactId>unitmarking</artifactId>
<version>0.9</version>
</dependency>
</dependencies>
<build>
......@@ -68,7 +75,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<version>3.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
......
package de.hdm_stuttgart.mi.sda1.exam.test.ignore;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Vector;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jdom2.Document;
import org.jdom2.JDOMException;
import org.jdom2.ProcessingInstruction;
import org.jdom2.filter.Filters;
import org.jdom2.input.SAXBuilder;
import org.jdom2.input.sax.XMLReaders;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
/**
* Evaluate a given set of XML document instances
*
*/
public class InstanceSetEvaluation {
/**
* Base directory containing xml test file instances.
*/
public final String xmlTestFileDir;
/**
* true if all tests have been successful.
*/
public final boolean allTestsSucceeded;
private static Logger log = LogManager.getLogger(InstanceSetEvaluation.class);
final static List<InstanceTest> tests = new Vector<>();
final StringBuffer messages = new StringBuffer(), errorMessages = new StringBuffer();
/**
* @param args unused
*/
public static void main(String[] args) {
final InstanceSetEvaluation ise = new InstanceSetEvaluation("SchemaTest", "Schema/dictionary.xsd");
System.out.println(ise.getMessages());
System.out.println("All tests succeeded:" + ise.allTestsSucceeded);
}
/**
* @return Individual failed test(s) error message(s).
*/
public String getErrorMessages() {
return errorMessages.toString();
}
/**
* @return Individual test's and aggregated results.
*/
public String getMessages() {
return messages.toString();
}
/**
*
* @param xmlTestFileDir Directory containing XML instances to be processed.
*/
public InstanceSetEvaluation(final String xmlTestFileDir, final String xsdSchemaFilename) {
this.xmlTestFileDir = xmlTestFileDir;
final File rootDirectory = new File(xmlTestFileDir);
for (final File f: rootDirectory.listFiles(
path -> path.getPath().endsWith(".xml"))) {
readTestHeader(f, xsdSchemaFilename);
}
tests.forEach(t -> messages.append(t + "\n"));
tests.stream().filter(t -> !t.testSucceeded).forEach(t -> errorMessages.append(t + "\n"));
allTestsSucceeded = tests.
stream().
map(t -> t.testSucceeded).
reduce((a, b) -> a && b).
get();
final int maxPoints, reachedPoints;
{
int tmpMaxPoints = 0, tmpReachedPoints = 0;
for (final InstanceTest t: tests) {
tmpMaxPoints += t.reachablePoints;
if (t.testSucceeded) {
tmpReachedPoints += t.reachablePoints;
}
}
maxPoints = tmpMaxPoints;
reachedPoints = tmpReachedPoints;
}
messages.append(reachedPoints + " of " + maxPoints +
" points have been reached");
}
void readTestHeader(final File instanceFilename, final String xsdSchemaFilename) {
final SAXBuilder metainfoParser = new SAXBuilder(XMLReaders.NONVALIDATING);
try {
final Document doc = metainfoParser.build(instanceFilename);
final XPathExpression<ProcessingInstruction> searchHeader =
XPathFactory.instance().compile(
"/processing-instruction('xmlTest')",
Filters.processinginstruction());
final List<ProcessingInstruction> xmlTestList =
searchHeader.evaluate(doc);
switch (xmlTestList.size()) {
case 0:
log.info("No 'xmlTest PI found, possible dependency file");
break;
case 1:
tests.add(new InstanceTest(xmlTestList.get(0), xmlTestFileDir, xsdSchemaFilename, instanceFilename));
break;
default:
log.error("Fatal: Found " + xmlTestList.size() +
" <?xmltest ... ?> annotations in file '" +
instanceFilename + "'");
System.exit(1);
}
} catch (final JDOMException e) {
log.fatal("Document '" + instanceFilename.getPath() +
"' is invalid:", e);
System.exit(1);
} catch (final IOException e) {
log.fatal("Unable to open '" + instanceFilename.getPath() +
"' for reading:", e);
System.exit(1);
}
}
}
package de.hdm_stuttgart.mi.sda1.exam.test.ignore;
import java.io.File;
import java.io.IOException;
import java.util.Optional;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jdom2.ProcessingInstruction;
import org.jdom2.input.SAXBuilder;
import org.jdom2.input.sax.XMLReaderSAX2Factory;
import org.jdom2.input.sax.XMLReaders;
import org.xml.sax.SAXException;
/**
* Test and evaluate an individual XML instance.
*
*/
public class InstanceTest {
final SAXBuilder parser = new SAXBuilder(XMLReaders.XSDVALIDATING);
final SAXBuilder b = new SAXBuilder(new XMLReaderSAX2Factory(false));
final File instanceFilename;
/**
* An examinee may reach this number of point. The value is being defined in
* the corresponding XML instance's PI like e.g.:
*
* &lt;?xmlTest
points = "2" ... ?&gt;
*
*/
public final int reachablePoints;
/**
* Is this instance expected to be valid or not? The value is being defined in
* the corresponding XML instance's PI like e.g.:
*
* &lt;?xmlTest expectedToBeValid= "false" ... ?&gt;
*/
public final boolean expectedToBeValid;
/**
* Does the current instance test succeed?
*/
public final boolean testSucceeded;
/**
* Error message in case of unexpected behavior e.g.
* when an invalid instance is supposed to be valid.
*/
public final Optional<String> errMsg;
private static Logger log = LogManager.getLogger(InstanceTest.class);
@Override
public String toString() {
if (testSucceeded) {
return instanceFilename + ": " + reachablePoints + " point" + (1 == reachablePoints ? "" : "s") ;
} else {
return instanceFilename + ": " + errMsg.get() + ",\n missing" + reachablePoints +" point(s)";
}
}
/**
* Testing an individual instance to be either valid or invalid.
*
* @param xmlTest The PI meta annotation header containing the expected
* result, points etc.
* @param xmlTestFileDir The directory containing the XML instance files.
* @param xmlInstance The current instance to be evaluated.
*/
public InstanceTest(final ProcessingInstruction xmlTest,
final String xmlTestFileDir, final String xsdSchemaFilename, final File xmlInstance) {
Validator validator = null;
SchemaFactory sf = SchemaFactory.newInstance(
"http://www.w3.org/XML/XMLSchema/v1.1");
Schema s;
try {
s = sf.newSchema (new File(xsdSchemaFilename));
validator = s.newValidator();
} catch (SAXException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
this.instanceFilename = xmlInstance;
if (null == xmlTest.getPseudoAttributeValue("points")) {
log.fatal("Mandatory <?xmlTest points='...' ... ?> is missing in file '"
+ xmlInstance + "'");
System.exit(1);
}
reachablePoints = Integer.parseInt(xmlTest.getPseudoAttributeValue("points"));
if (null == xmlTest.getPseudoAttributeValue("expectedToBeValid")) {
log.fatal("Mandatory <?xmlTest expectedToBeValid='true|false' ... ?> is missing in file '" + xmlInstance + "'");
System.exit(1);
}
expectedToBeValid = Boolean.parseBoolean(xmlTest.getPseudoAttributeValue("expectedToBeValid"));
final Optional<String> preconditionValidFilename;
boolean preconditionSucceeded = false;
String tmpPreconditionErrMsg = null;
if (null == xmlTest.getPseudoAttributeValue("preconditionValid")) {
preconditionValidFilename = Optional.empty();
} else {
preconditionValidFilename = Optional.of(xmlTestFileDir + File.separator +
xmlTest.getPseudoAttributeValue("preconditionValid"));
try {
validator.validate(new StreamSource(new File(preconditionValidFilename.get())));
log.info("Precondition file '" + preconditionValidFilename.get() + "' is valid");
preconditionSucceeded = true;
} catch (final SAXException e) {
tmpPreconditionErrMsg = "Precondition file '" + preconditionValidFilename.get() + "' is invalid:" + e;
log.info(tmpPreconditionErrMsg + ": " + e.getCause() + ":" + e.getMessage(), e);
} catch (final IOException e) {
tmpPreconditionErrMsg = "Precondition file '" + preconditionValidFilename.get() + "' is missing:" + e;
log.info(tmpPreconditionErrMsg, e);
}
}
if (!preconditionValidFilename.isPresent() || preconditionSucceeded) {
boolean tmpInstanceIsValid = false;
try {
validator.validate(new StreamSource(xmlInstance));
tmpInstanceIsValid = true;
} catch (final SAXException e) {
log.info("Instance file '" + xmlInstance + "' is invalid:", e);
} catch (final IOException e) {
log.info("Instance file '" + xmlInstance + "' cannot be opened:", e);
}
testSucceeded = expectedToBeValid == tmpInstanceIsValid;
if (testSucceeded) {
errMsg = Optional.empty();
} else if (expectedToBeValid) {
errMsg = Optional.of("Instance '" + xmlInstance +
"' is expected to be valid!");
} else {
errMsg = Optional.of("Instance '" + xmlInstance +
"' is expected to be invalid!");
}
} else {
testSucceeded = false;
errMsg = Optional.of(tmpPreconditionErrMsg);
}
}
}
......@@ -3,7 +3,7 @@ package de.hdm_stuttgart.mi.sda1.exam.xsd;
import org.junit.Assert;
import org.junit.Test;
import de.hdm_stuttgart.mi.sda1.exam.test.ignore.InstanceSetEvaluation;
import de.hdm_stuttgart.mi.sda1.exam.xsdmarking.InstanceSetEvaluation;
/**
* Unit test for an XML instance set.
......
......@@ -58,7 +58,7 @@ public class InstanceSetEvaluation {
* @return Individual test's and aggregated results.
*/
public String getMessages() {
return messages.toString();
return "\n" + messages.toString();
}
/**
......
......@@ -60,10 +60,12 @@ public class InstanceTest {
@Override
public String toString() {
final String result = reachablePoints + ((1 == reachablePoints) ? " point." : " points.");
if (testSucceeded) {
return instanceFilename + ": " + reachablePoints + " point" + (1 == reachablePoints ? "" : "s") ;
return "++Instance " + instanceFilename + ": Gaining " + result ;
} else {
return instanceFilename + ": " + errMsg.get() + ",\n missing" + reachablePoints +" point(s)";
return "--" + errMsg.get() + " Missing " + result;
}
}
......@@ -124,10 +126,10 @@ public class InstanceTest {
log.info("Precondition file '" + preconditionValidFilename.get() + "' is valid");
preconditionSucceeded = true;
} catch (final SAXException e) {
tmpPreconditionErrMsg = "Precondition file '" + preconditionValidFilename.get() + "' is invalid:" + e;
tmpPreconditionErrMsg = "Instance " + xmlInstance + ": Precondition file " + preconditionValidFilename.get() + " is invalid:" + e.getMessage();
log.info(tmpPreconditionErrMsg + ": " + e.getCause() + ":" + e.getMessage(), e);
} catch (final IOException e) {
tmpPreconditionErrMsg = "Precondition file '" + preconditionValidFilename.get() + "' is missing:" + e;
tmpPreconditionErrMsg = "Instance '" + xmlInstance + ": Precondition file '" + preconditionValidFilename.get() + "' cannot be read:" + e.getMessage();
log.info(tmpPreconditionErrMsg, e);
}
}
......@@ -139,19 +141,19 @@ public class InstanceTest {
tmpInstanceIsValid = true;
} catch (final SAXException e) {
log.info("Instance file '" + xmlInstance + "' is invalid:", e);
log.info("Instance file " + xmlInstance + " is invalid:" + e.getMessage());
} catch (final IOException e) {
log.info("Instance file '" + xmlInstance + "' cannot be opened:", e);
log.info("Instance file " + xmlInstance + " cannot be read:" + e.getMessage());
}
testSucceeded = expectedToBeValid == tmpInstanceIsValid;
if (testSucceeded) {
errMsg = Optional.empty();
} else if (expectedToBeValid) {
errMsg = Optional.of("Instance '" + xmlInstance +
"' is expected to be valid!");
errMsg = Optional.of("Instance " + xmlInstance +
" is expected to be valid!");
} else {
errMsg = Optional.of("Instance '" + xmlInstance +
"' is expected to be invalid!");
errMsg = Optional.of("Instance " + xmlInstance +
" is expected to be invalid!");
}
} else {
testSucceeded = false;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment