A cli rascal test runner for use from the maven plugin.

src/org/rascalmpl/interpreter/env/GlobalEnvironment.java

@@ -14,13 +14,17 @@
 *******************************************************************************/
 package org.rascalmpl.interpreter.env;
 
+import java.io.File;
 import java.io.PrintWriter;
 import java.net.URI;
+import java.util.AbstractMap;
 import java.util.ArrayDeque;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -32,11 +36,13 @@
 import org.rascalmpl.interpreter.result.ICallableValue;
 import org.rascalmpl.interpreter.staticErrors.UndeclaredModule;
 import org.rascalmpl.interpreter.utils.Names;
+import org.rascalmpl.uri.URIUtil;
 import org.rascalmpl.values.IRascalValueFactory;
 
 import io.usethesource.capsule.SetMultimap;
 import io.usethesource.vallang.IConstructor;
 import io.usethesource.vallang.IList;
+import io.usethesource.vallang.ISourceLocation;
 import io.usethesource.vallang.IString;
 
 
@@ -178,6 +184,12 @@ public void setModuleURI(String name, URI location) {
 		locationModules.put(location, name);
 	}
 
+	public Map<String, ISourceLocation> moduleFiles() {
+		return moduleLocations.entrySet()
+			.stream()
+			.collect(Collectors.toMap(e -> e.getKey(), e -> URIUtil.assumeCorrectLocation(e.getValue().toASCIIString())));
+	}	
+
 	public URI getModuleURI(String name) {
 		return moduleLocations.get(name);
 	}

src/org/rascalmpl/library/lang/xml/surefire-reports/TestSuites.rsc

@@ -0,0 +1,61 @@
+@synopsis{Schema for Surefire XML test report files}
+@description{
+This class is useful for analyzing the results of test runs,
+and computing descriptive statistics. Another interesting
+application is to extract stacktraces during testing and correlate those
+to earlier or later issue reports (statistical debugging).
+
+A constructive application is to produce these reports for
+the test runs of your own DSL, such that they can be integrated
+in Github CI run reports and other UX.
+}
+module lang::xml::\surefire-reports::TestSuites
+
+import DateTime;
+
+data TestSuites 
+	= testsuites(
+	    Properties properties,
+		list[TestSuite] suites, 
+		int disabled=0, 
+		int errors=0, 
+		int failures=0, 
+		str name="test suite", 
+		int tests=0, 
+		int time=0
+	);
+
+data Properties 
+	= properties(list[Property] properties);
+
+data Property 
+	= property(str name="", str \value="");
+
+data TestSuite 
+	= testsuite(
+		list[TestCase] cases, 
+		str name="test suite", 
+		int tests=0, 
+		int disabled=0, 
+		int errors=0, 
+		int failures=0, 
+		str hostname="localhost", 
+		int id=0,
+		str package="",
+		int skipped=0,
+		int time=0,
+		datetime timestamp=now()
+	);
+
+data TestCase 
+	= testcase(TestResult result, str name="", int assertions=0, str classname="", int time=0);
+
+data TestResult 
+	= skipped(str message)
+	| error(str text, str message="", str \type="")
+	| failure(str text, str message="", str \type="")
+	| \system-out(str text)
+	| \system-err(str text)
+	;   
+
+

src/org/rascalmpl/shell/CommandlineParser.java

@@ -88,6 +88,7 @@ public Map<String, IValue> parseKeywordCommandLineArgs(String name, String[] com
         for (int i = 0; i < commandline.length; i++) {
             if (List.of("-help", "--help", "/?", "?", "\\?", "-?", "--?").contains(commandline[i].trim())) {
                 printMainHelpMessage(kwTypes);
+                out.flush();
                 System.exit(0);
             }
             else if (commandline[i].startsWith("-")) {

src/org/rascalmpl/shell/RascalTest.java

@@ -0,0 +1,115 @@
+package org.rascalmpl.shell;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.LinkedList;
+import java.util.Map;
+
+import org.rascalmpl.debug.IRascalMonitor;
+import org.rascalmpl.exceptions.Throw;
+import org.rascalmpl.library.util.PathConfig;
+import org.rascalmpl.repl.streams.StreamUtil;
+import org.rascalmpl.test.infrastructure.JUnitXMLReportListener;
+import org.rascalmpl.uri.URIUtil;
+import org.rascalmpl.values.IRascalValueFactory;
+
+import io.usethesource.vallang.IConstructor;
+import io.usethesource.vallang.IList;
+import io.usethesource.vallang.ISourceLocation;
+import io.usethesource.vallang.IValue;
+import io.usethesource.vallang.io.StandardTextWriter;
+import io.usethesource.vallang.type.Type;
+import io.usethesource.vallang.type.TypeFactory;
+
+/**
+ * Runs all the tests in the modules given bij de -modules parameter (and their imported/extend modules)
+ * Given a -project parameter, the path configuration will be constructed automatically
+ */
+public class RascalTest extends AbstractCommandlineTool {
+    private static final IRascalValueFactory vf = IRascalValueFactory.getInstance();
+
+    public static void main(String[] args) {
+        try {
+            RascalShell.setupJavaProcessForREPL();
+
+            var term = RascalShell.connectToTerminal();
+            var monitor = IRascalMonitor.buildConsoleMonitor(term);
+            var err = (monitor instanceof Writer) ?  StreamUtil.generateErrorStream(term, (Writer)monitor) : new PrintWriter(System.err, true);
+            var out = (monitor instanceof PrintWriter) ? (PrintWriter) monitor : new PrintWriter(System.out, false);
+
+            try {
+                var parser = new CommandlineParser(out);
+                var parsedArgs = parser.parseKeywordCommandLineArgs("RascalTest", args, parameterTypes());  
+                var pcfgCons = (IConstructor) parsedArgs.get("pcfg");
+                PathConfig pcfg = pcfgCons != null ? new PathConfig(pcfgCons) : new PathConfig();
+                var projectRoot = pcfg.getProjectRoot().getScheme().equals("unknown") ? URIUtil.rootLocation("cwd") : pcfg.getProjectRoot();
+                var eval = ShellEvaluatorFactory.getDefaultEvaluatorForPathConfig(projectRoot, pcfg, term.reader(), out, err, monitor);
+
+                var modules = listParameter(parsedArgs, "modules");
+
+                var modNames = new LinkedList<String>();
+                for (IValue m : modules) {
+                    var l = (ISourceLocation) m;
+                    for (var src: pcfg.getSrcs()) {
+                        var rel = URIUtil.relativize((ISourceLocation) src, l);
+                        if (rel.getScheme().equals("relative")) {
+                            rel = URIUtil.changeExtension(rel, "");
+                            var mod = rel.getPath().substring(1).replaceAll("/", "::");
+                            modNames.add(mod);
+                        }
+                    }
+                }
+
+                eval.doImport(monitor, modNames.stream().toArray(String[]::new));
+
+                boolean reporting = vf.bool(true).equals(parsedArgs.get("reporting"));
+
+                if (reporting) {
+                    eval.setTestResultListener(new JUnitXMLReportListener(URIUtil.getChildLocation(projectRoot, "target"), eval.getHeap().moduleFiles()));
+                }
+
+                if (!eval.runTests(eval.getMonitor())) {
+                    System.exit(1);
+                }
+                else {
+                    System.exit(0);
+                }
+            }
+            catch (Throw e) {
+                try {
+                    err.println(e.getException());
+                    e.getTrace().prettyPrintedString(err, new StandardTextWriter());
+                }
+                catch (IOException ioe) {
+                    err.println(ioe.getMessage());
+                }
+
+                System.exit(1);
+            }
+            catch (Throwable e) {
+                e.printStackTrace();
+                System.exit(1);
+            }
+        }
+        catch (IOException e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+        }
+    }
+
+    private static Type parameterTypes() {
+		var tf = TypeFactory.getInstance();
+		var ll = tf.listType(tf.sourceLocationType());
+
+		return tf.tupleType(
+			PathConfig.PathConfigType, "pcfg",
+			ll, "modules",
+            tf.boolType(), "reporting"
+		);
+	}
+
+    private static IList listParameter(Map<String, IValue> args, String arg) {
+		return args.get(arg) == null ? vf.list() : (IList) args.get(arg);
+	}
+}

src/org/rascalmpl/test/infrastructure/JUnitXMLReportListener.java

@@ -0,0 +1,156 @@
+package org.rascalmpl.test.infrastructure;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import org.rascalmpl.interpreter.ITestResultListener;
+import org.rascalmpl.uri.URIResolverRegistry;
+import org.rascalmpl.uri.URIUtil;
+
+import io.usethesource.vallang.ISourceLocation;
+
+public class JUnitXMLReportListener implements ITestResultListener {
+    private final ISourceLocation folder;
+    private final Map<String, ISourceLocation> modules;
+    private String current = null;
+    private int tests = 0;
+    private int errors = 0;
+    private int failures = 0;
+    private int ignored = 0;
+    private long timestamp = 0L;
+    private List<Report> reports = new LinkedList<>();
+
+    public JUnitXMLReportListener(ISourceLocation outputFolder, Map<String, ISourceLocation> modules) {
+        this.folder = URIUtil.getChildLocation(outputFolder, "surefire-reports");
+        this.modules = modules;
+    }
+
+    private class Report {
+        public boolean ignored;
+        public boolean successful;
+        public String test;
+        public ISourceLocation loc;
+        public String message;
+        public Throwable exception;
+
+        public Report(boolean ignored, boolean successful, String test, ISourceLocation loc, String message, Throwable exception) {
+            this.ignored = ignored;
+            this.successful = successful;
+            this.test = test;
+            this.loc = loc;
+            this.message = message;
+            this.exception = exception;
+        }
+
+        public void write(XMLStreamWriter out) throws XMLStreamException {
+            out.writeStartElement("testcase");
+            out.writeAttribute("classname", current);
+            out.writeAttribute("name", test);
+            out.writeAttribute("file", modules.get(current).toString());
+
+            if (ignored) {
+                out.writeEmptyElement("skipped");
+                out.writeStartElement("system-out");
+                out.writeCharacters("location: " + loc);
+                out.writeEndElement(); // system-out
+            }
+            else {
+                if (!successful) {
+                    out.writeStartElement(exception != null ? "error" : "failure" );
+                    out.writeAttribute("message", message);
+                    out.writeEndElement(); // error or failure
+                }
+
+                out.writeStartElement("system-out");
+                out.writeCharacters("location: " + loc);
+                if (exception != null) {
+                    StringWriter sw = new StringWriter();
+                    exception.printStackTrace(new PrintWriter(sw));
+                    out.writeCharacters("stacktrace:\n" + sw.toString());
+                }
+                out.writeEndElement(); // system-out
+            }
+
+            out.writeEndElement(); // testcase
+        }
+    }
+
+    @Override
+    public void start(String module, int count) {
+        tests = 0;
+        errors = 0;
+        failures = 0;
+        ignored = 0;
+        timestamp = System.currentTimeMillis();
+        reports = new LinkedList<>();
+        current = module;  
+    }
+
+    private ISourceLocation targetXML(String context) {
+        return URIUtil.getChildLocation(folder, context.replaceAll("::", ".").concat(".xml"));
+    }
+
+    @Override
+    public void report(boolean successful, String test, ISourceLocation loc, String message, Throwable exception) {
+        tests++;
+        if (exception != null) {
+            errors += 1;
+        }
+        else {
+            failures += 1;
+        }
+        reports.add(new Report(false, successful, test, loc, message, exception));
+    }
+
+    @Override
+    public void ignored(String test, ISourceLocation loc) {
+        tests++;
+        ignored++;
+        reports.add(new Report(true, false, test, loc, "", null));
+    }
+
+    @Override
+    public void done() {
+        try {
+            var xmlFile = targetXML(current);
+
+            var out = XMLOutputFactory.newDefaultFactory()
+                .createXMLStreamWriter(URIResolverRegistry.getInstance().getOutputStream(xmlFile, false));
+            out.writeStartDocument();
+
+            out.writeStartElement("testsuite");
+
+            out.writeAttribute("time", Long.toString(System.currentTimeMillis() - timestamp));
+            out.writeAttribute("timestamp", Long.toString(timestamp));
+            out.writeAttribute("ignored", Integer.toString(ignored));
+            out.writeAttribute("errors", Integer.toString(errors));
+            out.writeAttribute("tests", Integer.toString(tests));
+            out.writeAttribute("failures", Integer.toString(failures));
+
+            for(Report r : reports) {
+                r.write(out);
+            }
+
+            out.writeEndElement();
+
+            out.writeEndDocument();
+            out.flush();
+            out.close();
+        }
+        catch (XMLStreamException | IOException e) {
+            System.err.println("unexpected error during test reporting");
+            throw new RuntimeException(e);
+        }
+    }
+}