View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *     http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.scxml2.w3c;
18  
19  import java.io.File;
20  import java.io.FileInputStream;
21  import java.io.FileOutputStream;
22  import java.io.FileReader;
23  import java.net.URL;
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.LinkedHashMap;
27  import java.util.List;
28  
29  import javax.xml.bind.JAXBContext;
30  import javax.xml.bind.Unmarshaller;
31  import javax.xml.bind.annotation.XmlAccessType;
32  import javax.xml.bind.annotation.XmlAccessorType;
33  import javax.xml.bind.annotation.XmlAttribute;
34  import javax.xml.bind.annotation.XmlElement;
35  import javax.xml.bind.annotation.XmlRootElement;
36  import javax.xml.bind.annotation.XmlValue;
37  import javax.xml.transform.Transformer;
38  import javax.xml.transform.TransformerFactory;
39  import javax.xml.transform.stream.StreamResult;
40  import javax.xml.transform.stream.StreamSource;
41  
42  import org.apache.commons.io.FileUtils;
43  import org.apache.commons.scxml2.PathResolver;
44  import org.apache.commons.scxml2.SCXMLExecutor;
45  import org.apache.commons.scxml2.env.Tracer;
46  import org.apache.commons.scxml2.env.URLResolver;
47  import org.apache.commons.scxml2.invoke.SimpleSCXMLInvoker;
48  import org.apache.commons.scxml2.io.SCXMLReader;
49  import org.apache.commons.scxml2.model.Final;
50  import org.apache.commons.scxml2.model.SCXML;
51  
52  /**
53   * W3C SCXML 1.0 IRP tests: <a href="http://www.w3.org/Voice/2013/scxml-irp/">http://www.w3.org/Voice/2013/scxml-irp/</a>.
54   * <p>
55   * The <b>W3CTests</b> class is standalone and can download and transform the IRP tests locally using respectively
56   * commandline parameter <b>get</b> or <b>make</b>.
57   * </p>
58   * <p>
59   * To execute one or multiple IRP tests the commandline parameter <b>run</b> must be specified.
60   * </p>
61   * <p>
62   * Optional environment parameter <b>-Ddatamodel=&lt;minimal|ecma|xpath&gt;</b> can be specified to limit the
63   * execution of the tests for and using only the specified datamodel language.
64   * </p>
65   * <p>
66   * Optional environment parameter <b>-Dtest=&lt;testId&gt;</b> can be specified to only execute a single test, which
67   * also can be combined with the <b>-Ddatamodel</b> parameter.
68   * </p>
69   * <p>
70   * The W3CTests also uses a separate <b><code>tests.xml</code></b> configuration file, located in the
71   * <b><code>src/test/resources/w3c</code></b> directory, which is manually maintained to enable|disable execution
72   * of tests (when <em>not</em> using the <b>-Dtest</b> parameter, which will always execute the specified test).<br/>
73   * Furthermore, in this configuration file the current <em>success</em> or <em>failure</em> status, and even more
74   * meta data per test is maintained.
75   * </p>
76   */
77  @SuppressWarnings("unused")
78  public class W3CTests {
79  
80      private static final String SCXML_IRP_BASE_URL = "http://www.w3.org/Voice/2013/scxml-irp/";
81      private static final String SCXML_IRP_MANIFEST_URI = "manifest.xml";
82      private static final String SCXML_IRP_ECMA_XSL_URI = "confEcma.xsl";
83      private static final String SCXML_IRP_XPATH_XSL_URI = "confXpath.xsl";
84  
85      private static final String TESTS_SRC_DIR = "src/w3c/scxml-irp/";
86      private static final String TXML_TESTS_DIR = TESTS_SRC_DIR + "txml/";
87      private static final String MINIMAL_TESTS_DIR = TESTS_SRC_DIR + "minimal/";
88      private static final String ECMA_TESTS_DIR = TESTS_SRC_DIR + "ecma/";
89      private static final String XPATH_TESTS_DIR = TESTS_SRC_DIR + "xpath/";
90      private static final String PACKAGE_PATH = "/"+W3CTests.class.getPackage().getName().replace('.','/');
91      private static final String TESTS_FILENAME = PACKAGE_PATH + "/tests.xml";
92      private static final String SCXML_IRP_MINIMAL_XSL_FILENAME = PACKAGE_PATH + "/confMinimal.xsl";
93  
94      /**
95       * Tests model class used for loading the <b>tests.xml</b> configuration file
96       */
97      @XmlRootElement(name="tests")
98      @XmlAccessorType(XmlAccessType.FIELD)
99      protected static class Tests {
100 
101         @XmlAccessorType(XmlAccessType.FIELD)
102         protected static class Test {
103 
104             @XmlAttribute(required=true)
105             private String id;
106             @XmlAttribute(required=true)
107             private Boolean mandatory;
108             @XmlAttribute(required=true)
109             private Boolean manual;
110             @XmlAttribute(required=true)
111             private boolean enabled;
112             @XmlAttribute
113             private String finalId;
114             @XmlAttribute
115             private Boolean implemented;
116             @XmlAttribute(name="minimal")
117             String minimalStatus;
118             @XmlAttribute(name="ecma")
119             String ecmaStatus;
120             @XmlAttribute(name="xpath")
121             String xpathStatus;
122             @XmlAttribute
123             Boolean xpathEnabled;
124             @XmlValue
125             private String comment;
126 
127             public String getId() {
128                 return id;
129             }
130 
131             public boolean isMandatory() {
132                 return mandatory;
133             }
134 
135             public boolean isManual() {
136                 return manual == null || manual;
137             }
138 
139             public boolean isEnabled() {
140                 return enabled;
141             }
142 
143             public String getFinalState() {
144                 return finalId;
145             }
146 
147             public boolean isImplemented() {
148                 return implemented == null || implemented;
149             }
150 
151             public String getMinimalStatus() {
152                 return minimalStatus;
153             }
154 
155             public String getEcmaStatus() {
156                 return ecmaStatus;
157             }
158 
159             public String getXpathStatus() {
160                 return xpathStatus;
161             }
162 
163             public boolean isXPathEnabled() {
164                 return xpathEnabled == null || xpathEnabled;
165             }
166 
167             public String getComment() {
168                 return comment;
169             }
170 
171             public String toString() {
172                 return id;
173             }
174         }
175 
176         @XmlElement(name="test")
177         private ArrayList<Test> tests;
178 
179         private LinkedHashMap<String, Test> testsMap;
180 
181         public LinkedHashMap<String, Test> getTests() {
182             if (testsMap == null) {
183                 testsMap = new LinkedHashMap<String, Test>();
184                 if (tests != null) {
185                     for (Test t : tests) {
186                         testsMap.put(t.getId(), t);
187                     }
188                 }
189             }
190             return testsMap;
191         }
192     }
193 
194     /**
195      * Datamodel enum representing the minimal, ecma and xpath datamodel types used and tested by the W3C IRP tests.
196      */
197     protected enum Datamodel {
198 
199         MINIMAL("minimal"),
200         ECMA("ecma"),
201         XPATH("xpath");
202 
203         private final String value;
204 
205         private Datamodel(final String value) {
206             this.value = value;
207         }
208 
209         public String value() {
210             return value;
211         }
212 
213         public static Datamodel fromValue(final String value) {
214             for (Datamodel datamodel : Datamodel.values()) {
215                 if (datamodel.value().equals(value)) {
216                     return datamodel;
217                 }
218             }
219             return null;
220         }
221     }
222 
223     /**
224      * Assertions model class used for loading the W3C IRP tests manifest.xml file, defining the meta data and
225      * source URIs for all the W3C IRP tests.
226      */
227     @XmlRootElement(name="assertions")
228     @XmlAccessorType(XmlAccessType.FIELD)
229     protected static class Assertions {
230 
231         @XmlAccessorType(XmlAccessType.FIELD)
232         protected static class Assertion {
233 
234             @XmlAttribute
235             private String id;
236             @XmlAttribute(name="specnum")
237             private String specnum;
238             @XmlAttribute(name="specid")
239             private String specid;
240             @XmlElement(name="test")
241             private ArrayList<TestCase> testCases;
242 
243             public String getId() {
244                 return id;
245             }
246 
247             public String getSpecNum() {
248                 return specnum;
249             }
250 
251             public String getSpecId() {
252                 return specid;
253             }
254 
255             public List<TestCase> getTestCases() {
256                 return testCases != null ? testCases : Collections.<TestCase>emptyList();
257             }
258 
259             public Datamodel getDatamodel() {
260                 if ("#minimal-profile".equals(specid)) {
261                     return Datamodel.MINIMAL;
262                 }
263                 else if ("#ecma-profile".equals(specid)) {
264                     return Datamodel.ECMA;
265                 }
266                 else if ("#xpath-profile".equals(specid)) {
267                     return Datamodel.XPATH;
268                 }
269                 return null;
270             }
271 
272             public String toString() {
273                 return id;
274             }
275         }
276 
277         @XmlAccessorType(XmlAccessType.FIELD)
278         protected static class TestCase {
279 
280             @XmlAttribute
281             private String id;
282             @XmlAttribute
283             private String manual;
284             @XmlAttribute
285             private String conformance;
286             @XmlElement(name="start")
287             private ArrayList<Resource> scxmlResources;
288             @XmlElement(name="dep")
289             private ArrayList<Resource> depResources;
290 
291             private ArrayList<Resource> resources;
292 
293             public String getId() {
294                 return id;
295             }
296 
297             public boolean isManual() {
298                 return Boolean.parseBoolean(manual);
299             }
300 
301             public boolean isOptional() {
302                 return "mandatory".equals(conformance);
303             }
304 
305             public List<Resource> getScxmlResources() {
306                 return scxmlResources != null ? scxmlResources : Collections.<Resource>emptyList();
307             }
308 
309             public List<Resource> getResources() {
310                 if (resources == null) {
311                     resources = new ArrayList<Resource>();
312                     if (scxmlResources != null) {
313                         resources.addAll(scxmlResources);
314                     }
315                     if (depResources != null) {
316                         resources.addAll(depResources);
317                         // no longer needed
318                         depResources = null;
319                     }
320                 }
321                 return resources;
322             }
323         }
324 
325         @XmlAccessorType(XmlAccessType.FIELD)
326         protected static class Resource {
327 
328             @XmlAttribute
329             private String uri;
330 
331             public String getUri() {
332                 return uri;
333             }
334 
335             public String getName() {
336                 return uri.substring(uri.indexOf("/")+1, uri.indexOf("."));
337             }
338 
339             public String getFilename() {
340                 return uri.substring(uri.indexOf("/")+1);
341             }
342         }
343 
344         @XmlElement(name="assert")
345         private ArrayList<Assertion> assertions;
346 
347         private LinkedHashMap<String, Assertion> assertionsMap;
348 
349         public LinkedHashMap<String, Assertion> getAssertions() {
350             if (assertionsMap == null) {
351                 assertionsMap = new LinkedHashMap<String, Assertion>();
352                 if (assertions != null) {
353                     for (Assertion a : assertions) {
354                         assertionsMap.put(a.getId(), a);
355                     }
356                 }
357             }
358             return assertionsMap;
359         }
360     }
361 
362     /**
363      * Simple TestResult data struct for tracking test results
364      */
365     protected static class TestResults {
366         int testsSkipped;
367         int testsPassed;
368         int testsFailed;
369         int minimalPassed;
370         int minimalFailed;
371         int ecmaPassed;
372         int ecmaFailed;
373         int xpathPassed;
374         int xpathFailed;
375         ArrayList<String> failedTests = new ArrayList<String>();
376     }
377 
378     /**
379      * W3CTests main function, see {@link #usage()} how to use.
380      * @param args
381      * @throws Exception
382      */
383     public static void main(final String[] args) throws Exception {
384         if (args.length > 0) {
385             if ("get".equals(args[0])) {
386                 new W3CTests().getTests();
387                 return;
388             }
389             else if ("make".equals(args[0])) {
390                 new W3CTests().makeTests();
391                 return;
392             }
393             else if ("run".equals(args[0])) {
394                 Datamodel datamodel = Datamodel.fromValue(System.getProperty("datamodel"));
395                 String testId = System.getProperty("test");
396                 new W3CTests().runTests(testId, datamodel);
397                 return;
398             }
399         }
400         usage();
401     }
402 
403     /**
404      * Usage prints the 'commandline' usage options.
405      */
406     protected static void usage() {
407         System.out.println("Usage: W3CTests <get|run>\n" +
408                 "  get  - downloads the W3C IRP tests\n" +
409                 "  make - make previously downloaded  W3C IRP tests by transforming the .txml templates\n" +
410                 "  run  - runs test(s), optionally only for a specific datamodel (default: all)\n\n" +
411                 "To run a single test, specify -Dtest=<testId>, otherwise all enabled tests will be run.\n" +
412                 "To only run test(s) for a specific datamodel, specify -Ddatamodel=<minimal|ecma|xpath>.\n");
413     }
414 
415     /**
416      * Downloads the W3C IRP manifest.xml, the IRP ecma and xpath stylesheets to transform the tests, and the
417      * actual test templates (.txml) as defined in the manifest.xml
418      * @throws Exception
419      */
420     protected void getTests() throws Exception {
421         final File testsSrcDir = new File(TESTS_SRC_DIR);
422         if (!testsSrcDir.mkdirs()) {
423             FileUtils.cleanDirectory(testsSrcDir);
424         }
425         new File(TXML_TESTS_DIR).mkdirs();
426         new File(MINIMAL_TESTS_DIR).mkdirs();
427         new File(ECMA_TESTS_DIR).mkdirs();
428         new File(XPATH_TESTS_DIR).mkdirs();
429         System.out.println("Downloading IRP manifest: " + SCXML_IRP_BASE_URL + SCXML_IRP_MANIFEST_URI);
430         FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + SCXML_IRP_MANIFEST_URI), new File(testsSrcDir, SCXML_IRP_MANIFEST_URI));
431         System.out.println("Downloading ecma stylesheet: " + SCXML_IRP_BASE_URL + SCXML_IRP_ECMA_XSL_URI);
432         FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + SCXML_IRP_ECMA_XSL_URI), new File(testsSrcDir, SCXML_IRP_ECMA_XSL_URI));
433         System.out.println("Downloading xpath stylesheet: " + SCXML_IRP_BASE_URL + SCXML_IRP_XPATH_XSL_URI);
434         FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + SCXML_IRP_XPATH_XSL_URI), new File(testsSrcDir, SCXML_IRP_XPATH_XSL_URI));
435         Assertions assertions = loadAssertions();
436         for (Assertions.Assertion entry : assertions.getAssertions().values()) {
437             for (Assertions.TestCase test : entry.getTestCases()) {
438                 for (Assertions.Resource resource : test.getResources()) {
439                     System.out.println("Downloading IRP test file: " + SCXML_IRP_BASE_URL + resource.getUri());
440                     FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + resource.getUri()), new File(TXML_TESTS_DIR + resource.getFilename()));
441                 }
442             }
443         }
444     }
445 
446     /**
447      * Transforms the W3C IRP tests.
448      * <p>
449      * Note: for transforming the IRP .txml test files XPath 2.0 is required, for which the Saxon library is used.
450      * </p>
451      * @throws Exception
452      */
453     protected void makeTests() throws Exception {
454         final File testsSrcDir = new File(TESTS_SRC_DIR);
455 
456         TransformerFactory factory = TransformerFactory.newInstance("net.sf.saxon.TransformerFactoryImpl",null);
457         factory.setFeature("http://saxon.sf.net/feature/suppressXsltNamespaceCheck", true);
458         Transformer ecmaTransformer = factory.newTransformer(new StreamSource(new FileInputStream(new File(testsSrcDir, SCXML_IRP_ECMA_XSL_URI))));
459         Transformer xpathTransformer = factory.newTransformer(new StreamSource(new FileInputStream(new File(testsSrcDir, SCXML_IRP_XPATH_XSL_URI))));
460         Transformer minimalTransformer = factory.newTransformer(new StreamSource(getClass().getResourceAsStream(SCXML_IRP_MINIMAL_XSL_FILENAME)));
461         Assertions assertions = loadAssertions();
462         for (Assertions.Assertion entry : assertions.getAssertions().values()) {
463             for (Assertions.TestCase test : entry.getTestCases()) {
464                 for (Assertions.Resource resource : test.getResources()) {
465                     processResource(entry.getSpecId(), resource, minimalTransformer, ecmaTransformer, xpathTransformer);
466                 }
467             }
468         }
469     }
470 
471     /**
472      * Unmarshall and return the W3C IRP tests manifest.xml
473      * @return an Assertions instance reprenting the W3C IRP tests manifest.xml
474      * @throws Exception
475      */
476     protected Assertions loadAssertions() throws Exception {
477         final JAXBContext jaxbContext = JAXBContext.newInstance(Assertions.class);
478         final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
479         return (Assertions)jaxbUnmarshaller.unmarshal(new File(TESTS_SRC_DIR, SCXML_IRP_MANIFEST_URI));
480     }
481 
482     /**
483      * Download and transform a W3C IRP test resource file
484      * @param specid the SCXML 1.0 spec id (anchor) for the current assertion,
485      *               which is used to determine if, how and where the resource should be transformed.
486      * @param resource The test resource definition
487      * @param minimalTransformer transformer to produce an minimal datamodel SCXML document from the txml resource
488      * @param ecmaTransformer transformer to produce an ecmascript datamodel SCXML document from the txml resource
489      * @param xpathTransformer transformer to produce a xpath datamodel based SCXML document from the txml resource
490      * @throws Exception
491      */
492     protected void processResource(final String specid, final Assertions.Resource resource,
493                                    final Transformer minimalTransformer, final Transformer ecmaTransformer,
494                                    final Transformer xpathTransformer)
495             throws Exception {
496         System.out.println("processing IRP test file " + resource.getFilename());
497         FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + resource.getUri()), new File(TXML_TESTS_DIR + resource.getFilename()));
498         if (specid.equals("#minimal-profile")) {
499             transformResource(resource, minimalTransformer, MINIMAL_TESTS_DIR);
500         }
501         else if (specid.equals("#ecma-profile")) {
502             transformResource(resource, ecmaTransformer, ECMA_TESTS_DIR);
503         }
504         else if (specid.equals("#xpath-profile")) {
505             transformResource(resource, xpathTransformer, XPATH_TESTS_DIR);
506         }
507         else {
508             transformResource(resource, ecmaTransformer, ECMA_TESTS_DIR);
509             transformResource(resource, xpathTransformer, XPATH_TESTS_DIR);
510         }
511     }
512 
513     /**
514      * XSL transform a W3C IRP test SCXML resource to a datamodel specific location and format,
515      * or simply copy a non SCXML resource to that location.
516      * @param resource the test resource definition
517      * @param transformer the XSL transformer to use
518      * @param targetDir the target location for the transformed SCXML document, or the non-SCXML resource
519      * @throws Exception
520      */
521     protected void transformResource(final Assertions.Resource resource, final Transformer transformer,
522                                      final String targetDir) throws Exception {
523         if (resource.getFilename().endsWith(".txml")) {
524             StreamSource txmlSource = new StreamSource(new FileInputStream(new File(TXML_TESTS_DIR, resource.getFilename())));
525             transformer.transform(txmlSource, new StreamResult(new FileOutputStream(new File(targetDir, resource.getName() + ".scxml"))));
526         }
527         else {
528             FileUtils.copyFile(new File(TXML_TESTS_DIR, resource.getFilename()), new File(targetDir, resource.getFilename()));
529         }
530     }
531 
532     protected void createCleanDirectory(final String path) throws Exception {
533         final File dir = new File(path);
534         if (!dir.mkdirs()) {
535             FileUtils.cleanDirectory(dir);
536         }
537     }
538 
539     /**
540      * Run one or multiple W3C IRP tests
541      * @param testId a W3C IRP test id, or null to specify all tests to run
542      * @param datamodel only tests available for or executable with the specified datamodel will be run (or all if null)
543      * @throws Exception
544      */
545     protected void runTests(final String testId, final Datamodel datamodel) throws Exception {
546         final Assertions assertions = loadAssertions();
547         final Tests tests = loadTests();
548         final TestResults results = new TestResults();
549         if (testId != null) {
550             final Assertions.Assertion assertion = assertions.getAssertions().get(testId);
551             if (assertion != null) {
552                 runTest(assertion, tests, datamodel, true, results);
553             }
554             else {
555                 throw new IllegalArgumentException("Unknown test with id: "+testId);
556             }
557         }
558         else {
559             for (Assertions.Assertion entry : assertions.getAssertions().values()) {
560                 runTest(entry, tests, datamodel, false, results);
561             }
562         }
563         System.out.println(
564                 "\nTest results running " +
565                 (testId == null ? "all enabled tests" : "test "+testId) +
566                 (datamodel != null ? " for the "+datamodel.value+" datamodel" : "") +
567                 ":\n" +
568                 "  number of tests    : "+(results.testsSkipped+results.testsPassed+results.testsFailed) +
569                    " ("+results.testsPassed+" passed,  "+results.testsFailed +" failed,  "+results.testsSkipped+" skipped)");
570         if (results.minimalPassed+results.minimalFailed > 0) {
571             System.out.println(
572                     "    mimimal datamodel: "+results.minimalPassed+" passed,  "+results.minimalFailed+" failed");
573         }
574         if (results.ecmaPassed+results.ecmaFailed > 0) {
575             System.out.println(
576                     "    ecma    datamodel: "+results.ecmaPassed+" passed,  "+results.ecmaFailed+" failed");
577         }
578         if (results.xpathPassed+results.xpathFailed > 0) {
579             System.out.println(
580                     "    xpath   datamodel: "+results.xpathPassed+" passed,  "+results.xpathFailed+" failed");
581         }
582         System.out.print("\n");
583         if (!results.failedTests.isEmpty()) {
584             System.out.println("  failed tests: ");
585             for (String filename : results.failedTests) {
586                 System.out.println("    "+filename);
587             }
588             System.out.print("\n");
589         }
590     }
591 
592     /**
593      * Loads the tests.xml configuration file into a Tests class configuration model instance.
594      * @return a Tests instance for the tests.xml configuration file.
595      * @throws Exception
596      */
597     protected Tests loadTests() throws Exception {
598         final JAXBContext jaxbContext = JAXBContext.newInstance(Tests.class);
599         final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
600         return (Tests)jaxbUnmarshaller.unmarshal(getClass().getResource(TESTS_FILENAME));
601     }
602 
603     /**
604      * Run a single W3C IRP test (assert)
605      * @param assertion The W3C IRP assert, defining one or more {@link Assertions.TestCase}s
606      * @param tests the tests configurations
607      * @param datamodel the datamodel to limit and restrict the execution of the test
608      * @param singleTest if true a single test id was specified which will be executed even if disabled in the configuration.
609      * @throws Exception
610      */
611     protected void runTest(final Assertions.Assertion assertion, final Tests tests, final Datamodel datamodel,
612                            final boolean singleTest, TestResults results) throws Exception {
613         final Tests.Test test = tests.getTests().get(assertion.getId());
614         if (test == null) {
615             throw new IllegalStateException("No test configuration found for W3C IRP test with id: "+assertion.getId());
616         }
617         boolean skipped = true;
618         boolean passed = true;
619         if (singleTest || test.isEnabled()) {
620             if (datamodel != Datamodel.MINIMAL || datamodel.equals(assertion.getDatamodel())) {
621                 if (datamodel == null || assertion.getDatamodel() == null || datamodel.equals(assertion.getDatamodel())) {
622                     final Datamodel effectiveDM = datamodel != null ? datamodel : assertion.getDatamodel();
623                     for (Assertions.TestCase testCase : assertion.getTestCases()) {
624                         if (effectiveDM != null) {
625                             switch (effectiveDM) {
626                                 case MINIMAL:
627                                     skipped = false;
628                                     if (runTests(assertion, testCase, test, MINIMAL_TESTS_DIR, results.failedTests)) {
629                                         results.minimalPassed++;
630                                     }
631                                     else {
632                                         passed = false;
633                                         results.minimalFailed++;
634                                     }
635                                     break;
636                                 case ECMA:
637                                     skipped = false;
638                                     if (runTests(assertion, testCase, test, ECMA_TESTS_DIR, results.failedTests)) {
639                                         results.ecmaPassed++;
640                                     }
641                                     else {
642                                         passed = false;
643                                         results.ecmaFailed++;
644                                     }
645                                     break;
646                                 case XPATH:
647                                     if (test.isXPathEnabled()) {
648                                         skipped = false;
649                                         if (runTests(assertion, testCase, test, XPATH_TESTS_DIR, results.failedTests)) {
650                                             results.xpathPassed++;
651                                         }
652                                         else {
653                                             passed = false;
654                                             results.xpathFailed++;
655                                         }
656                                     }
657                                     break;
658                             }
659                         }
660                         else {
661                             skipped = false;
662                             if (runTests(assertion, testCase, test, ECMA_TESTS_DIR, results.failedTests)) {
663                                 results.ecmaPassed++;
664                             }
665                             else {
666                                 passed = false;
667                                 results.ecmaFailed++;
668                             }
669                             if (test.isXPathEnabled()) {
670                                 if (runTests(assertion, testCase, test, XPATH_TESTS_DIR, results.failedTests)) {
671                                     results.xpathPassed++;
672                                 }
673                                 else {
674                                     passed = false;
675                                     results.xpathFailed++;
676                                 }
677                             }
678                         }
679                     }
680                 }
681             }
682         }
683         if (skipped) {
684             results.testsSkipped++;
685         }
686         else if (passed) {
687             results.testsPassed++;
688         }
689         else {
690             results.testsFailed++;
691         }
692     }
693 
694     /**
695      * Execute all W3C IRP SCXML tests for a specific {@link Assertions.TestCase}
696      * @param assertion the W3C IRP test assert definition
697      * @param testCase the W3C IRP test definition
698      * @param test the test configuration
699      * @param scxmlDir the datamodel specific directory path containing the SCXML document(s)
700      * @throws Exception
701      */
702     protected boolean runTests(final Assertions.Assertion assertion, final Assertions.TestCase testCase,
703                                final Tests.Test test, final String scxmlDir, ArrayList<String> failedTests)
704             throws Exception {
705         boolean passed = true;
706         for (Assertions.Resource scxmlResource : testCase.getScxmlResources()) {
707             File scxmlFile = new File(scxmlDir, scxmlResource.getName()+".scxml");
708             if (!runTest(testCase, test, scxmlFile)) {
709                 passed = false;
710                 failedTests.add(scxmlFile.getParentFile().getName()+"/"+scxmlFile.getName());
711             }
712         }
713         return passed;
714     }
715 
716     /**
717      * Run a single W3C IRP SCXML test
718      * @param testCase the W3C IRP test definition
719      * @param test the test configuration
720      * @param scxmlFile the file handle for the SCXML document
721      */
722     protected boolean runTest(final Assertions.TestCase testCase, final Tests.Test test, final File scxmlFile) {
723         try {
724             System.out.println("Executing test: "+scxmlFile.getParentFile().getName()+"/"+scxmlFile.getName());
725             final Tracer trc = new Tracer();
726             final PathResolver pathResolver = new URLResolver(scxmlFile.getParentFile().toURI().toURL());
727             final SCXMLReader.Configuration configuration = new SCXMLReader.Configuration(null, pathResolver);
728             final SCXML doc = SCXMLReader.read(new FileReader(scxmlFile), configuration);
729             if (doc == null) {
730                 System.out.println("                FAIL: the SCXML file " +
731                         scxmlFile.getCanonicalPath() + " can not be parsed!");
732                 return false;
733             }
734             final SCXMLExecutor exec = new SCXMLExecutor(null, null, trc);
735             exec.setSingleContext(true);
736             exec.setStateMachine(doc);
737             exec.addListener(doc, trc);
738             exec.registerInvokerClass("scxml", SimpleSCXMLInvoker.class);
739             exec.registerInvokerClass("http://www.w3.org/TR/scxml/", SimpleSCXMLInvoker.class);
740             exec.go();
741             Final end;
742             while ((end = exec.getStatus().getFinalState()) == null) {
743                 Thread.sleep(100);
744                 exec.triggerEvents();
745             }
746             System.out.println("                final state: "+end.getId());
747             if (!testCase.isManual()) {
748                 return end.getId().equals("pass");
749             }
750             else if (test.getFinalState() != null) {
751                 return end.getId().equals(test.getFinalState());
752             }
753             else {
754                 // todo: manual verification for specific tests
755                 return false;
756             }
757         }
758         catch (Exception e) {
759             System.out.println("                FAIL: "+e.getMessage());
760             return false;
761         }
762     }
763 }