1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.commons.scxml2.w3c;
18
19 import java.io.File;
20 import java.io.FileInputStream;
21 import java.io.FileOutputStream;
22 import java.io.FileReader;
23 import java.net.URL;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.LinkedHashMap;
27 import java.util.List;
28
29 import javax.xml.bind.JAXBContext;
30 import javax.xml.bind.Unmarshaller;
31 import javax.xml.bind.annotation.XmlAccessType;
32 import javax.xml.bind.annotation.XmlAccessorType;
33 import javax.xml.bind.annotation.XmlAttribute;
34 import javax.xml.bind.annotation.XmlElement;
35 import javax.xml.bind.annotation.XmlRootElement;
36 import javax.xml.bind.annotation.XmlValue;
37 import javax.xml.transform.Transformer;
38 import javax.xml.transform.TransformerFactory;
39 import javax.xml.transform.stream.StreamResult;
40 import javax.xml.transform.stream.StreamSource;
41
42 import org.apache.commons.io.FileUtils;
43 import org.apache.commons.scxml2.PathResolver;
44 import org.apache.commons.scxml2.SCXMLExecutor;
45 import org.apache.commons.scxml2.env.Tracer;
46 import org.apache.commons.scxml2.env.URLResolver;
47 import org.apache.commons.scxml2.invoke.SimpleSCXMLInvoker;
48 import org.apache.commons.scxml2.io.SCXMLReader;
49 import org.apache.commons.scxml2.model.Final;
50 import org.apache.commons.scxml2.model.SCXML;
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77 @SuppressWarnings("unused")
78 public class W3CTests {
79
80 private static final String SCXML_IRP_BASE_URL = "http://www.w3.org/Voice/2013/scxml-irp/";
81 private static final String SCXML_IRP_MANIFEST_URI = "manifest.xml";
82 private static final String SCXML_IRP_ECMA_XSL_URI = "confEcma.xsl";
83 private static final String SCXML_IRP_XPATH_XSL_URI = "confXpath.xsl";
84
85 private static final String TESTS_SRC_DIR = "src/w3c/scxml-irp/";
86 private static final String TXML_TESTS_DIR = TESTS_SRC_DIR + "txml/";
87 private static final String MINIMAL_TESTS_DIR = TESTS_SRC_DIR + "minimal/";
88 private static final String ECMA_TESTS_DIR = TESTS_SRC_DIR + "ecma/";
89 private static final String XPATH_TESTS_DIR = TESTS_SRC_DIR + "xpath/";
90 private static final String PACKAGE_PATH = "/"+W3CTests.class.getPackage().getName().replace('.','/');
91 private static final String TESTS_FILENAME = PACKAGE_PATH + "/tests.xml";
92 private static final String SCXML_IRP_MINIMAL_XSL_FILENAME = PACKAGE_PATH + "/confMinimal.xsl";
93
94
95
96
97 @XmlRootElement(name="tests")
98 @XmlAccessorType(XmlAccessType.FIELD)
99 protected static class Tests {
100
101 @XmlAccessorType(XmlAccessType.FIELD)
102 protected static class Test {
103
104 @XmlAttribute(required=true)
105 private String id;
106 @XmlAttribute(required=true)
107 private Boolean mandatory;
108 @XmlAttribute(required=true)
109 private Boolean manual;
110 @XmlAttribute(required=true)
111 private boolean enabled;
112 @XmlAttribute
113 private String finalId;
114 @XmlAttribute
115 private Boolean implemented;
116 @XmlAttribute(name="minimal")
117 String minimalStatus;
118 @XmlAttribute(name="ecma")
119 String ecmaStatus;
120 @XmlAttribute(name="xpath")
121 String xpathStatus;
122 @XmlAttribute
123 Boolean xpathEnabled;
124 @XmlValue
125 private String comment;
126
127 public String getId() {
128 return id;
129 }
130
131 public boolean isMandatory() {
132 return mandatory;
133 }
134
135 public boolean isManual() {
136 return manual == null || manual;
137 }
138
139 public boolean isEnabled() {
140 return enabled;
141 }
142
143 public String getFinalState() {
144 return finalId;
145 }
146
147 public boolean isImplemented() {
148 return implemented == null || implemented;
149 }
150
151 public String getMinimalStatus() {
152 return minimalStatus;
153 }
154
155 public String getEcmaStatus() {
156 return ecmaStatus;
157 }
158
159 public String getXpathStatus() {
160 return xpathStatus;
161 }
162
163 public boolean isXPathEnabled() {
164 return xpathEnabled == null || xpathEnabled;
165 }
166
167 public String getComment() {
168 return comment;
169 }
170
171 public String toString() {
172 return id;
173 }
174 }
175
176 @XmlElement(name="test")
177 private ArrayList<Test> tests;
178
179 private LinkedHashMap<String, Test> testsMap;
180
181 public LinkedHashMap<String, Test> getTests() {
182 if (testsMap == null) {
183 testsMap = new LinkedHashMap<String, Test>();
184 if (tests != null) {
185 for (Test t : tests) {
186 testsMap.put(t.getId(), t);
187 }
188 }
189 }
190 return testsMap;
191 }
192 }
193
194
195
196
197 protected enum Datamodel {
198
199 MINIMAL("minimal"),
200 ECMA("ecma"),
201 XPATH("xpath");
202
203 private final String value;
204
205 private Datamodel(final String value) {
206 this.value = value;
207 }
208
209 public String value() {
210 return value;
211 }
212
213 public static Datamodel fromValue(final String value) {
214 for (Datamodel datamodel : Datamodel.values()) {
215 if (datamodel.value().equals(value)) {
216 return datamodel;
217 }
218 }
219 return null;
220 }
221 }
222
223
224
225
226
227 @XmlRootElement(name="assertions")
228 @XmlAccessorType(XmlAccessType.FIELD)
229 protected static class Assertions {
230
231 @XmlAccessorType(XmlAccessType.FIELD)
232 protected static class Assertion {
233
234 @XmlAttribute
235 private String id;
236 @XmlAttribute(name="specnum")
237 private String specnum;
238 @XmlAttribute(name="specid")
239 private String specid;
240 @XmlElement(name="test")
241 private ArrayList<TestCase> testCases;
242
243 public String getId() {
244 return id;
245 }
246
247 public String getSpecNum() {
248 return specnum;
249 }
250
251 public String getSpecId() {
252 return specid;
253 }
254
255 public List<TestCase> getTestCases() {
256 return testCases != null ? testCases : Collections.<TestCase>emptyList();
257 }
258
259 public Datamodel getDatamodel() {
260 if ("#minimal-profile".equals(specid)) {
261 return Datamodel.MINIMAL;
262 }
263 else if ("#ecma-profile".equals(specid)) {
264 return Datamodel.ECMA;
265 }
266 else if ("#xpath-profile".equals(specid)) {
267 return Datamodel.XPATH;
268 }
269 return null;
270 }
271
272 public String toString() {
273 return id;
274 }
275 }
276
277 @XmlAccessorType(XmlAccessType.FIELD)
278 protected static class TestCase {
279
280 @XmlAttribute
281 private String id;
282 @XmlAttribute
283 private String manual;
284 @XmlAttribute
285 private String conformance;
286 @XmlElement(name="start")
287 private ArrayList<Resource> scxmlResources;
288 @XmlElement(name="dep")
289 private ArrayList<Resource> depResources;
290
291 private ArrayList<Resource> resources;
292
293 public String getId() {
294 return id;
295 }
296
297 public boolean isManual() {
298 return Boolean.parseBoolean(manual);
299 }
300
301 public boolean isOptional() {
302 return "mandatory".equals(conformance);
303 }
304
305 public List<Resource> getScxmlResources() {
306 return scxmlResources != null ? scxmlResources : Collections.<Resource>emptyList();
307 }
308
309 public List<Resource> getResources() {
310 if (resources == null) {
311 resources = new ArrayList<Resource>();
312 if (scxmlResources != null) {
313 resources.addAll(scxmlResources);
314 }
315 if (depResources != null) {
316 resources.addAll(depResources);
317
318 depResources = null;
319 }
320 }
321 return resources;
322 }
323 }
324
325 @XmlAccessorType(XmlAccessType.FIELD)
326 protected static class Resource {
327
328 @XmlAttribute
329 private String uri;
330
331 public String getUri() {
332 return uri;
333 }
334
335 public String getName() {
336 return uri.substring(uri.indexOf("/")+1, uri.indexOf("."));
337 }
338
339 public String getFilename() {
340 return uri.substring(uri.indexOf("/")+1);
341 }
342 }
343
344 @XmlElement(name="assert")
345 private ArrayList<Assertion> assertions;
346
347 private LinkedHashMap<String, Assertion> assertionsMap;
348
349 public LinkedHashMap<String, Assertion> getAssertions() {
350 if (assertionsMap == null) {
351 assertionsMap = new LinkedHashMap<String, Assertion>();
352 if (assertions != null) {
353 for (Assertion a : assertions) {
354 assertionsMap.put(a.getId(), a);
355 }
356 }
357 }
358 return assertionsMap;
359 }
360 }
361
362
363
364
365 protected static class TestResults {
366 int testsSkipped;
367 int testsPassed;
368 int testsFailed;
369 int minimalPassed;
370 int minimalFailed;
371 int ecmaPassed;
372 int ecmaFailed;
373 int xpathPassed;
374 int xpathFailed;
375 ArrayList<String> failedTests = new ArrayList<String>();
376 }
377
378
379
380
381
382
383 public static void main(final String[] args) throws Exception {
384 if (args.length > 0) {
385 if ("get".equals(args[0])) {
386 new W3CTests().getTests();
387 return;
388 }
389 else if ("make".equals(args[0])) {
390 new W3CTests().makeTests();
391 return;
392 }
393 else if ("run".equals(args[0])) {
394 Datamodel datamodel = Datamodel.fromValue(System.getProperty("datamodel"));
395 String testId = System.getProperty("test");
396 new W3CTests().runTests(testId, datamodel);
397 return;
398 }
399 }
400 usage();
401 }
402
403
404
405
406 protected static void usage() {
407 System.out.println("Usage: W3CTests <get|run>\n" +
408 " get - downloads the W3C IRP tests\n" +
409 " make - make previously downloaded W3C IRP tests by transforming the .txml templates\n" +
410 " run - runs test(s), optionally only for a specific datamodel (default: all)\n\n" +
411 "To run a single test, specify -Dtest=<testId>, otherwise all enabled tests will be run.\n" +
412 "To only run test(s) for a specific datamodel, specify -Ddatamodel=<minimal|ecma|xpath>.\n");
413 }
414
415
416
417
418
419
420 protected void getTests() throws Exception {
421 final File testsSrcDir = new File(TESTS_SRC_DIR);
422 if (!testsSrcDir.mkdirs()) {
423 FileUtils.cleanDirectory(testsSrcDir);
424 }
425 new File(TXML_TESTS_DIR).mkdirs();
426 new File(MINIMAL_TESTS_DIR).mkdirs();
427 new File(ECMA_TESTS_DIR).mkdirs();
428 new File(XPATH_TESTS_DIR).mkdirs();
429 System.out.println("Downloading IRP manifest: " + SCXML_IRP_BASE_URL + SCXML_IRP_MANIFEST_URI);
430 FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + SCXML_IRP_MANIFEST_URI), new File(testsSrcDir, SCXML_IRP_MANIFEST_URI));
431 System.out.println("Downloading ecma stylesheet: " + SCXML_IRP_BASE_URL + SCXML_IRP_ECMA_XSL_URI);
432 FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + SCXML_IRP_ECMA_XSL_URI), new File(testsSrcDir, SCXML_IRP_ECMA_XSL_URI));
433 System.out.println("Downloading xpath stylesheet: " + SCXML_IRP_BASE_URL + SCXML_IRP_XPATH_XSL_URI);
434 FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + SCXML_IRP_XPATH_XSL_URI), new File(testsSrcDir, SCXML_IRP_XPATH_XSL_URI));
435 Assertions assertions = loadAssertions();
436 for (Assertions.Assertion entry : assertions.getAssertions().values()) {
437 for (Assertions.TestCase test : entry.getTestCases()) {
438 for (Assertions.Resource resource : test.getResources()) {
439 System.out.println("Downloading IRP test file: " + SCXML_IRP_BASE_URL + resource.getUri());
440 FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + resource.getUri()), new File(TXML_TESTS_DIR + resource.getFilename()));
441 }
442 }
443 }
444 }
445
446
447
448
449
450
451
452
453 protected void makeTests() throws Exception {
454 final File testsSrcDir = new File(TESTS_SRC_DIR);
455
456 TransformerFactory factory = TransformerFactory.newInstance("net.sf.saxon.TransformerFactoryImpl",null);
457 factory.setFeature("http://saxon.sf.net/feature/suppressXsltNamespaceCheck", true);
458 Transformer ecmaTransformer = factory.newTransformer(new StreamSource(new FileInputStream(new File(testsSrcDir, SCXML_IRP_ECMA_XSL_URI))));
459 Transformer xpathTransformer = factory.newTransformer(new StreamSource(new FileInputStream(new File(testsSrcDir, SCXML_IRP_XPATH_XSL_URI))));
460 Transformer minimalTransformer = factory.newTransformer(new StreamSource(getClass().getResourceAsStream(SCXML_IRP_MINIMAL_XSL_FILENAME)));
461 Assertions assertions = loadAssertions();
462 for (Assertions.Assertion entry : assertions.getAssertions().values()) {
463 for (Assertions.TestCase test : entry.getTestCases()) {
464 for (Assertions.Resource resource : test.getResources()) {
465 processResource(entry.getSpecId(), resource, minimalTransformer, ecmaTransformer, xpathTransformer);
466 }
467 }
468 }
469 }
470
471
472
473
474
475
476 protected Assertions loadAssertions() throws Exception {
477 final JAXBContext jaxbContext = JAXBContext.newInstance(Assertions.class);
478 final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
479 return (Assertions)jaxbUnmarshaller.unmarshal(new File(TESTS_SRC_DIR, SCXML_IRP_MANIFEST_URI));
480 }
481
482
483
484
485
486
487
488
489
490
491
492 protected void processResource(final String specid, final Assertions.Resource resource,
493 final Transformer minimalTransformer, final Transformer ecmaTransformer,
494 final Transformer xpathTransformer)
495 throws Exception {
496 System.out.println("processing IRP test file " + resource.getFilename());
497 FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + resource.getUri()), new File(TXML_TESTS_DIR + resource.getFilename()));
498 if (specid.equals("#minimal-profile")) {
499 transformResource(resource, minimalTransformer, MINIMAL_TESTS_DIR);
500 }
501 else if (specid.equals("#ecma-profile")) {
502 transformResource(resource, ecmaTransformer, ECMA_TESTS_DIR);
503 }
504 else if (specid.equals("#xpath-profile")) {
505 transformResource(resource, xpathTransformer, XPATH_TESTS_DIR);
506 }
507 else {
508 transformResource(resource, ecmaTransformer, ECMA_TESTS_DIR);
509 transformResource(resource, xpathTransformer, XPATH_TESTS_DIR);
510 }
511 }
512
513
514
515
516
517
518
519
520
521 protected void transformResource(final Assertions.Resource resource, final Transformer transformer,
522 final String targetDir) throws Exception {
523 if (resource.getFilename().endsWith(".txml")) {
524 StreamSource txmlSource = new StreamSource(new FileInputStream(new File(TXML_TESTS_DIR, resource.getFilename())));
525 transformer.transform(txmlSource, new StreamResult(new FileOutputStream(new File(targetDir, resource.getName() + ".scxml"))));
526 }
527 else {
528 FileUtils.copyFile(new File(TXML_TESTS_DIR, resource.getFilename()), new File(targetDir, resource.getFilename()));
529 }
530 }
531
532 protected void createCleanDirectory(final String path) throws Exception {
533 final File dir = new File(path);
534 if (!dir.mkdirs()) {
535 FileUtils.cleanDirectory(dir);
536 }
537 }
538
539
540
541
542
543
544
545 protected void runTests(final String testId, final Datamodel datamodel) throws Exception {
546 final Assertions assertions = loadAssertions();
547 final Tests tests = loadTests();
548 final TestResults results = new TestResults();
549 if (testId != null) {
550 final Assertions.Assertion assertion = assertions.getAssertions().get(testId);
551 if (assertion != null) {
552 runTest(assertion, tests, datamodel, true, results);
553 }
554 else {
555 throw new IllegalArgumentException("Unknown test with id: "+testId);
556 }
557 }
558 else {
559 for (Assertions.Assertion entry : assertions.getAssertions().values()) {
560 runTest(entry, tests, datamodel, false, results);
561 }
562 }
563 System.out.println(
564 "\nTest results running " +
565 (testId == null ? "all enabled tests" : "test "+testId) +
566 (datamodel != null ? " for the "+datamodel.value+" datamodel" : "") +
567 ":\n" +
568 " number of tests : "+(results.testsSkipped+results.testsPassed+results.testsFailed) +
569 " ("+results.testsPassed+" passed, "+results.testsFailed +" failed, "+results.testsSkipped+" skipped)");
570 if (results.minimalPassed+results.minimalFailed > 0) {
571 System.out.println(
572 " mimimal datamodel: "+results.minimalPassed+" passed, "+results.minimalFailed+" failed");
573 }
574 if (results.ecmaPassed+results.ecmaFailed > 0) {
575 System.out.println(
576 " ecma datamodel: "+results.ecmaPassed+" passed, "+results.ecmaFailed+" failed");
577 }
578 if (results.xpathPassed+results.xpathFailed > 0) {
579 System.out.println(
580 " xpath datamodel: "+results.xpathPassed+" passed, "+results.xpathFailed+" failed");
581 }
582 System.out.print("\n");
583 if (!results.failedTests.isEmpty()) {
584 System.out.println(" failed tests: ");
585 for (String filename : results.failedTests) {
586 System.out.println(" "+filename);
587 }
588 System.out.print("\n");
589 }
590 }
591
592
593
594
595
596
597 protected Tests loadTests() throws Exception {
598 final JAXBContext jaxbContext = JAXBContext.newInstance(Tests.class);
599 final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
600 return (Tests)jaxbUnmarshaller.unmarshal(getClass().getResource(TESTS_FILENAME));
601 }
602
603
604
605
606
607
608
609
610
611 protected void runTest(final Assertions.Assertion assertion, final Tests tests, final Datamodel datamodel,
612 final boolean singleTest, TestResults results) throws Exception {
613 final Tests.Test test = tests.getTests().get(assertion.getId());
614 if (test == null) {
615 throw new IllegalStateException("No test configuration found for W3C IRP test with id: "+assertion.getId());
616 }
617 boolean skipped = true;
618 boolean passed = true;
619 if (singleTest || test.isEnabled()) {
620 if (datamodel != Datamodel.MINIMAL || datamodel.equals(assertion.getDatamodel())) {
621 if (datamodel == null || assertion.getDatamodel() == null || datamodel.equals(assertion.getDatamodel())) {
622 final Datamodel effectiveDM = datamodel != null ? datamodel : assertion.getDatamodel();
623 for (Assertions.TestCase testCase : assertion.getTestCases()) {
624 if (effectiveDM != null) {
625 switch (effectiveDM) {
626 case MINIMAL:
627 skipped = false;
628 if (runTests(assertion, testCase, test, MINIMAL_TESTS_DIR, results.failedTests)) {
629 results.minimalPassed++;
630 }
631 else {
632 passed = false;
633 results.minimalFailed++;
634 }
635 break;
636 case ECMA:
637 skipped = false;
638 if (runTests(assertion, testCase, test, ECMA_TESTS_DIR, results.failedTests)) {
639 results.ecmaPassed++;
640 }
641 else {
642 passed = false;
643 results.ecmaFailed++;
644 }
645 break;
646 case XPATH:
647 if (test.isXPathEnabled()) {
648 skipped = false;
649 if (runTests(assertion, testCase, test, XPATH_TESTS_DIR, results.failedTests)) {
650 results.xpathPassed++;
651 }
652 else {
653 passed = false;
654 results.xpathFailed++;
655 }
656 }
657 break;
658 }
659 }
660 else {
661 skipped = false;
662 if (runTests(assertion, testCase, test, ECMA_TESTS_DIR, results.failedTests)) {
663 results.ecmaPassed++;
664 }
665 else {
666 passed = false;
667 results.ecmaFailed++;
668 }
669 if (test.isXPathEnabled()) {
670 if (runTests(assertion, testCase, test, XPATH_TESTS_DIR, results.failedTests)) {
671 results.xpathPassed++;
672 }
673 else {
674 passed = false;
675 results.xpathFailed++;
676 }
677 }
678 }
679 }
680 }
681 }
682 }
683 if (skipped) {
684 results.testsSkipped++;
685 }
686 else if (passed) {
687 results.testsPassed++;
688 }
689 else {
690 results.testsFailed++;
691 }
692 }
693
694
695
696
697
698
699
700
701
702 protected boolean runTests(final Assertions.Assertion assertion, final Assertions.TestCase testCase,
703 final Tests.Test test, final String scxmlDir, ArrayList<String> failedTests)
704 throws Exception {
705 boolean passed = true;
706 for (Assertions.Resource scxmlResource : testCase.getScxmlResources()) {
707 File scxmlFile = new File(scxmlDir, scxmlResource.getName()+".scxml");
708 if (!runTest(testCase, test, scxmlFile)) {
709 passed = false;
710 failedTests.add(scxmlFile.getParentFile().getName()+"/"+scxmlFile.getName());
711 }
712 }
713 return passed;
714 }
715
716
717
718
719
720
721
722 protected boolean runTest(final Assertions.TestCase testCase, final Tests.Test test, final File scxmlFile) {
723 try {
724 System.out.println("Executing test: "+scxmlFile.getParentFile().getName()+"/"+scxmlFile.getName());
725 final Tracer trc = new Tracer();
726 final PathResolver pathResolver = new URLResolver(scxmlFile.getParentFile().toURI().toURL());
727 final SCXMLReader.Configuration configuration = new SCXMLReader.Configuration(null, pathResolver);
728 final SCXML doc = SCXMLReader.read(new FileReader(scxmlFile), configuration);
729 if (doc == null) {
730 System.out.println(" FAIL: the SCXML file " +
731 scxmlFile.getCanonicalPath() + " can not be parsed!");
732 return false;
733 }
734 final SCXMLExecutor exec = new SCXMLExecutor(null, null, trc);
735 exec.setSingleContext(true);
736 exec.setStateMachine(doc);
737 exec.addListener(doc, trc);
738 exec.registerInvokerClass("scxml", SimpleSCXMLInvoker.class);
739 exec.registerInvokerClass("http://www.w3.org/TR/scxml/", SimpleSCXMLInvoker.class);
740 exec.go();
741 Final end;
742 while ((end = exec.getStatus().getFinalState()) == null) {
743 Thread.sleep(100);
744 exec.triggerEvents();
745 }
746 System.out.println(" final state: "+end.getId());
747 if (!testCase.isManual()) {
748 return end.getId().equals("pass");
749 }
750 else if (test.getFinalState() != null) {
751 return end.getId().equals(test.getFinalState());
752 }
753 else {
754
755 return false;
756 }
757 }
758 catch (Exception e) {
759 System.out.println(" FAIL: "+e.getMessage());
760 return false;
761 }
762 }
763 }