View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.rng.examples.stress;
18  
19  import org.apache.commons.rng.simple.RandomSource;
20  
21  import picocli.CommandLine.Command;
22  import picocli.CommandLine.Mixin;
23  import picocli.CommandLine.Option;
24  import picocli.CommandLine.Parameters;
25  
26  import java.io.BufferedReader;
27  import java.io.BufferedWriter;
28  import java.io.File;
29  import java.io.FilterOutputStream;
30  import java.io.IOException;
31  import java.io.OutputStream;
32  import java.io.OutputStreamWriter;
33  import java.nio.charset.StandardCharsets;
34  import java.nio.file.Files;
35  import java.util.ArrayList;
36  import java.util.Collections;
37  import java.util.EnumSet;
38  import java.util.Formatter;
39  import java.util.HashMap;
40  import java.util.HashSet;
41  import java.util.Iterator;
42  import java.util.LinkedHashSet;
43  import java.util.List;
44  import java.util.ListIterator;
45  import java.util.Map.Entry;
46  import java.util.Set;
47  import java.util.concurrent.Callable;
48  import java.util.function.Function;
49  import java.util.function.Supplier;
50  import java.util.regex.Matcher;
51  import java.util.regex.Pattern;
52  import java.util.stream.Collectors;
53  
54  /**
55   * Specification for the "results" command.
56   *
57   * <p>This command creates a summary of the results from known test applications.</p>
58   *
59   * <ul>
60   *   <li>Dieharder
61   *   <li>Test U01 (BigCrush, Crush, SmallCrush)
62   * </ul>
63   */
64  @Command(name = "results",
65           description = {"Collate results from stress test applications."})
66  class ResultsCommand implements Callable<Void> {
67      /** The pattern to identify the RandomSource in the stress test result header. */
68      private static final Pattern RANDOM_SOURCE_PATTERN = Pattern.compile("^# RandomSource: (.*)");
69      /** The pattern to identify the RNG in the stress test result header. */
70      private static final Pattern RNG_PATTERN = Pattern.compile("^# RNG: (.*)");
71      /** The pattern to identify the test exit code in the stress test result footer. */
72      private static final Pattern TEST_EXIT_PATTERN = Pattern.compile("^# Exit value: (\\d+)");
73      /** The pattern to identify the Dieharder test format. */
74      private static final Pattern DIEHARDER_PATTERN = Pattern.compile("^# *dieharder version");
75      /** The pattern to identify a Dieharder failed test result. */
76      private static final Pattern DIEHARDER_FAILED_PATTERN = Pattern.compile("FAILED *$");
77      /** The pattern to identify the Test U01 test format. */
78      private static final Pattern TESTU01_PATTERN = Pattern.compile("^ *Version: TestU01");
79      /** The pattern to identify the Test U01 summary header. */
80      private static final Pattern TESTU01_SUMMARY_PATTERN = Pattern.compile("^========= Summary results of (\\S*) ");
81      /** The pattern to identify the Test U01 failed test result. */
82      private static final Pattern TESTU01_TEST_RESULT_PATTERN = Pattern.compile("^  ?(\\d+  .*)    ");
83      /** The pattern to identify the Test U01 test starting entry. */
84      private static final Pattern TESTU01_STARTING_PATTERN = Pattern.compile("^ *Starting (\\S*)");
85      /** The pattern to identify the PractRand test format. */
86      private static final Pattern PRACTRAND_PATTERN = Pattern.compile("PractRand version");
87      /** The pattern to identify the PractRand output byte size. */
88      private static final Pattern PRACTRAND_OUTPUT_SIZE_PATTERN = Pattern.compile("\\(2\\^(\\d+) bytes\\)");
89      /** The pattern to identify a PractRand failed test result. */
90      private static final Pattern PRACTRAND_FAILED_PATTERN = Pattern.compile("FAIL *!* *$");
91      /** The name of the Dieharder sums test. */
92      private static final String DIEHARDER_SUMS = "diehard_sums";
93      /** The string identifying a bit-reversed generator. */
94      private static final String BIT_REVERSED = "Bit-reversed";
95      /** Character '\'. */
96      private static final char FORWARD_SLASH = '\\';
97      /** Character '/'. */
98      private static final char BACK_SLASH = '\\';
99      /** Character '|'. */
100     private static final char PIPE = '|';
101     /** The column name for the RNG. */
102     private static final String COLUMN_RNG = "RNG";
103     /** Constant for conversion of bytes to binary units, prefixed with a space. */
104     private static final String[] BINARY_UNITS = {" B", " KiB", " MiB", " GiB", " TiB", " PiB", " EiB"};
105 
106     /** The standard options. */
107     @Mixin
108     private StandardOptions reusableOptions;
109 
110     /** The results files. */
111     @Parameters(arity = "1..*",
112                 description = "The results files.",
113                 paramLabel = "<file>")
114     private List<File> resultsFiles = new ArrayList<>();
115 
116     /** The file output prefix. */
117     @Option(names = {"-o", "--out"},
118             description = "The output file (default: stdout).")
119     private File fileOutput;
120 
121     /** The output format. */
122     @Option(names = {"-f", "--format"},
123             description = {"Output format (default: ${DEFAULT-VALUE}).",
124                            "Valid values: ${COMPLETION-CANDIDATES}."})
125     private ResultsCommand.OutputFormat outputFormat = OutputFormat.TXT;
126 
127     /** The flag to show failed tests. */
128     @Option(names = {"--failed"},
129             description = {"Output failed tests (support varies by format).",
130                            "CSV: List individual test failures.",
131                            "APT: Count of systematic test failures."})
132     private boolean showFailedTests;
133 
134     /** The flag to include the Dieharder sums test. */
135     @Option(names = {"--include-sums"},
136             description = "Include Dieharder sums test.")
137     private boolean includeDiehardSums;
138 
139     /** The common file path prefix used when outputting file paths. */
140     @Option(names = {"--path-prefix"},
141             description = {"Common path prefix.",
142                            "If specified this will replace the common prefix from all " +
143                            "files when the path is output, e.g. for the APT report."})
144     private String pathPrefix = "";
145 
146     /** The flag to ignore partial results. */
147     @Option(names = {"-i", "--ignore"},
148             description = "Ignore partial results.")
149     private boolean ignorePartialResults;
150 
151     /** The flag to delete partial results files. */
152     @Option(names = {"--delete"},
153             description = {"Delete partial results files.",
154                            "This is not reversible!"})
155     private boolean deletePartialResults;
156 
157     /**
158      * The output mode for the results.
159      */
160     enum OutputFormat {
161         /** Comma-Separated Values (CSV) output. */
162         CSV,
163         /** Almost Plain Text (APT) output. */
164         APT,
165         /** Text table output. */
166         TXT,
167         /** Systematic failures output. */
168         FAILURES,
169     }
170 
171     /**
172      * The test application file format.
173      */
174     enum TestFormat {
175         /** Dieharder. */
176         DIEHARDER,
177         /** Test U01. */
178         TESTU01,
179         /** PractRand. */
180         PRACTRAND,
181     }
182 
183     /**
184      * Encapsulate the results of a test application.
185      */
186     private static class TestResult {
187         /** The result file. */
188         private final File resultFile;
189         /** The random source. */
190         private final RandomSource randomSource;
191         /** Flag indicating the generator was bit reversed. */
192         private final boolean bitReversed;
193         /** The test application format. */
194         private final TestFormat testFormat;
195         /** The names of the failed tests. */
196         private final List<String> failedTests = new ArrayList<>();
197         /** The test application name. */
198         private String testApplicationName;
199         /**
200          * Store the exit code.
201          * Initialised to {@link Integer#MIN_VALUE}. Exit values are expected to be 8-bit numbers
202          * with zero for success.
203          */
204         private int exitCode = Integer.MIN_VALUE;
205 
206         /**
207          * @param resultFile the result file
208          * @param randomSource the random source
209          * @param bitReversed the bit reversed flag
210          * @param testFormat the test format
211          */
212         TestResult(File resultFile,
213                    RandomSource randomSource,
214                    boolean bitReversed,
215                    TestFormat testFormat) {
216             this.resultFile = resultFile;
217             this.randomSource = randomSource;
218             this.bitReversed = bitReversed;
219             this.testFormat = testFormat;
220         }
221 
222         /**
223          * Adds the failed test.
224          *
225          * @param testId the test id
226          */
227         void addFailedTest(String testId) {
228             failedTests.add(testId);
229         }
230 
231         /**
232          * Gets the result file.
233          *
234          * @return the result file
235          */
236         File getResultFile() {
237             return resultFile;
238         }
239 
240         /**
241          * Gets the random source.
242          *
243          * @return the random source
244          */
245         RandomSource getRandomSource() {
246             return randomSource;
247         }
248 
249         /**
250          * Checks if the generator was bit reversed.
251          *
252          * @return true if bit reversed
253          */
254         boolean isBitReversed() {
255             return bitReversed;
256         }
257 
258         /**
259          * Gets the test format.
260          *
261          * @return the test format
262          */
263         TestFormat getTestFormat() {
264             return testFormat;
265         }
266 
267         /**
268          * Gets the failed tests.
269          *
270          * @return the failed tests
271          */
272         List<String> getFailedTests() {
273             return failedTests;
274         }
275 
276         /**
277          * Gets the failure count.
278          *
279          * @return the failure count
280          */
281         int getFailureCount() {
282             return failedTests.size();
283         }
284 
285         /**
286          * Gets the a failure summary string.
287          *
288          * <p>The default implementation is the count of the number of failures.
289          * This will be negative if the test is not complete.</p>
290          *
291          * @return the failure summary
292          */
293         String getFailureSummaryString() {
294             return (isComplete()) ? Integer.toString(failedTests.size()) : "-" + failedTests.size();
295         }
296 
297         /**
298          * Sets the test application name.
299          *
300          * @param testApplicationName the new test application name
301          */
302         void setTestApplicationName(String testApplicationName) {
303             this.testApplicationName = testApplicationName;
304         }
305 
306         /**
307          * Gets the test application name.
308          *
309          * @return the test application name
310          */
311         String getTestApplicationName() {
312             return testApplicationName == null ? String.valueOf(getTestFormat()) : testApplicationName;
313         }
314 
315         /**
316          * Checks if the test result is complete.
317          * This is {@code true} only if the exit code was found and is zero.
318          *
319          * @return true if complete
320          */
321         boolean isComplete() {
322             return exitCode == 0;
323         }
324 
325         /**
326          * Sets the exit code flag.
327          *
328          * @param exitCode the new exit code
329          */
330         void setExitCode(int exitCode) {
331             this.exitCode = exitCode;
332         }
333     }
334 
335     /**
336      * Encapsulate the results of a PractRand test application. This is a specialisation that
337      * allows handling PractRand results which are linked to the output length used by the RNG.
338      */
339     private static class PractRandTestResult extends TestResult {
340         /** The length of the RNG output used to generate failed tests. */
341         private int lengthExponent;
342 
343         /**
344          * @param resultFile the result file
345          * @param randomSource the random source
346          * @param bitReversed the bit reversed flag
347          * @param testFormat the test format
348          */
349         PractRandTestResult(File resultFile, RandomSource randomSource, boolean bitReversed, TestFormat testFormat) {
350             super(resultFile, randomSource, bitReversed, testFormat);
351         }
352 
353         /**
354          * Gets the length of the RNG output used to generate failed tests.
355          * If this is zero then no failures occurred.
356          *
357          * @return the length exponent
358          */
359         int getLengthExponent() {
360             return lengthExponent;
361         }
362 
363         /**
364          * Sets the length of the RNG output used to generate failed tests.
365          *
366          * @param lengthExponent the length exponent
367          */
368         void setLengthExponent(int lengthExponent) {
369             this.lengthExponent = lengthExponent;
370         }
371 
372         /**
373          * {@inheritDoc}
374          *
375          * <p>The PractRand summary is the exponent of the length of byte output from the RNG where
376          * a failure occurred.
377          */
378         @Override
379         String getFailureSummaryString() {
380             return lengthExponent == 0 ? "-" : Integer.toString(lengthExponent);
381         }
382     }
383 
384     /**
385      * Reads the results files and outputs in the chosen format.
386      */
387     @Override
388     public Void call() {
389         LogUtils.setLogLevel(reusableOptions.logLevel);
390 
391         // Read the results
392         final List<TestResult> results = readResults();
393 
394         if (deletePartialResults) {
395             deleteIfIncomplete(results);
396             return null;
397         }
398 
399         try (OutputStream out = createOutputStream()) {
400             switch (outputFormat) {
401             case CSV:
402                 writeCSVData(out, results);
403                 break;
404             case APT:
405                 writeAPT(out, results);
406                 break;
407             case TXT:
408                 writeTXT(out, results);
409                 break;
410             case FAILURES:
411                 writeFailures(out, results);
412                 break;
413             default:
414                 throw new ApplicationException("Unknown output format: " + outputFormat);
415             }
416         } catch (final IOException ex) {
417             throw new ApplicationException("IO error: " + ex.getMessage(), ex);
418         }
419         return null;
420     }
421 
422     /**
423      * Read the results.
424      *
425      * @return the results
426      */
427     private List<TestResult> readResults() {
428         final ArrayList<TestResult> results = new ArrayList<>();
429         for (final File resultFile : resultsFiles) {
430             readResults(results, resultFile);
431         }
432         return results;
433     }
434 
435     /**
436      * Read the file and extract any test results.
437      *
438      * @param results Results.
439      * @param resultFile Result file.
440      * @throws ApplicationException If the results cannot be parsed.
441      */
442     private void readResults(List<TestResult> results,
443                              File resultFile) {
444         final List<String> contents = readFileContents(resultFile);
445         // Files may have multiple test results per file (i.e. appended output)
446         final List<List<String>> outputs = splitContents(contents);
447         if (outputs.isEmpty()) {
448             LogUtils.error("No test output in file: %s", resultFile);
449         } else {
450             for (final List<String> testOutput : outputs) {
451                 final TestResult result = readResult(resultFile, testOutput);
452                 if (!result.isComplete()) {
453                     LogUtils.info("Partial results in file: %s", resultFile);
454                     if (ignorePartialResults) {
455                         continue;
456                     }
457                 }
458                 results.add(result);
459             }
460         }
461     }
462 
463     /**
464      * Read the file contents.
465      *
466      * @param resultFile Result file.
467      * @return the file contents
468      * @throws ApplicationException If the file cannot be read.
469      */
470     private static List<String> readFileContents(File resultFile) {
471         final ArrayList<String> contents = new ArrayList<>();
472         try (BufferedReader reader = Files.newBufferedReader(resultFile.toPath())) {
473             for (String line = reader.readLine(); line != null; line = reader.readLine()) {
474                 contents.add(line);
475             }
476         } catch (final IOException ex) {
477             throw new ApplicationException("Failed to read file contents: " + resultFile, ex);
478         }
479         return contents;
480     }
481 
482     /**
483      * Split the file contents into separate test output. This is used in the event
484      * that results have been appended to the same file.
485      *
486      * @param contents File contents.
487      * @return the test output
488      */
489     private static List<List<String>> splitContents(List<String> contents) {
490         final ArrayList<List<String>> testOutputs = new ArrayList<>();
491         // Assume each output begins with e.g. # RandomSource: SPLIT_MIX_64
492         // Note each beginning.
493         int begin = -1;
494         for (int i = 0; i < contents.size(); i++) {
495             if (RANDOM_SOURCE_PATTERN.matcher(contents.get(i)).matches()) {
496                 if (begin >= 0) {
497                     testOutputs.add(contents.subList(begin, i));
498                 }
499                 begin = i;
500             }
501         }
502         if (begin >= 0) {
503             testOutputs.add(contents.subList(begin, contents.size()));
504         }
505         return testOutputs;
506     }
507 
508     /**
509      * Read the file into a test result.
510      *
511      * @param resultFile Result file.
512      * @param testOutput Test output.
513      * @return the test result
514      * @throws ApplicationException If the result cannot be parsed.
515      */
516     private TestResult readResult(File resultFile,
517                                   List<String> testOutput) {
518         // Use an iterator for a single pass over the test output
519         final ListIterator<String> iter = testOutput.listIterator();
520 
521         // Identify the RandomSource and bit reversed flag from the header:
522         final RandomSource randomSource = getRandomSource(resultFile, iter);
523         final boolean bitReversed = getBitReversed(resultFile, iter);
524 
525         // Identify the test application format. This may return null.
526         final TestFormat testFormat = getTestFormat(resultFile, iter);
527 
528         // Read the application results
529         final TestResult testResult = createTestResult(resultFile, randomSource, bitReversed, testFormat);
530         if (testFormat == TestFormat.DIEHARDER) {
531             readDieharder(iter, testResult);
532         } else if (testFormat == TestFormat.TESTU01) {
533             readTestU01(resultFile, iter, testResult);
534         } else {
535             readPractRand(iter, (PractRandTestResult) testResult);
536         }
537         return testResult;
538     }
539 
540     /**
541      * Creates the test result.
542      *
543      * @param resultFile Result file.
544      * @param randomSource Random source.
545      * @param bitReversed True if the random source was bit reversed.
546      * @param testFormat Test format.
547      * @return the test result
548      */
549     private static TestResult createTestResult(File resultFile, RandomSource randomSource,
550                                                boolean bitReversed, TestFormat testFormat) {
551         return testFormat == TestFormat.PRACTRAND ?
552             new PractRandTestResult(resultFile, randomSource, bitReversed, testFormat) :
553             new TestResult(resultFile, randomSource, bitReversed, testFormat);
554     }
555 
556     /**
557      * Gets the random source from the output header.
558      *
559      * @param resultFile Result file (for the exception message).
560      * @param iter Iterator of the test output.
561      * @return the random source
562      * @throws ApplicationException If the RandomSource header line cannot be found.
563      */
564     private static RandomSource getRandomSource(File resultFile, Iterator<String> iter) {
565         while (iter.hasNext()) {
566             final Matcher matcher = RANDOM_SOURCE_PATTERN.matcher(iter.next());
567             if (matcher.matches()) {
568                 return RandomSource.valueOf(matcher.group(1));
569             }
570         }
571         throw new ApplicationException("Failed to find RandomSource header line: " + resultFile);
572     }
573 
574     /**
575      * Gets the bit-reversed flag from the output header.
576      *
577      * @param resultFile Result file (for the exception message).
578      * @param iter Iterator of the test output.
579      * @return the bit-reversed flag
580      * @throws ApplicationException If the RNG header line cannot be found.
581      */
582     private static boolean getBitReversed(File resultFile, Iterator<String> iter) {
583         while (iter.hasNext()) {
584             final Matcher matcher = RNG_PATTERN.matcher(iter.next());
585             if (matcher.matches()) {
586                 return matcher.group(1).contains(BIT_REVERSED);
587             }
588         }
589         throw new ApplicationException("Failed to find RNG header line: " + resultFile);
590     }
591 
592     /**
593      * Gets the test format from the output. This scans the stdout produced by a test application.
594      * If it is not recognised this may be a valid partial result or an unknown result. Throw
595      * an exception if not allowing partial results, otherwise log an error.
596      *
597      * @param resultFile Result file (for the exception message).
598      * @param iter Iterator of the test output.
599      * @return the test format (or null)
600      * @throws ApplicationException If the test format cannot be found.
601      */
602     private TestFormat getTestFormat(File resultFile, Iterator<String> iter) {
603         while (iter.hasNext()) {
604             final String line = iter.next();
605             if (DIEHARDER_PATTERN.matcher(line).find()) {
606                 return TestFormat.DIEHARDER;
607             }
608             if (TESTU01_PATTERN.matcher(line).find()) {
609                 return TestFormat.TESTU01;
610             }
611             if (PRACTRAND_PATTERN.matcher(line).find()) {
612                 return TestFormat.PRACTRAND;
613             }
614         }
615         if (!ignorePartialResults) {
616             throw new ApplicationException("Failed to identify the test application format: " + resultFile);
617         }
618         LogUtils.error("Failed to identify the test application format: %s", resultFile);
619         return null;
620     }
621 
622     /**
623      * Read the result output from the Dieharder test application.
624      *
625      * @param iter Iterator of the test output.
626      * @param testResult Test result.
627      */
628     private void readDieharder(Iterator<String> iter,
629                                TestResult testResult) {
630         // Dieharder results are printed in-line using the following format:
631         //#=============================================================================#
632         //        test_name   |ntup| tsamples |psamples|  p-value |Assessment
633         //#=============================================================================#
634         //   diehard_birthdays|   0|       100|     100|0.97484422|  PASSED
635         //   ...
636         //        diehard_oqso|   0|   2097152|     100|0.00000000|  FAILED
637 
638         testResult.setTestApplicationName("Dieharder");
639 
640         // Identify any line containing FAILED and then get the test ID using
641         // the first two columns (test_name + ntup).
642         while (iter.hasNext()) {
643             final String line = iter.next();
644             if (DIEHARDER_FAILED_PATTERN.matcher(line).find()) {
645                 // Optionally include the flawed Dieharder sums test
646                 if (!includeDiehardSums && line.contains(DIEHARDER_SUMS)) {
647                     continue;
648                 }
649                 final int index1 = line.indexOf('|');
650                 final int index2 = line.indexOf('|', index1 + 1);
651                 testResult.addFailedTest(line.substring(0, index1).trim() + ":" +
652                                          line.substring(index1 + 1, index2).trim());
653             } else if (findExitCode(testResult, line)) {
654                 return;
655             }
656         }
657     }
658 
659     /**
660      * Find the exit code in the line. Update the test result with the code if found.
661      *
662      * @param testResult Test result.
663      * @param line Line from the test result output.
664      * @return true, if the exit code was found
665      */
666     private static boolean findExitCode(TestResult testResult, String line) {
667         final Matcher matcher = TEST_EXIT_PATTERN.matcher(line);
668         if (matcher.find()) {
669             testResult.setExitCode(Integer.parseInt(matcher.group(1)));
670             return true;
671         }
672         return false;
673     }
674 
675     /**
676      * Read the result output from the Test U01 test application.
677      *
678      * <p>Test U01 outputs a summary of results at the end of the test output. If this cannot
679      * be found the method will throw an exception unless partial results are allowed.</p>
680      *
681      * @param resultFile Result file (for the exception message).
682      * @param iter Iterator of the test output.
683      * @param testResult Test result.
684      * @throws ApplicationException If the TestU01 summary cannot be found.
685      */
686     private void readTestU01(File resultFile,
687                              ListIterator<String> iter,
688                              TestResult testResult) {
689         // Results are summarised at the end of the file:
690         //========= Summary results of BigCrush =========
691         //
692         //Version:          TestU01 1.2.3
693         //Generator:        stdin
694         //Number of statistics:  155
695         //Total CPU time:   06:14:50.43
696         //The following tests gave p-values outside [0.001, 0.9990]:
697         //(eps  means a value < 1.0e-300):
698         //(eps1 means a value < 1.0e-15):
699         //
700         //      Test                          p-value
701         //----------------------------------------------
702         // 1  SerialOver, r = 0                eps
703         // 3  CollisionOver, t = 2           1 - eps1
704         // 5  CollisionOver, t = 3           1 - eps1
705         // 7  CollisionOver, t = 7             eps
706 
707         // Identify the summary line
708         final String testSuiteName = skipToTestU01Summary(resultFile, iter);
709 
710         // This may not be present if the results are not complete
711         if (testSuiteName == null) {
712             // Rewind
713             while (iter.hasPrevious()) {
714                 iter.previous();
715             }
716             updateTestU01ApplicationName(iter, testResult);
717             return;
718         }
719 
720         setTestU01ApplicationName(testResult, testSuiteName);
721 
722         // Read test results using the entire line except the p-value for the test Id
723         // Note:
724         // This will count sub-parts of the same test as distinct failures.
725         while (iter.hasNext()) {
726             final String line = iter.next();
727             final Matcher matcher = TESTU01_TEST_RESULT_PATTERN.matcher(line);
728             if (matcher.find()) {
729                 testResult.addFailedTest(matcher.group(1).trim());
730             } else if (findExitCode(testResult, line)) {
731                 return;
732             }
733         }
734     }
735 
736     /**
737      * Sets the Test U01 application name using the provide test suite name.
738      *
739      * @param testResult Test result.
740      * @param testSuiteName Test suite name.
741      */
742     private static void setTestU01ApplicationName(TestResult testResult, String testSuiteName) {
743         testResult.setTestApplicationName("TestU01 (" + testSuiteName + ")");
744     }
745 
746     /**
747      * Skip to the Test U01 result summary.
748      *
749      * <p>If this cannot be found the method will throw an exception unless partial results
750      * are allowed.</p>
751      *
752      * @param resultFile Result file (for the exception message).
753      * @param iter Iterator of the test output.
754      * @return the name of the test suite
755      * @throws ApplicationException If the TestU01 summary cannot be found.
756      */
757     private String skipToTestU01Summary(File resultFile, Iterator<String> iter) {
758         final String testSuiteName = findMatcherGroup1(iter, TESTU01_SUMMARY_PATTERN);
759         // Allow the partial result to be ignored
760         if (testSuiteName != null || ignorePartialResults) {
761             return testSuiteName;
762         }
763         throw new ApplicationException("Failed to identify the Test U01 result summary: " + resultFile);
764     }
765 
766     /**
767      * Update the test application name from the Test U01 results. This can be used to identify
768      * the test suite from the start of the results in the event that the results summary has
769      * not been output.
770      *
771      * @param iter Iterator of the test output.
772      * @param testResult Test result.
773      */
774     private static void updateTestU01ApplicationName(Iterator<String> iter,
775                                                      TestResult testResult) {
776         final String testSuiteName = findMatcherGroup1(iter, TESTU01_STARTING_PATTERN);
777         if (testSuiteName != null) {
778             setTestU01ApplicationName(testResult, testSuiteName);
779         }
780     }
781 
782     /**
783      * Create a matcher for each item in the iterator and if a match is identified return
784      * group 1.
785      *
786      * @param iter Iterator of text to match.
787      * @param pattern Pattern to match.
788      * @return the string (or null)
789      */
790     private static String findMatcherGroup1(Iterator<String> iter,
791                                             Pattern pattern) {
792         while (iter.hasNext()) {
793             final String line = iter.next();
794             final Matcher matcher = pattern.matcher(line);
795             if (matcher.find()) {
796                 return matcher.group(1);
797             }
798         }
799         return null;
800     }
801 
802     /**
803      * Read the result output from the PractRand test application.
804      *
805      * @param iter Iterator of the test output.
806      * @param testResult Test result.
807      */
808     private static void readPractRand(Iterator<String> iter,
809                                       PractRandTestResult testResult) {
810         // PractRand results are printed for blocks of byte output that double in size.
811         // The results report 'unusual' and 'suspicious' output and the test stops
812         // at the first failure.
813         // Results are typically reported as the size of output that failed, e.g.
814         // the JDK random fails at 256KiB.
815         //
816         // rng=RNG_stdin32, seed=0xfc6c7332
817         // length= 128 kilobytes (2^17 bytes), time= 5.9 seconds
818         //   no anomalies in 118 test result(s)
819         //
820         // rng=RNG_stdin32, seed=0xfc6c7332
821         // length= 256 kilobytes (2^18 bytes), time= 7.6 seconds
822         //   Test Name                         Raw       Processed     Evaluation
823         //   [Low1/64]Gap-16:A                 R= +12.4  p =  8.7e-11   VERY SUSPICIOUS
824         //   [Low1/64]Gap-16:B                 R= +13.4  p =  3.0e-11    FAIL
825         //   [Low8/32]DC6-9x1Bytes-1           R=  +7.6  p =  5.7e-4   unusual
826         //   ...and 136 test result(s) without anomalies
827 
828         testResult.setTestApplicationName("PractRand");
829 
830         // Store the exponent of the current output length.
831         int exp = 0;
832 
833         // Identify any line containing FAIL and then get the test name using
834         // the first token in the line.
835         while (iter.hasNext()) {
836             String line = iter.next();
837             final Matcher matcher = PRACTRAND_OUTPUT_SIZE_PATTERN.matcher(line);
838             if (matcher.find()) {
839                 // Store the current output length
840                 exp = Integer.parseInt(matcher.group(1));
841             } else if (PRACTRAND_FAILED_PATTERN.matcher(line).find()) {
842                 // Record the output length where failures occurred.
843                 testResult.setLengthExponent(exp);
844                 // Add the failed test name. This does not include the length exponent
845                 // allowing meta-processing of systematic failures.
846                 // Remove initial whitespace
847                 line = line.trim();
848                 final int index = line.indexOf(' ');
849                 testResult.addFailedTest(line.substring(0, index));
850             } else if (findExitCode(testResult, line)) {
851                 return;
852             }
853         }
854     }
855 
856     /**
857      * Delete any result file if incomplete.
858      *
859      * @param results Results.
860      * @throws ApplicationException If a file could not be deleted.
861      */
862     private static void deleteIfIncomplete(List<TestResult> results) {
863         results.forEach(ResultsCommand::deleteIfIncomplete);
864     }
865 
866     /**
867      * Delete the result file if incomplete.
868      *
869      * @param result Test result.
870      * @throws ApplicationException If the file could not be deleted.
871      */
872     private static void deleteIfIncomplete(TestResult result) {
873         if (!result.isComplete()) {
874             try {
875                 Files.delete(result.getResultFile().toPath());
876                 LogUtils.info("Deleted file: %s", result.getResultFile());
877             } catch (IOException ex) {
878                 throw new ApplicationException("Failed to delete file: " + result.getResultFile(), ex);
879             }
880         }
881     }
882 
883     /**
884      * Creates the output stream. This will not be buffered.
885      *
886      * @return the output stream
887      */
888     private OutputStream createOutputStream() {
889         if (fileOutput != null) {
890             try {
891                 return Files.newOutputStream(fileOutput.toPath());
892             } catch (final IOException ex) {
893                 throw new ApplicationException("Failed to create output: " + fileOutput, ex);
894             }
895         }
896         return new FilterOutputStream(System.out) {
897             @Override
898             public void close() {
899                 // Do not close stdout
900             }
901         };
902     }
903 
904     /**
905      * Write the results to a table in Comma Separated Value (CSV) format.
906      *
907      * @param out Output stream.
908      * @param results Results.
909      * @throws IOException Signals that an I/O exception has occurred.
910      */
911     private void writeCSVData(OutputStream out,
912                               List<TestResult> results) throws IOException {
913         // Sort by columns
914         Collections.sort(results, (o1, o2) -> {
915             int result = Integer.compare(o1.getRandomSource().ordinal(), o2.getRandomSource().ordinal());
916             if (result != 0) {
917                 return result;
918             }
919             result = Boolean.compare(o1.isBitReversed(), o2.isBitReversed());
920             if (result != 0) {
921                 return result;
922             }
923             result = o1.getTestApplicationName().compareTo(o2.getTestApplicationName());
924             if (result != 0) {
925                 return result;
926             }
927             return Integer.compare(o1.getFailureCount(), o2.getFailureCount());
928         });
929 
930         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
931             output.write("RandomSource,Bit-reversed,Test,Failures");
932             if (showFailedTests) {
933                 output.write(",Failed");
934             }
935             output.newLine();
936             for (final TestResult result : results) {
937                 output.write(result.getRandomSource().toString());
938                 output.write(',');
939                 output.write(Boolean.toString(result.isBitReversed()));
940                 output.write(',');
941                 output.write(result.getTestApplicationName());
942                 output.write(',');
943                 output.write(result.getFailureSummaryString());
944                 // Optionally write out failed test names.
945                 if (showFailedTests) {
946                     output.write(',');
947                     output.write(result.getFailedTests().stream().collect(Collectors.joining("|")));
948                 }
949                 output.newLine();
950             }
951         }
952     }
953 
954     /**
955      * Write the results using the Almost Plain Text (APT) format for a table. This
956      * table can be included in the documentation for the Commons RNG site.
957      *
958      * @param out Output stream.
959      * @param results Results.
960      * @throws IOException Signals that an I/O exception has occurred.
961      */
962     private void writeAPT(OutputStream out,
963                           List<TestResult> results) throws IOException {
964         // Identify all:
965         // RandomSources, bit-reversed, test names,
966         final List<RandomSource> randomSources = getRandomSources(results);
967         final List<Boolean> bitReversed = getBitReversed(results);
968         final List<String> testNames = getTestNames(results);
969 
970         // Identify the common path prefix to be replaced
971         final int prefixLength = (pathPrefix.isEmpty()) ? 0 : findCommonPathPrefixLength(results);
972 
973         // Create a function to update the file path and then output the failure count
974         // as a link to the file using the APT format.
975         final Function<TestResult, String> toAPTString = result -> {
976             String path = result.getResultFile().getPath();
977             // Remove common path prefix
978             path = path.substring(prefixLength);
979             // Build the APT relative link
980             final StringBuilder sb = new StringBuilder()
981                 .append("{{{").append(pathPrefix).append(path).append('}')
982                 .append(result.getFailureSummaryString()).append("}}");
983             // Convert to web-link name separators
984             for (int i = 0; i < sb.length(); i++) {
985                 if (sb.charAt(i) == BACK_SLASH) {
986                     sb.setCharAt(i, FORWARD_SLASH);
987                 }
988             }
989             return sb.toString();
990         };
991 
992         // Create columns for RandomSource, bit-reversed, each test name.
993         // Make bit-reversed column optional if no generators are bit reversed.
994         final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);
995 
996         final String header = createAPTHeader(showBitReversedColumn, testNames);
997         final String separator = createAPTSeparator(header);
998 
999         // Output
1000         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
1001             // For the first line using '*' (centre) character instead of '+' (left align)
1002             output.write(separator.replace('+', '*'));
1003             output.write(header);
1004             output.newLine();
1005             output.write(separator);
1006 
1007             final StringBuilder sb = new StringBuilder();
1008 
1009             // This will collate results for each combination of 'RandomSource + bitReversed'
1010             for (final RandomSource randomSource : randomSources) {
1011                 for (final boolean reversed : bitReversed) {
1012                     // Highlight in bold a RNG with no systematic failures
1013                     boolean highlight = true;
1014 
1015                     // Buffer the column output
1016                     sb.setLength(0);
1017 
1018                     if (showBitReversedColumn) {
1019                         writeAPTColumn(sb, Boolean.toString(reversed), false);
1020                     }
1021                     for (final String testName : testNames) {
1022                         final List<TestResult> testResults = getTestResults(results, randomSource, reversed, testName);
1023                         String text = testResults.stream()
1024                                                  .map(toAPTString)
1025                                                  .collect(Collectors.joining(", "));
1026                         // Summarise the failures across all tests
1027                         final String summary = getFailuresSummary(testResults);
1028                         if (!summary.isEmpty()) {
1029                             // Identify RNGs with no systematic failures
1030                             highlight = false;
1031                             if (showFailedTests) {
1032                                 // Add summary in brackets
1033                                 text += " (" + summary + ")";
1034                             }
1035                         }
1036                         writeAPTColumn(sb, text, false);
1037                     }
1038 
1039                     output.write('|');
1040                     writeAPTColumn(output, randomSource.toString(), highlight);
1041                     output.write(sb.toString());
1042                     output.newLine();
1043                     output.write(separator);
1044                 }
1045             }
1046         }
1047     }
1048 
1049     /**
1050      * Gets the random sources present in the results.
1051      *
1052      * @param results Results.
1053      * @return the random sources
1054      */
1055     private static List<RandomSource> getRandomSources(List<TestResult> results) {
1056         final EnumSet<RandomSource> set = EnumSet.noneOf(RandomSource.class);
1057         for (final TestResult result : results) {
1058             set.add(result.getRandomSource());
1059         }
1060         final ArrayList<RandomSource> list = new ArrayList<>(set);
1061         Collections.sort(list);
1062         return list;
1063     }
1064 
1065     /**
1066      * Gets the bit-reversed options present in the results.
1067      *
1068      * @param results Results.
1069      * @return the bit-reversed options
1070      */
1071     private static List<Boolean> getBitReversed(List<TestResult> results) {
1072         final ArrayList<Boolean> list = new ArrayList<>(2);
1073         if (results.isEmpty()) {
1074             // Default to no bit-reversed results
1075             list.add(Boolean.FALSE);
1076         } else {
1077             final boolean first = results.get(0).isBitReversed();
1078             list.add(first);
1079             for (final TestResult result : results) {
1080                 if (first != result.isBitReversed()) {
1081                     list.add(!first);
1082                     break;
1083                 }
1084             }
1085         }
1086         Collections.sort(list);
1087         return list;
1088     }
1089 
1090     /**
1091      * Gets the test names present in the results. These are returned in encountered order.
1092      *
1093      * @param results Results.
1094      * @return the test names
1095      */
1096     private static List<String> getTestNames(List<TestResult> results) {
1097         // Enforce encountered order with a linked hash set.
1098         final Set<String> set = new LinkedHashSet<>();
1099         for (final TestResult result : results) {
1100             set.add(result.getTestApplicationName());
1101         }
1102         return new ArrayList<>(set);
1103     }
1104 
1105     /**
1106      * Find the common path prefix for all result files. This is returned as the length of the
1107      * common prefix.
1108      *
1109      * @param results Results.
1110      * @return the length
1111      */
1112     private static int findCommonPathPrefixLength(List<TestResult> results) {
1113         if (results.isEmpty()) {
1114             return 0;
1115         }
1116         // Find the first prefix
1117         final String prefix1 = getPathPrefix(results.get(0));
1118         int length = prefix1.length();
1119         for (int i = 1; i < results.size() && length != 0; i++) {
1120             final String prefix2 = getPathPrefix(results.get(i));
1121             // Update
1122             final int size = Math.min(prefix2.length(), length);
1123             length = 0;
1124             while (length < size && prefix1.charAt(length) == prefix2.charAt(length)) {
1125                 length++;
1126             }
1127         }
1128         return length;
1129     }
1130 
1131     /**
1132      * Gets the path prefix.
1133      *
1134      * @param testResult Test result.
1135      * @return the path prefix (or the empty string)
1136      */
1137     private static String getPathPrefix(TestResult testResult) {
1138         final String parent = testResult.getResultFile().getParent();
1139         return parent == null ? "" : parent;
1140     }
1141 
1142     /**
1143      * Creates the APT header.
1144      *
1145      * @param showBitReversedColumn Set to true to the show bit reversed column.
1146      * @param testNames Test names.
1147      * @return the header
1148      */
1149     private static String createAPTHeader(boolean showBitReversedColumn,
1150                                           List<String> testNames) {
1151         final StringBuilder sb = new StringBuilder(100).append("|| RNG identifier ||");
1152         if (showBitReversedColumn) {
1153             sb.append(" Bit-reversed ||");
1154         }
1155         for (final String name : testNames) {
1156             sb.append(' ').append(name).append(" ||");
1157         }
1158         return sb.toString();
1159     }
1160 
1161     /**
1162      * Creates the APT separator for each table row.
1163      *
1164      * <p>The separator is created using the '+' character to left align the columns.
1165      *
1166      * @param header Header.
1167      * @return the separator
1168      */
1169     private static String createAPTSeparator(String header) {
1170         // Replace everything with '-' except '|' which is replaced with "*-" for the first
1171         // character, "+-" for all other occurrences except "-+" at the end
1172         final StringBuilder sb = new StringBuilder(header);
1173         for (int i = 0; i < header.length(); i++) {
1174             if (sb.charAt(i) == PIPE) {
1175                 sb.setCharAt(i, i == 0 ? '*' : '+');
1176                 sb.setCharAt(i + 1,  '-');
1177             } else {
1178                 sb.setCharAt(i,  '-');
1179             }
1180         }
1181         // Fix the end
1182         sb.setCharAt(header.length() - 2, '-');
1183         sb.setCharAt(header.length() - 1, '+');
1184         sb.append(System.lineSeparator());
1185         return sb.toString();
1186     }
1187 
1188     /**
1189      * Write the column text to the output.
1190      *
1191      * @param output Output.
1192      * @param text Text.
1193      * @param highlight If {@code true} highlight the text in bold.
1194      * @throws IOException Signals that an I/O exception has occurred.
1195      */
1196     private static void writeAPTColumn(Appendable output,
1197                                        String text,
1198                                        boolean highlight) throws IOException {
1199         output.append(' ');
1200         if (highlight) {
1201             output.append("<<");
1202         }
1203         output.append(text);
1204         if (highlight) {
1205             output.append(">>");
1206         }
1207         output.append(" |");
1208     }
1209 
1210     /**
1211      * Gets the test results that match the arguments.
1212      *
1213      * @param results Results.
1214      * @param randomSource Random source.
1215      * @param bitReversed Bit reversed flag.
1216      * @param testName Test name.
1217      * @return the matching results
1218      */
1219     private static List<TestResult> getTestResults(List<TestResult> results,
1220                                                    RandomSource randomSource,
1221                                                    boolean bitReversed,
1222                                                    String testName) {
1223         final ArrayList<TestResult> list = new ArrayList<>();
1224         for (final TestResult result : results) {
1225             if (result.getRandomSource() == randomSource &&
1226                 result.bitReversed == bitReversed &&
1227                 result.getTestApplicationName().equals(testName)) {
1228                 list.add(result);
1229             }
1230         }
1231         return list;
1232     }
1233 
1234     /**
1235      * Gets the systematic failures (tests that fail in every test result).
1236      *
1237      * @param results Results.
1238      * @return the systematic failures
1239      */
1240     private static List<String> getSystematicFailures(List<TestResult> results) {
1241         final HashMap<String, Integer> map = new HashMap<>();
1242         for (final TestResult result : results) {
1243             // Ignore partial results
1244             if (!result.isComplete()) {
1245                 continue;
1246             }
1247             // Some named tests can fail more than once on different statistics.
1248             // For example TestU01 BigCrush LongestHeadRun can output in the summary:
1249             // 86  LongestHeadRun, r = 0            eps
1250             // 86  LongestHeadRun, r = 0          1 - eps1
1251             // This will be counted as 2 failed tests. For the purpose of systematic
1252             // failures the name of the test is the same and should be counted once.
1253             final HashSet<String> unique = new HashSet<>(result.getFailedTests());
1254             for (final String test : unique) {
1255                 map.merge(test, 1, (i, j) -> i + j);
1256             }
1257         }
1258         final int completeCount = (int) results.stream().filter(TestResult::isComplete).count();
1259         final List<String> list = map.entrySet().stream()
1260                                                 .filter(e -> e.getValue() == completeCount)
1261                                                 .map(Entry::getKey)
1262                                                 .collect(Collectors.toCollection(
1263                                                     (Supplier<List<String>>) ArrayList::new));
1264         // Special case for PractRand. Add the maximum RNG output length before failure.
1265         // This is because some PractRand tests may not be counted as systematic failures
1266         // as they have not been run to the same output length due to earlier failure of
1267         // another test.
1268         final int max = getMaxLengthExponent(results);
1269         if (max != 0) {
1270             list.add(bytesToString(max));
1271         }
1272         return list;
1273     }
1274 
1275     /**
1276      * Gets the maximum length exponent from the PractRand results if <strong>all</strong> failed.
1277      * Otherwise return zero (i.e. some passed the full length of the test).
1278      *
1279      * <p>This method excludes those results that are not complete. It assumes all complete
1280      * tests are for the same length of RNG output. Thus if all failed then the max exponent
1281      * is the systematic failure length.</p>
1282      *
1283      * @param results Results.
1284      * @return the maximum length exponent (or zero)
1285      */
1286     private static int getMaxLengthExponent(List<TestResult> results) {
1287         if (results.isEmpty()) {
1288             return 0;
1289         }
1290         // [0] = count of zeros
1291         // [1] = max non-zero
1292         final int[] data = new int[2];
1293         results.stream()
1294                .filter(TestResult::isComplete)
1295                .filter(r -> r instanceof PractRandTestResult)
1296                .mapToInt(r -> ((PractRandTestResult) r).getLengthExponent())
1297                .forEach(i -> {
1298                    if (i == 0) {
1299                        // Count results that passed
1300                        data[0]++;
1301                    } else {
1302                        // Find the max of the failures
1303                        data[1] = Math.max(i, data[1]);
1304                    }
1305                });
1306         // If all failed (i.e. no zeros) then return the max, otherwise zero.
1307         return data[0] == 0 ? data[1] : 0;
1308     }
1309 
1310     /**
1311      * Gets a summary of the failures across all results. The text is empty if there are no
1312      * failures to report.
1313      *
1314      * <p>For Dieharder and TestU01 this is the number of systematic failures (tests that fail
1315      * in every test result). For PractRand it is the maximum byte output size that was reached
1316      * before failure.
1317      *
1318      * <p>It is assumed all the results are for the same test suite.</p>
1319      *
1320      * @param results Results.
1321      * @return the failures summary
1322      */
1323     private static String getFailuresSummary(List<TestResult> results) {
1324         if (results.isEmpty()) {
1325             return "";
1326         }
1327         if (results.get(0).getTestFormat() == TestFormat.PRACTRAND) {
1328             final int max = getMaxLengthExponent(results);
1329             return max == 0 ? "" : bytesToString(max);
1330         }
1331         final int count = getSystematicFailures(results).size();
1332         return count == 0 ? "" : Integer.toString(count);
1333     }
1334 
1335     /**
1336      * Write the results as a text table.
1337      *
1338      * @param out Output stream.
1339      * @param results Results.
1340      * @throws IOException Signals that an I/O exception has occurred.
1341      */
1342     private static void writeTXT(OutputStream out,
1343                                  List<TestResult> results) throws IOException {
1344         // Identify all:
1345         // RandomSources, bit-reversed, test names,
1346         final List<RandomSource> randomSources = getRandomSources(results);
1347         final List<Boolean> bitReversed = getBitReversed(results);
1348         final List<String> testNames = getTestNames(results);
1349 
1350         // Create columns for RandomSource, bit-reversed, each test name.
1351         // Make bit-reversed column optional if no generators are bit reversed.
1352         final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);
1353 
1354         final List<List<String>> columns = createTXTColumns(testNames, showBitReversedColumn);
1355 
1356         // Add all data
1357         // This will collate results for each combination of 'RandomSource + bitReversed'
1358         for (final RandomSource randomSource : randomSources) {
1359             for (final boolean reversed : bitReversed) {
1360                 int i = 0;
1361                 columns.get(i++).add(randomSource.toString());
1362                 if (showBitReversedColumn) {
1363                     columns.get(i++).add(Boolean.toString(reversed));
1364                 }
1365                 for (final String testName : testNames) {
1366                     final List<TestResult> testResults = getTestResults(results, randomSource,
1367                             reversed, testName);
1368                     columns.get(i++).add(testResults.stream()
1369                                                     .map(TestResult::getFailureSummaryString)
1370                                                     .collect(Collectors.joining(",")));
1371                     columns.get(i++).add(getFailuresSummary(testResults));
1372                 }
1373             }
1374         }
1375 
1376         writeColumns(out, columns);
1377     }
1378 
1379     /**
1380      * Creates the columns for the text output.
1381      *
1382      * @param testNames the test names
1383      * @param showBitReversedColumn Set to true to show the bit reversed column
1384      * @return the list of columns
1385      */
1386     private static List<List<String>> createTXTColumns(final List<String> testNames,
1387         final boolean showBitReversedColumn) {
1388         final ArrayList<List<String>> columns = new ArrayList<>();
1389         columns.add(createColumn(COLUMN_RNG));
1390         if (showBitReversedColumn) {
1391             columns.add(createColumn(BIT_REVERSED));
1392         }
1393         for (final String testName : testNames) {
1394             columns.add(createColumn(testName));
1395             columns.add(createColumn("∩"));
1396         }
1397         return columns;
1398     }
1399 
1400     /**
1401      * Creates the column.
1402      *
1403      * @param columnName Column name.
1404      * @return the list
1405      */
1406     private static List<String> createColumn(String columnName) {
1407         final ArrayList<String> list = new ArrayList<>();
1408         list.add(columnName);
1409         return list;
1410     }
1411 
1412 
1413     /**
1414      * Creates the text format from column widths.
1415      *
1416      * @param columns Columns.
1417      * @return the text format string
1418      */
1419     private static String createTextFormatFromColumnWidths(final List<List<String>> columns) {
1420         final StringBuilder sb = new StringBuilder();
1421         try (Formatter formatter = new Formatter(sb)) {
1422             for (int i = 0; i < columns.size(); i++) {
1423                 if (i != 0) {
1424                     sb.append('\t');
1425                 }
1426                 formatter.format("%%-%ds", getColumnWidth(columns.get(i)));
1427             }
1428         }
1429         sb.append(System.lineSeparator());
1430         return sb.toString();
1431     }
1432 
1433     /**
1434      * Gets the column width using the maximum length of the column items.
1435      *
1436      * @param column Column.
1437      * @return the column width
1438      */
1439     private static int getColumnWidth(List<String> column) {
1440         int width = 0;
1441         for (final String text : column) {
1442             width = Math.max(width, text.length());
1443         }
1444         return width;
1445     }
1446 
1447     /**
1448      * Write the columns as fixed width text to the output.
1449      *
1450      * @param out Output stream.
1451      * @param columns Columns
1452      * @throws IOException Signals that an I/O exception has occurred.
1453      */
1454     private static void writeColumns(OutputStream out,
1455                                      List<List<String>> columns) throws IOException {
1456         // Create format using the column widths
1457         final String format = createTextFormatFromColumnWidths(columns);
1458 
1459         // Output
1460         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
1461              Formatter formatter = new Formatter(output)) {
1462             final int rows = columns.get(0).size();
1463             final Object[] args = new Object[columns.size()];
1464             for (int row = 0; row < rows; row++) {
1465                 for (int i = 0; i < args.length; i++) {
1466                     args[i] = columns.get(i).get(row);
1467                 }
1468                 formatter.format(format, args);
1469             }
1470         }
1471     }
1472 
1473     /**
1474      * Write the systematic failures as a text table.
1475      *
1476      * @param out Output stream.
1477      * @param results Results.
1478      * @throws IOException Signals that an I/O exception has occurred.
1479      */
1480     private static void writeFailures(OutputStream out,
1481                                       List<TestResult> results) throws IOException {
1482         // Identify all:
1483         // RandomSources, bit-reversed, test names,
1484         final List<RandomSource> randomSources = getRandomSources(results);
1485         final List<Boolean> bitReversed = getBitReversed(results);
1486         final List<String> testNames = getTestNames(results);
1487 
1488         // Create columns for RandomSource, bit-reversed, each test name.
1489         // Make bit-reversed column optional if no generators are bit reversed.
1490         final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);
1491 
1492         final List<List<String>> columns = createFailuresColumns(testNames, showBitReversedColumn);
1493 
1494         final AlphaNumericComparator cmp = new AlphaNumericComparator();
1495 
1496         // Add all data for each combination of 'RandomSource + bitReversed'
1497         for (final RandomSource randomSource : randomSources) {
1498             for (final boolean reversed : bitReversed) {
1499                 for (final String testName : testNames) {
1500                     final List<TestResult> testResults = getTestResults(results, randomSource,
1501                             reversed, testName);
1502                     final List<String> failures = getSystematicFailures(testResults);
1503                     if (failures.isEmpty()) {
1504                         continue;
1505                     }
1506                     Collections.sort(failures, cmp);
1507                     for (final String failed : failures) {
1508                         int i = 0;
1509                         columns.get(i++).add(randomSource.toString());
1510                         if (showBitReversedColumn) {
1511                             columns.get(i++).add(Boolean.toString(reversed));
1512                         }
1513                         columns.get(i++).add(testName);
1514                         columns.get(i).add(failed);
1515                     }
1516                 }
1517             }
1518         }
1519 
1520         writeColumns(out, columns);
1521     }
1522 
1523     /**
1524      * Creates the columns for the failures output.
1525      *
1526      * @param testNames the test names
1527      * @param showBitReversedColumn Set to true to show the bit reversed column
1528      * @return the list of columns
1529      */
1530     private static List<List<String>> createFailuresColumns(final List<String> testNames,
1531         final boolean showBitReversedColumn) {
1532         final ArrayList<List<String>> columns = new ArrayList<>();
1533         columns.add(createColumn(COLUMN_RNG));
1534         if (showBitReversedColumn) {
1535             columns.add(createColumn(BIT_REVERSED));
1536         }
1537         columns.add(createColumn("Test Suite"));
1538         columns.add(createColumn("Test"));
1539         return columns;
1540     }
1541 
1542     /**
1543      * Convert bytes to a human readable string. The byte size is expressed in powers of 2.
1544      * The output units use the ISO binary prefix for increments of 2<sup>10</sup> or 1024.
1545      *
1546      * <p>This is a utility function used for reporting PractRand output sizes.
1547      * Example output:
1548      *
1549      * <pre>
1550      *            exponent       Binary
1551      *                   0          0 B
1552      *                  10        1 KiB
1553      *                  13        8 KiB
1554      *                  20        1 MiB
1555      *                  27      128 MiB
1556      *                  30        1 GiB
1557      *                  40        1 TiB
1558      *                  50        1 PiB
1559      *                  60        1 EiB
1560      * </pre>
1561      *
1562      * @param exponent Exponent of the byte size (i.e. 2^exponent).
1563      * @return the string
1564      */
1565     static String bytesToString(int exponent) {
1566         final int unit = exponent / 10;
1567         final int size = 1 << (exponent - 10 * unit);
1568         return size + BINARY_UNITS[unit];
1569     }
1570 }