/*
    * Licensed to the Apache Software Foundation (ASF) under one or more
    * contributor license agreements.  See the NOTICE file distributed with
    * this work for additional information regarding copyright ownership.
    * The ASF licenses this file to You under the Apache License, Version 2.0
    * (the "License"); you may not use this file except in compliance with
    * the License.  You may obtain a copy of the License at
    *
    *      http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.commons.rng.examples.stress;
  
  import org.apache.commons.rng.simple.RandomSource;
  
  import picocli.CommandLine.Command;
  import picocli.CommandLine.Mixin;
  import picocli.CommandLine.Option;
  import picocli.CommandLine.Parameters;
  
  import java.io.BufferedReader;
  import java.io.BufferedWriter;
  import java.io.File;
  import java.io.FilterOutputStream;
  import java.io.IOException;
  import java.io.OutputStream;
  import java.io.OutputStreamWriter;
  import java.nio.charset.StandardCharsets;
  import java.nio.file.Files;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.EnumSet;
  import java.util.Formatter;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Iterator;
  import java.util.LinkedHashSet;
  import java.util.List;
  import java.util.ListIterator;
  import java.util.Map.Entry;
  import java.util.Set;
  import java.util.concurrent.Callable;
  import java.util.function.Function;
  import java.util.function.Supplier;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  import java.util.stream.Collectors;
  
  /**
   * Specification for the "results" command.
   *
   * <p>This command creates a summary of the results from known test applications.</p>
   *
   * <ul>
   *   <li>Dieharder
   *   <li>Test U01 (BigCrush, Crush, SmallCrush)
   * </ul>
   */
  @Command(name = "results",
           description = {"Collate results from stress test applications."})
  class ResultsCommand implements Callable<Void> {
      /** The pattern to identify the RandomSource in the stress test result header. */
      private static final Pattern RANDOM_SOURCE_PATTERN = Pattern.compile("^# RandomSource: (.*)");
      /** The pattern to identify the RNG in the stress test result header. */
      private static final Pattern RNG_PATTERN = Pattern.compile("^# RNG: (.*)");
      /** The pattern to identify the test exit code in the stress test result footer. */
      private static final Pattern TEST_EXIT_PATTERN = Pattern.compile("^# Exit value: (\\d+)");
      /** The pattern to identify the Dieharder test format. */
      private static final Pattern DIEHARDER_PATTERN = Pattern.compile("^# *dieharder version");
      /** The pattern to identify a Dieharder failed test result. */
      private static final Pattern DIEHARDER_FAILED_PATTERN = Pattern.compile("FAILED *$");
      /** The pattern to identify the Test U01 test format. */
      private static final Pattern TESTU01_PATTERN = Pattern.compile("^ *Version: TestU01");
      /** The pattern to identify the Test U01 summary header. */
      private static final Pattern TESTU01_SUMMARY_PATTERN = Pattern.compile("^========= Summary results of (\\S*) ");
      /** The pattern to identify the Test U01 failed test result. */
      private static final Pattern TESTU01_TEST_RESULT_PATTERN = Pattern.compile("^  ?(\\d+  .*)    ");
      /** The pattern to identify the Test U01 test starting entry. */
      private static final Pattern TESTU01_STARTING_PATTERN = Pattern.compile("^ *Starting (\\S*)");
      /** The pattern to identify the PractRand test format. */
      private static final Pattern PRACTRAND_PATTERN = Pattern.compile("PractRand version");
      /** The pattern to identify the PractRand output byte size. */
      private static final Pattern PRACTRAND_OUTPUT_SIZE_PATTERN = Pattern.compile("\\(2\\^(\\d+) bytes\\)");
      /** The pattern to identify a PractRand failed test result. */
      private static final Pattern PRACTRAND_FAILED_PATTERN = Pattern.compile("FAIL *!* *$");
      /** The name of the Dieharder sums test. */
      private static final String DIEHARDER_SUMS = "diehard_sums";
      /** The string identifying a bit-reversed generator. */
      private static final String BIT_REVERSED = "Bit-reversed";
      /** Character '\'. */
      private static final char FORWARD_SLASH = '\\';
      /** Character '/'. */
      private static final char BACK_SLASH = '\\';
      /** Character '|'. */
     private static final char PIPE = '|';
     /** The column name for the RNG. */
     private static final String COLUMN_RNG = "RNG";
     /** Constant for conversion of bytes to binary units, prefixed with a space. */
     private static final String[] BINARY_UNITS = {" B", " KiB", " MiB", " GiB", " TiB", " PiB", " EiB"};
 
     /** The standard options. */
     @Mixin
     private StandardOptions reusableOptions;
 
     /** The results files. */
     @Parameters(arity = "1..*",
                 description = "The results files.",
                 paramLabel = "<file>")
     private List<File> resultsFiles = new ArrayList<>();
 
     /** The file output prefix. */
     @Option(names = {"-o", "--out"},
             description = "The output file (default: stdout).")
     private File fileOutput;
 
     /** The output format. */
     @Option(names = {"-f", "--format"},
             description = {"Output format (default: ${DEFAULT-VALUE}).",
                            "Valid values: ${COMPLETION-CANDIDATES}."})
     private ResultsCommand.OutputFormat outputFormat = OutputFormat.TXT;
 
     /** The flag to show failed tests. */
     @Option(names = {"--failed"},
             description = {"Output failed tests (support varies by format).",
                            "CSV: List individual test failures.",
                            "APT: Count of systematic test failures."})
     private boolean showFailedTests;
 
     /** The flag to include the Dieharder sums test. */
     @Option(names = {"--include-sums"},
             description = "Include Dieharder sums test.")
     private boolean includeDiehardSums;
 
     /** The common file path prefix used when outputting file paths. */
     @Option(names = {"--path-prefix"},
             description = {"Common path prefix.",
                            "If specified this will replace the common prefix from all " +
                            "files when the path is output, e.g. for the APT report."})
     private String pathPrefix = "";
 
     /** The flag to ignore partial results. */
     @Option(names = {"-i", "--ignore"},
             description = "Ignore partial results.")
     private boolean ignorePartialResults;
 
     /** The flag to delete partial results files. */
     @Option(names = {"--delete"},
             description = {"Delete partial results files.",
                            "This is not reversible!"})
     private boolean deletePartialResults;
 
     /**
      * The output mode for the results.
      */
     enum OutputFormat {
         /** Comma-Separated Values (CSV) output. */
         CSV,
         /** Almost Plain Text (APT) output. */
         APT,
         /** Text table output. */
         TXT,
         /** Systematic failures output. */
         FAILURES,
     }
 
     /**
      * The test application file format.
      */
     enum TestFormat {
         /** Dieharder. */
         DIEHARDER,
         /** Test U01. */
         TESTU01,
         /** PractRand. */
         PRACTRAND,
     }
 
     /**
      * Encapsulate the results of a test application.
      */
     private static class TestResult {
         /** The result file. */
         private final File resultFile;
         /** The random source. */
         private final RandomSource randomSource;
         /** Flag indicating the generator was bit reversed. */
         private final boolean bitReversed;
         /** The test application format. */
         private final TestFormat testFormat;
         /** The names of the failed tests. */
         private final List<String> failedTests = new ArrayList<>();
         /** The test application name. */
         private String testApplicationName;
         /**
          * Store the exit code.
          * Initialised to {@link Integer#MIN_VALUE}. Exit values are expected to be 8-bit numbers
          * with zero for success.
          */
         private int exitCode = Integer.MIN_VALUE;
 
         /**
          * @param resultFile the result file
          * @param randomSource the random source
          * @param bitReversed the bit reversed flag
          * @param testFormat the test format
          */
         TestResult(File resultFile,
                    RandomSource randomSource,
                    boolean bitReversed,
                    TestFormat testFormat) {
             this.resultFile = resultFile;
             this.randomSource = randomSource;
             this.bitReversed = bitReversed;
             this.testFormat = testFormat;
         }
 
         /**
          * Adds the failed test.
          *
          * @param testId the test id
          */
         void addFailedTest(String testId) {
             failedTests.add(testId);
         }
 
         /**
          * Gets the result file.
          *
          * @return the result file
          */
         File getResultFile() {
             return resultFile;
         }
 
         /**
          * Gets the random source.
          *
          * @return the random source
          */
         RandomSource getRandomSource() {
             return randomSource;
         }
 
         /**
          * Checks if the generator was bit reversed.
          *
          * @return true if bit reversed
          */
         boolean isBitReversed() {
             return bitReversed;
         }
 
         /**
          * Gets the test format.
          *
          * @return the test format
          */
         TestFormat getTestFormat() {
             return testFormat;
         }
 
         /**
          * Gets the failed tests.
          *
          * @return the failed tests
          */
         List<String> getFailedTests() {
             return failedTests;
         }
 
         /**
          * Gets the failure count.
          *
          * @return the failure count
          */
         int getFailureCount() {
             return failedTests.size();
         }
 
         /**
          * Gets the a failure summary string.
          *
          * <p>The default implementation is the count of the number of failures.
          * This will be negative if the test is not complete.</p>
          *
          * @return the failure summary
          */
         String getFailureSummaryString() {
             return isComplete() ? Integer.toString(failedTests.size()) : "-" + failedTests.size();
         }
 
         /**
          * Sets the test application name.
          *
          * @param testApplicationName the new test application name
          */
         void setTestApplicationName(String testApplicationName) {
             this.testApplicationName = testApplicationName;
         }
 
         /**
          * Gets the test application name.
          *
          * @return the test application name
          */
         String getTestApplicationName() {
             return testApplicationName == null ? String.valueOf(getTestFormat()) : testApplicationName;
         }
 
         /**
          * Checks if the test result is complete.
          * This is {@code true} only if the exit code was found and is zero.
          *
          * @return true if complete
          */
         boolean isComplete() {
             return exitCode == 0;
         }
 
         /**
          * Sets the exit code flag.
          *
          * @param exitCode the new exit code
          */
         void setExitCode(int exitCode) {
             this.exitCode = exitCode;
         }
     }
 
     /**
      * Encapsulate the results of a PractRand test application. This is a specialisation that
      * allows handling PractRand results which are linked to the output length used by the RNG.
      */
     private static class PractRandTestResult extends TestResult {
         /** The length of the RNG output used to generate failed tests. */
         private int lengthExponent;
 
         /**
          * @param resultFile the result file
          * @param randomSource the random source
          * @param bitReversed the bit reversed flag
          * @param testFormat the test format
          */
         PractRandTestResult(File resultFile, RandomSource randomSource, boolean bitReversed, TestFormat testFormat) {
             super(resultFile, randomSource, bitReversed, testFormat);
         }
 
         /**
          * Gets the length of the RNG output used to generate failed tests.
          * If this is zero then no failures occurred.
          *
          * @return the length exponent
          */
         int getLengthExponent() {
             return lengthExponent;
         }
 
         /**
          * Sets the length of the RNG output used to generate failed tests.
          *
          * @param lengthExponent the length exponent
          */
         void setLengthExponent(int lengthExponent) {
             this.lengthExponent = lengthExponent;
         }
 
         /**
          * {@inheritDoc}
          *
          * <p>The PractRand summary is the exponent of the length of byte output from the RNG where
          * a failure occurred.
          */
         @Override
         String getFailureSummaryString() {
             return lengthExponent == 0 ? "-" : Integer.toString(lengthExponent);
         }
     }
 
     /**
      * Reads the results files and outputs in the chosen format.
      */
     @Override
     public Void call() {
         LogUtils.setLogLevel(reusableOptions.logLevel);
 
         // Read the results
         final List<TestResult> results = readResults();
 
         if (deletePartialResults) {
             deleteIfIncomplete(results);
             return null;
         }
 
         try (OutputStream out = createOutputStream()) {
             switch (outputFormat) {
             case CSV:
                 writeCSVData(out, results);
                 break;
             case APT:
                 writeAPT(out, results);
                 break;
             case TXT:
                 writeTXT(out, results);
                 break;
             case FAILURES:
                 writeFailures(out, results);
                 break;
             default:
                 throw new ApplicationException("Unknown output format: " + outputFormat);
             }
         } catch (final IOException ex) {
             throw new ApplicationException("IO error: " + ex.getMessage(), ex);
         }
         return null;
     }
 
     /**
      * Read the results.
      *
      * @return the results
      */
     private List<TestResult> readResults() {
         final ArrayList<TestResult> results = new ArrayList<>();
         for (final File resultFile : resultsFiles) {
             readResults(results, resultFile);
         }
         return results;
     }
 
     /**
      * Read the file and extract any test results.
      *
      * @param results Results.
      * @param resultFile Result file.
      * @throws ApplicationException If the results cannot be parsed.
      */
     private void readResults(List<TestResult> results,
                              File resultFile) {
         final List<String> contents = readFileContents(resultFile);
         // Files may have multiple test results per file (i.e. appended output)
         final List<List<String>> outputs = splitContents(contents);
         if (outputs.isEmpty()) {
             LogUtils.error("No test output in file: %s", resultFile);
         } else {
             for (final List<String> testOutput : outputs) {
                 final TestResult result = readResult(resultFile, testOutput);
                 if (!result.isComplete()) {
                     LogUtils.info("Partial results in file: %s", resultFile);
                     if (ignorePartialResults) {
                         continue;
                     }
                 }
                 results.add(result);
             }
         }
     }
 
     /**
      * Read the file contents.
      *
      * @param resultFile Result file.
      * @return the file contents
      * @throws ApplicationException If the file cannot be read.
      */
     private static List<String> readFileContents(File resultFile) {
         final ArrayList<String> contents = new ArrayList<>();
         try (BufferedReader reader = Files.newBufferedReader(resultFile.toPath())) {
             for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                 contents.add(line);
             }
         } catch (final IOException ex) {
             throw new ApplicationException("Failed to read file contents: " + resultFile, ex);
         }
         return contents;
     }
 
     /**
      * Split the file contents into separate test output. This is used in the event
      * that results have been appended to the same file.
      *
      * @param contents File contents.
      * @return the test output
      */
     private static List<List<String>> splitContents(List<String> contents) {
         final ArrayList<List<String>> testOutputs = new ArrayList<>();
         // Assume each output begins with e.g. # RandomSource: SPLIT_MIX_64
         // Note each beginning.
         int begin = -1;
         for (int i = 0; i < contents.size(); i++) {
             if (RANDOM_SOURCE_PATTERN.matcher(contents.get(i)).matches()) {
                 if (begin >= 0) {
                     testOutputs.add(contents.subList(begin, i));
                 }
                 begin = i;
             }
         }
         if (begin >= 0) {
             testOutputs.add(contents.subList(begin, contents.size()));
         }
         return testOutputs;
     }
 
     /**
      * Read the file into a test result.
      *
      * @param resultFile Result file.
      * @param testOutput Test output.
      * @return the test result
      * @throws ApplicationException If the result cannot be parsed.
      */
     private TestResult readResult(File resultFile,
                                   List<String> testOutput) {
         // Use an iterator for a single pass over the test output
         final ListIterator<String> iter = testOutput.listIterator();
 
         // Identify the RandomSource and bit reversed flag from the header:
         final RandomSource randomSource = getRandomSource(resultFile, iter);
         final boolean bitReversed = getBitReversed(resultFile, iter);
 
         // Identify the test application format. This may return null.
         final TestFormat testFormat = getTestFormat(resultFile, iter);
 
         // Read the application results
         final TestResult testResult = createTestResult(resultFile, randomSource, bitReversed, testFormat);
         if (testFormat == TestFormat.DIEHARDER) {
             readDieharder(iter, testResult);
         } else if (testFormat == TestFormat.TESTU01) {
             readTestU01(resultFile, iter, testResult);
         } else {
             readPractRand(iter, (PractRandTestResult) testResult);
         }
         return testResult;
     }
 
     /**
      * Creates the test result.
      *
      * @param resultFile Result file.
      * @param randomSource Random source.
      * @param bitReversed True if the random source was bit reversed.
      * @param testFormat Test format.
      * @return the test result
      */
     private static TestResult createTestResult(File resultFile, RandomSource randomSource,
                                                boolean bitReversed, TestFormat testFormat) {
         return testFormat == TestFormat.PRACTRAND ?
             new PractRandTestResult(resultFile, randomSource, bitReversed, testFormat) :
             new TestResult(resultFile, randomSource, bitReversed, testFormat);
     }
 
     /**
      * Gets the random source from the output header.
      *
      * @param resultFile Result file (for the exception message).
      * @param iter Iterator of the test output.
      * @return the random source
      * @throws ApplicationException If the RandomSource header line cannot be found.
      */
     private static RandomSource getRandomSource(File resultFile, Iterator<String> iter) {
         while (iter.hasNext()) {
             final Matcher matcher = RANDOM_SOURCE_PATTERN.matcher(iter.next());
             if (matcher.matches()) {
                 return RandomSource.valueOf(matcher.group(1));
             }
         }
         throw new ApplicationException("Failed to find RandomSource header line: " + resultFile);
     }
 
     /**
      * Gets the bit-reversed flag from the output header.
      *
      * @param resultFile Result file (for the exception message).
      * @param iter Iterator of the test output.
      * @return the bit-reversed flag
      * @throws ApplicationException If the RNG header line cannot be found.
      */
     private static boolean getBitReversed(File resultFile, Iterator<String> iter) {
         while (iter.hasNext()) {
             final Matcher matcher = RNG_PATTERN.matcher(iter.next());
             if (matcher.matches()) {
                 return matcher.group(1).contains(BIT_REVERSED);
             }
         }
         throw new ApplicationException("Failed to find RNG header line: " + resultFile);
     }
 
     /**
      * Gets the test format from the output. This scans the stdout produced by a test application.
      * If it is not recognized this may be a valid partial result or an unknown result. Throw
      * an exception if not allowing partial results, otherwise log an error.
      *
      * @param resultFile Result file (for the exception message).
      * @param iter Iterator of the test output.
      * @return the test format (or null)
      * @throws ApplicationException If the test format cannot be found.
      */
     private TestFormat getTestFormat(File resultFile, Iterator<String> iter) {
         while (iter.hasNext()) {
             final String line = iter.next();
             if (DIEHARDER_PATTERN.matcher(line).find()) {
                 return TestFormat.DIEHARDER;
             }
             if (TESTU01_PATTERN.matcher(line).find()) {
                 return TestFormat.TESTU01;
             }
             if (PRACTRAND_PATTERN.matcher(line).find()) {
                 return TestFormat.PRACTRAND;
             }
         }
         if (!ignorePartialResults) {
             throw new ApplicationException("Failed to identify the test application format: " + resultFile);
         }
         LogUtils.error("Failed to identify the test application format: %s", resultFile);
         return null;
     }
 
     /**
      * Read the result output from the Dieharder test application.
      *
      * @param iter Iterator of the test output.
      * @param testResult Test result.
      */
     private void readDieharder(Iterator<String> iter,
                                TestResult testResult) {
         // Dieharder results are printed in-line using the following format:
         //#=============================================================================#
         //        test_name   |ntup| tsamples |psamples|  p-value |Assessment
         //#=============================================================================#
         //   diehard_birthdays|   0|       100|     100|0.97484422|  PASSED
         //   ...
         //        diehard_oqso|   0|   2097152|     100|0.00000000|  FAILED
 
         testResult.setTestApplicationName("Dieharder");
 
         // Identify any line containing FAILED and then get the test ID using
         // the first two columns (test_name + ntup).
         while (iter.hasNext()) {
             final String line = iter.next();
             if (DIEHARDER_FAILED_PATTERN.matcher(line).find()) {
                 // Optionally include the flawed Dieharder sums test
                 if (!includeDiehardSums && line.contains(DIEHARDER_SUMS)) {
                     continue;
                 }
                 final int index1 = line.indexOf('|');
                 final int index2 = line.indexOf('|', index1 + 1);
                 testResult.addFailedTest(line.substring(0, index1).trim() + ":" +
                                          line.substring(index1 + 1, index2).trim());
             } else if (findExitCode(testResult, line)) {
                 return;
             }
         }
     }
 
     /**
      * Find the exit code in the line. Update the test result with the code if found.
      *
      * @param testResult Test result.
      * @param line Line from the test result output.
      * @return true, if the exit code was found
      */
     private static boolean findExitCode(TestResult testResult, String line) {
         final Matcher matcher = TEST_EXIT_PATTERN.matcher(line);
         if (matcher.find()) {
             testResult.setExitCode(Integer.parseInt(matcher.group(1)));
             return true;
         }
         return false;
     }
 
     /**
      * Read the result output from the Test U01 test application.
      *
      * <p>Test U01 outputs a summary of results at the end of the test output. If this cannot
      * be found the method will throw an exception unless partial results are allowed.</p>
      *
      * @param resultFile Result file (for the exception message).
      * @param iter Iterator of the test output.
      * @param testResult Test result.
      * @throws ApplicationException If the TestU01 summary cannot be found.
      */
     private void readTestU01(File resultFile,
                              ListIterator<String> iter,
                              TestResult testResult) {
         // Results are summarised at the end of the file:
         //========= Summary results of BigCrush =========
         //
         //Version:          TestU01 1.2.3
         //Generator:        stdin
         //Number of statistics:  155
         //Total CPU time:   06:14:50.43
         //The following tests gave p-values outside [0.001, 0.9990]:
         //(eps  means a value < 1.0e-300):
         //(eps1 means a value < 1.0e-15):
         //
         //      Test                          p-value
         //----------------------------------------------
         // 1  SerialOver, r = 0                eps
         // 3  CollisionOver, t = 2           1 - eps1
         // 5  CollisionOver, t = 3           1 - eps1
         // 7  CollisionOver, t = 7             eps
 
         // Identify the summary line
         final String testSuiteName = skipToTestU01Summary(resultFile, iter);
 
         // This may not be present if the results are not complete
         if (testSuiteName == null) {
             // Rewind
             while (iter.hasPrevious()) {
                 iter.previous();
             }
             updateTestU01ApplicationName(iter, testResult);
             return;
         }
 
         setTestU01ApplicationName(testResult, testSuiteName);
 
         // Read test results using the entire line except the p-value for the test Id
         // Note:
         // This will count sub-parts of the same test as distinct failures.
         while (iter.hasNext()) {
             final String line = iter.next();
             final Matcher matcher = TESTU01_TEST_RESULT_PATTERN.matcher(line);
             if (matcher.find()) {
                 testResult.addFailedTest(matcher.group(1).trim());
             } else if (findExitCode(testResult, line)) {
                 return;
             }
         }
     }
 
     /**
      * Sets the Test U01 application name using the provide test suite name.
      *
      * @param testResult Test result.
      * @param testSuiteName Test suite name.
      */
     private static void setTestU01ApplicationName(TestResult testResult, String testSuiteName) {
         testResult.setTestApplicationName("TestU01 (" + testSuiteName + ")");
     }
 
     /**
      * Skip to the Test U01 result summary.
      *
      * <p>If this cannot be found the method will throw an exception unless partial results
      * are allowed.</p>
      *
      * @param resultFile Result file (for the exception message).
      * @param iter Iterator of the test output.
      * @return the name of the test suite
      * @throws ApplicationException If the TestU01 summary cannot be found.
      */
     private String skipToTestU01Summary(File resultFile, Iterator<String> iter) {
         final String testSuiteName = findMatcherGroup1(iter, TESTU01_SUMMARY_PATTERN);
         // Allow the partial result to be ignored
         if (testSuiteName != null || ignorePartialResults) {
             return testSuiteName;
         }
         throw new ApplicationException("Failed to identify the Test U01 result summary: " + resultFile);
     }
 
     /**
      * Update the test application name from the Test U01 results. This can be used to identify
      * the test suite from the start of the results in the event that the results summary has
      * not been output.
      *
      * @param iter Iterator of the test output.
      * @param testResult Test result.
      */
     private static void updateTestU01ApplicationName(Iterator<String> iter,
                                                      TestResult testResult) {
         final String testSuiteName = findMatcherGroup1(iter, TESTU01_STARTING_PATTERN);
         if (testSuiteName != null) {
             setTestU01ApplicationName(testResult, testSuiteName);
         }
     }
 
     /**
      * Create a matcher for each item in the iterator and if a match is identified return
      * group 1.
      *
      * @param iter Iterator of text to match.
      * @param pattern Pattern to match.
      * @return the string (or null)
      */
     private static String findMatcherGroup1(Iterator<String> iter,
                                             Pattern pattern) {
         while (iter.hasNext()) {
             final String line = iter.next();
             final Matcher matcher = pattern.matcher(line);
             if (matcher.find()) {
                 return matcher.group(1);
             }
         }
         return null;
     }
 
     /**
      * Read the result output from the PractRand test application.
      *
      * @param iter Iterator of the test output.
      * @param testResult Test result.
      */
     private static void readPractRand(Iterator<String> iter,
                                       PractRandTestResult testResult) {
         // PractRand results are printed for blocks of byte output that double in size.
         // The results report 'unusual' and 'suspicious' output and the test stops
         // at the first failure.
         // Results are typically reported as the size of output that failed, e.g.
         // the JDK random fails at 256KiB.
         //
         // rng=RNG_stdin32, seed=0xfc6c7332
         // length= 128 kilobytes (2^17 bytes), time= 5.9 seconds
         //   no anomalies in 118 test result(s)
         //
         // rng=RNG_stdin32, seed=0xfc6c7332
         // length= 256 kilobytes (2^18 bytes), time= 7.6 seconds
         //   Test Name                         Raw       Processed     Evaluation
         //   [Low1/64]Gap-16:A                 R= +12.4  p =  8.7e-11   VERY SUSPICIOUS
         //   [Low1/64]Gap-16:B                 R= +13.4  p =  3.0e-11    FAIL
         //   [Low8/32]DC6-9x1Bytes-1           R=  +7.6  p =  5.7e-4   unusual
         //   ...and 136 test result(s) without anomalies
 
         testResult.setTestApplicationName("PractRand");
 
         // Store the exponent of the current output length.
         int exp = 0;
 
         // Identify any line containing FAIL and then get the test name using
         // the first token in the line.
         while (iter.hasNext()) {
             String line = iter.next();
             final Matcher matcher = PRACTRAND_OUTPUT_SIZE_PATTERN.matcher(line);
             if (matcher.find()) {
                 // Store the current output length
                 exp = Integer.parseInt(matcher.group(1));
             } else if (PRACTRAND_FAILED_PATTERN.matcher(line).find()) {
                 // Record the output length where failures occurred.
                 testResult.setLengthExponent(exp);
                 // Add the failed test name. This does not include the length exponent
                 // allowing meta-processing of systematic failures.
                 // Remove initial whitespace
                 line = line.trim();
                 final int index = line.indexOf(' ');
                 testResult.addFailedTest(line.substring(0, index));
             } else if (findExitCode(testResult, line)) {
                 return;
             }
         }
     }
 
     /**
      * Delete any result file if incomplete.
      *
      * @param results Results.
      * @throws ApplicationException If a file could not be deleted.
      */
     private static void deleteIfIncomplete(List<TestResult> results) {
         results.forEach(ResultsCommand::deleteIfIncomplete);
     }
 
     /**
      * Delete the result file if incomplete.
      *
      * @param result Test result.
      * @throws ApplicationException If the file could not be deleted.
      */
     private static void deleteIfIncomplete(TestResult result) {
         if (!result.isComplete()) {
             try {
                 Files.delete(result.getResultFile().toPath());
                 LogUtils.info("Deleted file: %s", result.getResultFile());
             } catch (IOException ex) {
                 throw new ApplicationException("Failed to delete file: " + result.getResultFile(), ex);
             }
         }
     }
 
     /**
      * Creates the output stream. This will not be buffered.
      *
      * @return the output stream
      */
     private OutputStream createOutputStream() {
         if (fileOutput != null) {
             try {
                 return Files.newOutputStream(fileOutput.toPath());
             } catch (final IOException ex) {
                 throw new ApplicationException("Failed to create output: " + fileOutput, ex);
             }
         }
         return new FilterOutputStream(System.out) {
             @Override
             public void close() {
                 // Do not close stdout
             }
         };
     }
 
     /**
      * Write the results to a table in Comma Separated Value (CSV) format.
      *
      * @param out Output stream.
      * @param results Results.
      * @throws IOException Signals that an I/O exception has occurred.
      */
     private void writeCSVData(OutputStream out,
                               List<TestResult> results) throws IOException {
         // Sort by columns
         Collections.sort(results, (o1, o2) -> {
             int result = Integer.compare(o1.getRandomSource().ordinal(), o2.getRandomSource().ordinal());
             if (result != 0) {
                 return result;
             }
             result = Boolean.compare(o1.isBitReversed(), o2.isBitReversed());
             if (result != 0) {
                 return result;
             }
             result = o1.getTestApplicationName().compareTo(o2.getTestApplicationName());
             if (result != 0) {
                 return result;
             }
             return Integer.compare(o1.getFailureCount(), o2.getFailureCount());
         });
 
         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
             output.write("RandomSource,Bit-reversed,Test,Failures");
             if (showFailedTests) {
                 output.write(",Failed");
             }
             output.newLine();
             for (final TestResult result : results) {
                 output.write(result.getRandomSource().toString());
                 output.write(',');
                 output.write(Boolean.toString(result.isBitReversed()));
                 output.write(',');
                 output.write(result.getTestApplicationName());
                 output.write(',');
                 output.write(result.getFailureSummaryString());
                 // Optionally write out failed test names.
                 if (showFailedTests) {
                     output.write(',');
                     output.write(result.getFailedTests().stream().collect(Collectors.joining("|")));
                 }
                 output.newLine();
             }
         }
     }
 
     /**
      * Write the results using the Almost Plain Text (APT) format for a table. This
      * table can be included in the documentation for the Commons RNG site.
      *
      * @param out Output stream.
      * @param results Results.
      * @throws IOException Signals that an I/O exception has occurred.
      */
     private void writeAPT(OutputStream out,
                           List<TestResult> results) throws IOException {
         // Identify all:
         // RandomSources, bit-reversed, test names,
         final List<RandomSource> randomSources = getRandomSources(results);
         final List<Boolean> bitReversed = getBitReversed(results);
         final List<String> testNames = getTestNames(results);
 
         // Identify the common path prefix to be replaced
         final int prefixLength = (pathPrefix.isEmpty()) ? 0 : findCommonPathPrefixLength(results);
 
         // Create a function to update the file path and then output the failure count
         // as a link to the file using the APT format.
         final Function<TestResult, String> toAPTString = result -> {
             String path = result.getResultFile().getPath();
             // Remove common path prefix
             path = path.substring(prefixLength);
             // Build the APT relative link
             final StringBuilder sb = new StringBuilder()
                 .append("{{{").append(pathPrefix).append(path).append('}')
                 .append(result.getFailureSummaryString()).append("}}");
             // Convert to web-link name separators
             for (int i = 0; i < sb.length(); i++) {
                 if (sb.charAt(i) == BACK_SLASH) {
                     sb.setCharAt(i, FORWARD_SLASH);
                 }
             }
             return sb.toString();
         };
 
         // Create columns for RandomSource, bit-reversed, each test name.
         // Make bit-reversed column optional if no generators are bit reversed.
         final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);
 
         final String header = createAPTHeader(showBitReversedColumn, testNames);
         final String separator = createAPTSeparator(header);
 
         // Output
         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
             // For the first line using '*' (centre) character instead of '+' (left align)
             output.write(separator.replace('+', '*'));
             output.write(header);
             output.newLine();
             output.write(separator);
 
             final StringBuilder sb = new StringBuilder();
 
             // This will collate results for each combination of 'RandomSource + bitReversed'
             for (final RandomSource randomSource : randomSources) {
                 for (final boolean reversed : bitReversed) {
                     // Highlight in bold a RNG with no systematic failures
                     boolean highlight = true;
 
                     // Buffer the column output
                     sb.setLength(0);
 
                     if (showBitReversedColumn) {
                         writeAPTColumn(sb, Boolean.toString(reversed), false);
                     }
                     for (final String testName : testNames) {
                         final List<TestResult> testResults = getTestResults(results, randomSource, reversed, testName);
                         String text = testResults.stream()
                                                  .map(toAPTString)
                                                  .collect(Collectors.joining(", "));
                         // Summarise the failures across all tests
                         final String summary = getFailuresSummary(testResults);
                         if (!summary.isEmpty()) {
                             // Identify RNGs with no systematic failures
                             highlight = false;
                             if (showFailedTests) {
                                 // Add summary in brackets
                                 text += " (" + summary + ")";
                             }
                         }
                         writeAPTColumn(sb, text, false);
                     }
 
                     output.write('|');
                     writeAPTColumn(output, randomSource.toString(), highlight);
                     output.write(sb.toString());
                     output.newLine();
                     output.write(separator);
                 }
             }
         }
     }
 
     /**
      * Gets the random sources present in the results.
      *
      * @param results Results.
      * @return the random sources
      */
     private static List<RandomSource> getRandomSources(List<TestResult> results) {
         final EnumSet<RandomSource> set = EnumSet.noneOf(RandomSource.class);
         for (final TestResult result : results) {
             set.add(result.getRandomSource());
         }
         final ArrayList<RandomSource> list = new ArrayList<>(set);
         Collections.sort(list);
         return list;
     }
 
     /**
      * Gets the bit-reversed options present in the results.
      *
      * @param results Results.
      * @return the bit-reversed options
      */
     private static List<Boolean> getBitReversed(List<TestResult> results) {
         final ArrayList<Boolean> list = new ArrayList<>(2);
         if (results.isEmpty()) {
             // Default to no bit-reversed results
             list.add(Boolean.FALSE);
         } else {
             final boolean first = results.get(0).isBitReversed();
             list.add(first);
             for (final TestResult result : results) {
                 if (first != result.isBitReversed()) {
                     list.add(!first);
                     break;
                 }
             }
         }
         Collections.sort(list);
         return list;
     }
 
     /**
      * Gets the test names present in the results. These are returned in encountered order.
      *
      * @param results Results.
      * @return the test names
      */
     private static List<String> getTestNames(List<TestResult> results) {
         // Enforce encountered order with a linked hash set.
         final Set<String> set = new LinkedHashSet<>();
         for (final TestResult result : results) {
             set.add(result.getTestApplicationName());
         }
         return new ArrayList<>(set);
     }
 
     /**
      * Find the common path prefix for all result files. This is returned as the length of the
      * common prefix.
      *
      * @param results Results.
      * @return the length
      */
     private static int findCommonPathPrefixLength(List<TestResult> results) {
         if (results.isEmpty()) {
             return 0;
         }
         // Find the first prefix
         final String prefix1 = getPathPrefix(results.get(0));
         int length = prefix1.length();
         for (int i = 1; i < results.size() && length != 0; i++) {
             final String prefix2 = getPathPrefix(results.get(i));
             // Update
             final int size = Math.min(prefix2.length(), length);
             length = 0;
             while (length < size && prefix1.charAt(length) == prefix2.charAt(length)) {
                 length++;
             }
         }
         return length;
     }
 
     /**
      * Gets the path prefix.
      *
      * @param testResult Test result.
      * @return the path prefix (or the empty string)
      */
     private static String getPathPrefix(TestResult testResult) {
         final String parent = testResult.getResultFile().getParent();
         return parent == null ? "" : parent;
     }
 
     /**
      * Creates the APT header.
      *
      * @param showBitReversedColumn Set to true to the show bit reversed column.
      * @param testNames Test names.
      * @return the header
      */
     private static String createAPTHeader(boolean showBitReversedColumn,
                                           List<String> testNames) {
         final StringBuilder sb = new StringBuilder(100).append("|| RNG identifier ||");
         if (showBitReversedColumn) {
             sb.append(" Bit-reversed ||");
         }
         for (final String name : testNames) {
             sb.append(' ').append(name).append(" ||");
         }
         return sb.toString();
     }
 
     /**
      * Creates the APT separator for each table row.
      *
      * <p>The separator is created using the '+' character to left align the columns.
      *
      * @param header Header.
      * @return the separator
      */
     private static String createAPTSeparator(String header) {
         // Replace everything with '-' except '|' which is replaced with "*-" for the first
         // character, "+-" for all other occurrences except "-+" at the end
         final StringBuilder sb = new StringBuilder(header);
         for (int i = 0; i < header.length(); i++) {
             if (sb.charAt(i) == PIPE) {
                 sb.setCharAt(i, i == 0 ? '*' : '+');
                 sb.setCharAt(i + 1,  '-');
             } else {
                 sb.setCharAt(i,  '-');
             }
         }
         // Fix the end
         sb.setCharAt(header.length() - 2, '-');
         sb.setCharAt(header.length() - 1, '+');
         sb.append(System.lineSeparator());
         return sb.toString();
     }
 
     /**
      * Write the column text to the output.
      *
      * @param output Output.
      * @param text Text.
      * @param highlight If {@code true} highlight the text in bold.
      * @throws IOException Signals that an I/O exception has occurred.
      */
     private static void writeAPTColumn(Appendable output,
                                        String text,
                                        boolean highlight) throws IOException {
         output.append(' ');
         if (highlight) {
             output.append("<<");
         }
         output.append(text);
         if (highlight) {
             output.append(">>");
         }
         output.append(" |");
     }
 
     /**
      * Gets the test results that match the arguments.
      *
      * @param results Results.
      * @param randomSource Random source.
      * @param bitReversed Bit reversed flag.
      * @param testName Test name.
      * @return the matching results
      */
     private static List<TestResult> getTestResults(List<TestResult> results,
                                                    RandomSource randomSource,
                                                    boolean bitReversed,
                                                    String testName) {
         final ArrayList<TestResult> list = new ArrayList<>();
         for (final TestResult result : results) {
             if (result.getRandomSource() == randomSource &&
                 result.bitReversed == bitReversed &&
                 result.getTestApplicationName().equals(testName)) {
                 list.add(result);
             }
         }
         return list;
     }
 
     /**
      * Gets the systematic failures (tests that fail in every test result).
      *
      * @param results Results.
      * @return the systematic failures
      */
     private static List<String> getSystematicFailures(List<TestResult> results) {
         final HashMap<String, Integer> map = new HashMap<>();
         for (final TestResult result : results) {
             // Ignore partial results
             if (!result.isComplete()) {
                 continue;
             }
             // Some named tests can fail more than once on different statistics.
             // For example TestU01 BigCrush LongestHeadRun can output in the summary:
             // 86  LongestHeadRun, r = 0            eps
             // 86  LongestHeadRun, r = 0          1 - eps1
             // This will be counted as 2 failed tests. For the purpose of systematic
             // failures the name of the test is the same and should be counted once.
             final HashSet<String> unique = new HashSet<>(result.getFailedTests());
             for (final String test : unique) {
                 map.merge(test, 1, (i, j) -> i + j);
             }
         }
         final int completeCount = (int) results.stream().filter(TestResult::isComplete).count();
         final List<String> list = map.entrySet().stream()
                                                 .filter(e -> e.getValue() == completeCount)
                                                 .map(Entry::getKey)
                                                 .collect(Collectors.toCollection(
                                                     (Supplier<List<String>>) ArrayList::new));
         // Special case for PractRand. Add the maximum RNG output length before failure.
         // This is because some PractRand tests may not be counted as systematic failures
         // as they have not been run to the same output length due to earlier failure of
         // another test.
         final int max = getMaxLengthExponent(results);
         if (max != 0) {
             list.add(bytesToString(max));
         }
         return list;
     }
 
     /**
      * Gets the maximum length exponent from the PractRand results if <strong>all</strong> failed.
      * Otherwise return zero (i.e. some passed the full length of the test).
      *
      * <p>This method excludes those results that are not complete. It assumes all complete
      * tests are for the same length of RNG output. Thus if all failed then the max exponent
      * is the systematic failure length.</p>
      *
      * @param results Results.
      * @return the maximum length exponent (or zero)
      */
     private static int getMaxLengthExponent(List<TestResult> results) {
         if (results.isEmpty()) {
             return 0;
         }
         // [0] = count of zeros
         // [1] = max non-zero
         final int[] data = new int[2];
         results.stream()
                .filter(TestResult::isComplete)
                .filter(r -> r instanceof PractRandTestResult)
                .mapToInt(r -> ((PractRandTestResult) r).getLengthExponent())
                .forEach(i -> {
                    if (i == 0) {
                        // Count results that passed
                        data[0]++;
                    } else {
                        // Find the max of the failures
                        data[1] = Math.max(i, data[1]);
                    }
                });
         // If all failed (i.e. no zeros) then return the max, otherwise zero.
         return data[0] == 0 ? data[1] : 0;
     }
 
     /**
      * Gets a summary of the failures across all results. The text is empty if there are no
      * failures to report.
      *
      * <p>For Dieharder and TestU01 this is the number of systematic failures (tests that fail
      * in every test result). For PractRand it is the maximum byte output size that was reached
      * before failure.
      *
      * <p>It is assumed all the results are for the same test suite.</p>
      *
      * @param results Results.
      * @return the failures summary
      */
     private static String getFailuresSummary(List<TestResult> results) {
         if (results.isEmpty()) {
             return "";
         }
         if (results.get(0).getTestFormat() == TestFormat.PRACTRAND) {
             final int max = getMaxLengthExponent(results);
             return max == 0 ? "" : bytesToString(max);
         }
         final int count = getSystematicFailures(results).size();
         return count == 0 ? "" : Integer.toString(count);
     }
 
     /**
      * Write the results as a text table.
      *
      * @param out Output stream.
      * @param results Results.
      * @throws IOException Signals that an I/O exception has occurred.
      */
     private static void writeTXT(OutputStream out,
                                  List<TestResult> results) throws IOException {
         // Identify all:
         // RandomSources, bit-reversed, test names,
         final List<RandomSource> randomSources = getRandomSources(results);
         final List<Boolean> bitReversed = getBitReversed(results);
         final List<String> testNames = getTestNames(results);
 
         // Create columns for RandomSource, bit-reversed, each test name.
         // Make bit-reversed column optional if no generators are bit reversed.
         final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);
 
         final List<List<String>> columns = createTXTColumns(testNames, showBitReversedColumn);
 
         // Add all data
         // This will collate results for each combination of 'RandomSource + bitReversed'
         for (final RandomSource randomSource : randomSources) {
             for (final boolean reversed : bitReversed) {
                 int i = 0;
                 columns.get(i++).add(randomSource.toString());
                 if (showBitReversedColumn) {
                     columns.get(i++).add(Boolean.toString(reversed));
                 }
                 for (final String testName : testNames) {
                     final List<TestResult> testResults = getTestResults(results, randomSource,
                             reversed, testName);
                     columns.get(i++).add(testResults.stream()
                                                     .map(TestResult::getFailureSummaryString)
                                                     .collect(Collectors.joining(",")));
                     columns.get(i++).add(getFailuresSummary(testResults));
                 }
             }
         }
 
         writeColumns(out, columns);
     }
 
     /**
      * Creates the columns for the text output.
      *
      * @param testNames the test names
      * @param showBitReversedColumn Set to true to show the bit reversed column
      * @return the list of columns
      */
     private static List<List<String>> createTXTColumns(final List<String> testNames,
         final boolean showBitReversedColumn) {
         final ArrayList<List<String>> columns = new ArrayList<>();
         columns.add(createColumn(COLUMN_RNG));
         if (showBitReversedColumn) {
             columns.add(createColumn(BIT_REVERSED));
         }
         for (final String testName : testNames) {
             columns.add(createColumn(testName));
             columns.add(createColumn("∩"));
         }
         return columns;
     }
 
     /**
      * Creates the column.
      *
      * @param columnName Column name.
      * @return the list
      */
     private static List<String> createColumn(String columnName) {
         final ArrayList<String> list = new ArrayList<>();
         list.add(columnName);
         return list;
     }
 
 
     /**
      * Creates the text format from column widths.
      *
      * @param columns Columns.
      * @return the text format string
      */
     private static String createTextFormatFromColumnWidths(final List<List<String>> columns) {
         final StringBuilder sb = new StringBuilder();
         try (Formatter formatter = new Formatter(sb)) {
             for (int i = 0; i < columns.size(); i++) {
                 if (i != 0) {
                     sb.append('\t');
                 }
                 formatter.format("%%-%ds", getColumnWidth(columns.get(i)));
             }
         }
         sb.append(System.lineSeparator());
         return sb.toString();
     }
 
     /**
      * Gets the column width using the maximum length of the column items.
      *
      * @param column Column.
      * @return the column width
      */
     private static int getColumnWidth(List<String> column) {
         int width = 0;
         for (final String text : column) {
             width = Math.max(width, text.length());
         }
         return width;
     }
 
     /**
      * Write the columns as fixed width text to the output.
      *
      * @param out Output stream.
      * @param columns Columns
      * @throws IOException Signals that an I/O exception has occurred.
      */
     private static void writeColumns(OutputStream out,
                                      List<List<String>> columns) throws IOException {
         // Create format using the column widths
         final String format = createTextFormatFromColumnWidths(columns);
 
         // Output
         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
              Formatter formatter = new Formatter(output)) {
             final int rows = columns.get(0).size();
             final Object[] args = new Object[columns.size()];
             for (int row = 0; row < rows; row++) {
                 for (int i = 0; i < args.length; i++) {
                     args[i] = columns.get(i).get(row);
                 }
                 formatter.format(format, args);
             }
         }
     }
 
     /**
      * Write the systematic failures as a text table.
      *
      * @param out Output stream.
      * @param results Results.
      * @throws IOException Signals that an I/O exception has occurred.
      */
     private static void writeFailures(OutputStream out,
                                       List<TestResult> results) throws IOException {
         // Identify all:
         // RandomSources, bit-reversed, test names,
         final List<RandomSource> randomSources = getRandomSources(results);
         final List<Boolean> bitReversed = getBitReversed(results);
         final List<String> testNames = getTestNames(results);
 
         // Create columns for RandomSource, bit-reversed, each test name.
         // Make bit-reversed column optional if no generators are bit reversed.
         final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);
 
         final List<List<String>> columns = createFailuresColumns(testNames, showBitReversedColumn);
 
         final AlphaNumericComparator cmp = new AlphaNumericComparator();
 
         // Add all data for each combination of 'RandomSource + bitReversed'
         for (final RandomSource randomSource : randomSources) {
             for (final boolean reversed : bitReversed) {
                 for (final String testName : testNames) {
                     final List<TestResult> testResults = getTestResults(results, randomSource,
                             reversed, testName);
                     final List<String> failures = getSystematicFailures(testResults);
                     if (failures.isEmpty()) {
                         continue;
                     }
                     Collections.sort(failures, cmp);
                     for (final String failed : failures) {
                         int i = 0;
                         columns.get(i++).add(randomSource.toString());
                         if (showBitReversedColumn) {
                             columns.get(i++).add(Boolean.toString(reversed));
                         }
                         columns.get(i++).add(testName);
                         columns.get(i).add(failed);
                     }
                 }
             }
         }
 
         writeColumns(out, columns);
     }
 
     /**
      * Creates the columns for the failures output.
      *
      * @param testNames the test names
      * @param showBitReversedColumn Set to true to show the bit reversed column
      * @return the list of columns
      */
     private static List<List<String>> createFailuresColumns(final List<String> testNames,
         final boolean showBitReversedColumn) {
         final ArrayList<List<String>> columns = new ArrayList<>();
         columns.add(createColumn(COLUMN_RNG));
         if (showBitReversedColumn) {
             columns.add(createColumn(BIT_REVERSED));
         }
         columns.add(createColumn("Test Suite"));
         columns.add(createColumn("Test"));
         return columns;
     }
 
     /**
      * Convert bytes to a human readable string. The byte size is expressed in powers of 2.
      * The output units use the ISO binary prefix for increments of 2<sup>10</sup> or 1024.
      *
      * <p>This is a utility function used for reporting PractRand output sizes.
      * Example output:
      *
      * <pre>
      *            exponent       Binary
      *                   0          0 B
      *                  10        1 KiB
      *                  13        8 KiB
      *                  20        1 MiB
      *                  27      128 MiB
      *                  30        1 GiB
      *                  40        1 TiB
      *                  50        1 PiB
      *                  60        1 EiB
      * </pre>
      *
      * @param exponent Exponent of the byte size (i.e. 2^exponent).
      * @return the string
      */
     static String bytesToString(int exponent) {
         final int unit = exponent / 10;
         final int size = 1 << (exponent - 10 * unit);
         return size + BINARY_UNITS[unit];
     }
 }