View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.rng.examples.stress;
18  
19  import org.apache.commons.rng.UniformRandomProvider;
20  import org.apache.commons.rng.core.source64.RandomLongSource;
21  import org.apache.commons.rng.simple.RandomSource;
22  
23  import picocli.CommandLine.Command;
24  import picocli.CommandLine.Mixin;
25  import picocli.CommandLine.Option;
26  import picocli.CommandLine.Parameters;
27  
28  import java.io.BufferedWriter;
29  import java.io.File;
30  import java.io.FilterOutputStream;
31  import java.io.IOException;
32  import java.io.OutputStream;
33  import java.io.OutputStreamWriter;
34  import java.io.Writer;
35  import java.nio.ByteOrder;
36  import java.nio.charset.StandardCharsets;
37  import java.nio.file.Files;
38  import java.util.ArrayList;
39  import java.util.Formatter;
40  import java.util.List;
41  import java.util.concurrent.Callable;
42  
43  /**
44   * Specification for the "output" command.
45   *
46   * <p>This command creates a named random generator and outputs data in a specified format.</p>
47   */
48  @Command(name = "output",
49           description = {"Output data from a named random data generator."})
50  class OutputCommand implements Callable<Void> {
51      /** The new line characters. */
52      private static final String NEW_LINE = System.lineSeparator();
53      /** Character '['. */
54      private static final char LEFT_SQUARE_BRACKET = '[';
55      /** Character ']'. */
56      private static final char RIGHT_SQUARE_BRACKET = ']';
57  
58      /** Lookup table for binary representation of bytes. */
59      private static final String[] BIT_REP = {
60          "0000", "0001", "0010", "0011",
61          "0100", "0101", "0110", "0111",
62          "1000", "1001", "1010", "1011",
63          "1100", "1101", "1110", "1111",
64      };
65  
66      /** The standard options. */
67      @Mixin
68      private StandardOptions reusableOptions;
69  
70      /** The random source. */
71      @Parameters(index = "0",
72                  description = "The random source.")
73      private RandomSource randomSource;
74  
75      /** The executable arguments. */
76      @Parameters(index = "1..*",
77                  description = "The arguments to pass to the constructor.",
78                  paramLabel = "<argument>")
79      private List<String> arguments = new ArrayList<>();
80  
81      /** The file output prefix. */
82      @Option(names = {"-o", "--out"},
83              description = "The output file (default: stdout).")
84      private File fileOutput;
85  
86      /** The output format. */
87      @Option(names = {"-f", "--format"},
88              description = {"Output format (default: ${DEFAULT-VALUE}).",
89                             "Valid values: ${COMPLETION-CANDIDATES}."})
90      private OutputCommand.OutputFormat outputFormat = OutputFormat.DIEHARDER;
91  
92      /** The random seed. */
93      @Option(names = {"-s", "--seed"},
94              description = {"The 64-bit number random seed (default: auto)."})
95      private Long seed;
96  
97      /** The random seed as a byte[]. */
98      @Option(names = {"-x", "--hex-seed"},
99              description = {"The hex-encoded random seed.",
100                            "Seed conversion for multi-byte primitives use little-endian format.",
101                            "Over-rides the --seed parameter."})
102     private String byteSeed;
103 
104     /** The count of numbers to output. */
105     @Option(names = {"-n", "--count"},
106             description = {"The count of numbers to output.",
107                            "Use negative for an unlimited stream."})
108     private long count = 10;
109 
110     /** The size of the byte buffer for the binary data. */
111     @Option(names = {"--buffer-size"},
112             description = {"Byte-buffer size for binary data (default: ${DEFAULT-VALUE}).",
113                            "When outputing binary data the count parameter controls the " +
114                            "number of buffers written."})
115     private int bufferSize = 8192;
116 
117     /** The output byte order of the binary data. */
118     @Option(names = {"-b", "--byte-order"},
119             description = {"Byte-order of the output data (default: ${DEFAULT-VALUE}).",
120                            "Uses the Java default of big-endian. This may not match the platform byte-order.",
121                            "Valid values: BIG_ENDIAN, LITTLE_ENDIAN."})
122     private ByteOrder byteOrder = ByteOrder.BIG_ENDIAN;
123 
124     /** The output byte order of the binary data. */
125     @Option(names = {"-r", "--reverse-bits"},
126             description = {"Reverse the bits in the data (default: ${DEFAULT-VALUE})."})
127     private boolean reverseBits;
128 
129     /** Flag to use 64-bit long output. */
130     @Option(names = {"--raw64"},
131             description = {"Use 64-bit output (default is 32-bit).",
132                            "This is ignored if not a native 64-bit generator.",
133                            "Set to true sets the source64 mode to LONG."})
134     private boolean raw64;
135 
136     /** Output mode for 64-bit long output. */
137     @Option(names = {"--source64"},
138             description = {"Output mode for 64-bit generators (default: ${DEFAULT-VALUE}).",
139                            "This is ignored if not a native 64-bit generator.",
140                            "In 32-bit mode the output uses a combination of upper and " +
141                            "lower bits of the 64-bit value.",
142                            "Valid values: ${COMPLETION-CANDIDATES}."})
143     private Source64Mode source64 = RNGUtils.getSource64Default();
144 
145     /**
146      * The output mode for existing files.
147      */
148     enum OutputFormat {
149         /** Binary output. */
150         BINARY,
151         /** Use the Dieharder text format. */
152         DIEHARDER,
153         /** Output the bits in a text format. */
154         BITS,
155     }
156 
157     /**
158      * Validates the command arguments, creates the generator and outputs numbers.
159      */
160     @Override
161     public Void call() {
162         LogUtils.setLogLevel(reusableOptions.logLevel);
163         final Object objectSeed = createSeed();
164         UniformRandomProvider rng = createRNG(objectSeed);
165 
166         // raw64 flag overrides the source64 mode
167         if (raw64) {
168             source64 = Source64Mode.LONG;
169         }
170         if (source64 == Source64Mode.LONG && !(rng instanceof RandomLongSource)) {
171             throw new ApplicationException("Not a 64-bit RNG: " + rng);
172         }
173 
174         // Upper or lower bits from 64-bit generators must be created first.
175         // Note this does not test source64 != Source64Mode.LONG as the full long
176         // output split into hi-lo or lo-hi is supported by the RngDataOutput.
177         if (rng instanceof RandomLongSource &&
178             (source64 == Source64Mode.HI || source64 == Source64Mode.LO || source64 == Source64Mode.INT)) {
179             rng = RNGUtils.createIntProvider((UniformRandomProvider & RandomLongSource) rng, source64);
180         }
181         if (reverseBits) {
182             rng = RNGUtils.createReverseBitsProvider(rng);
183         }
184 
185         // -------
186         // Note: Manipulation of the byte order for the platform is done during output
187         // for the binary format. Otherwise do it in Java.
188         // -------
189         if (outputFormat != OutputFormat.BINARY) {
190             rng = toOutputFormat(rng);
191         }
192 
193         try (OutputStream out = createOutputStream()) {
194             switch (outputFormat) {
195             case BINARY:
196                 writeBinaryData(rng, out);
197                 break;
198             case DIEHARDER:
199                 writeDieharder(rng, out);
200                 break;
201             case BITS:
202                 writeBitData(rng, out);
203                 break;
204             default:
205                 throw new ApplicationException("Unknown output format: " + outputFormat);
206             }
207         } catch (IOException ex) {
208             throw new ApplicationException("IO error: " + ex.getMessage(), ex);
209         }
210         return null;
211     }
212 
213     /**
214      * Creates the seed.
215      *
216      * @return the seed
217      */
218     private Object createSeed() {
219         if (byteSeed != null) {
220             try {
221                 return Hex.decodeHex(byteSeed);
222             } catch (IllegalArgumentException ex) {
223                 throw new ApplicationException("Invalid hex seed: " + ex.getMessage(), ex);
224             }
225         }
226         if (seed != null) {
227             return seed;
228         }
229         // Let the factory constructor create the native seed.
230         return null;
231     }
232 
233     /**
234      * Creates the seed.
235      *
236      * @return the seed
237      */
238     private String createSeedString() {
239         if (byteSeed != null) {
240             return byteSeed;
241         }
242         if (seed != null) {
243             return seed.toString();
244         }
245         return "auto";
246     }
247 
248     /**
249      * Creates the RNG.
250      *
251      * @param objectSeed Seed.
252      * @return the uniform random provider
253      * @throws ApplicationException If the RNG cannot be created
254      */
255     private UniformRandomProvider createRNG(Object objectSeed) {
256         if (randomSource == null) {
257             throw new ApplicationException("Random source is null");
258         }
259         final ArrayList<Object> data = new ArrayList<>();
260         // Note: The list command outputs arguments as an array bracketed by [ and ]
261         // Strip these for convenience.
262         stripArrayFormatting(arguments);
263 
264         for (final String argument : arguments) {
265             data.add(RNGUtils.parseArgument(argument));
266         }
267         try {
268             return randomSource.create(objectSeed, data.toArray());
269         } catch (IllegalStateException | IllegalArgumentException ex) {
270             throw new ApplicationException("Failed to create RNG: " + randomSource + ". " + ex.getMessage(), ex);
271         }
272     }
273 
274     /**
275      * Strip leading bracket from the first argument, trailing bracket from the last
276      * argument, and any trailing commas from any argument.
277      *
278      * <p>This is used to remove the array formatting used by the list command.
279      *
280      * @param arguments the arguments
281      */
282     private static void stripArrayFormatting(List<String> arguments) {
283         final int size = arguments.size();
284         if (size > 1) {
285             // These will not be empty as they were created from command-line args.
286             final String first = arguments.get(0);
287             if (first.charAt(0) == LEFT_SQUARE_BRACKET) {
288                 arguments.set(0, first.substring(1));
289             }
290             final String last = arguments.get(size - 1);
291             if (last.charAt(last.length() - 1) == RIGHT_SQUARE_BRACKET) {
292                 arguments.set(size - 1, last.substring(0, last.length() - 1));
293             }
294         }
295         for (int i = 0; i < size; i++) {
296             final String argument = arguments.get(i);
297             if (argument.endsWith(",")) {
298                 arguments.set(i, argument.substring(0, argument.length() - 1));
299             }
300         }
301     }
302 
303     /**
304      * Convert the native RNG to the requested output format. This will convert a 64-bit
305      * generator to a 32-bit generator unless the 64-bit mode is active. It then optionally
306      * reverses the byte order of the output.
307      *
308      * @param rng The random generator.
309      * @return the uniform random provider
310      */
311     private UniformRandomProvider toOutputFormat(UniformRandomProvider rng) {
312         UniformRandomProvider convertedRng = rng;
313         if (rng instanceof RandomLongSource && source64 != Source64Mode.LONG) {
314             // Convert to 32-bit generator
315             convertedRng = RNGUtils.createIntProvider((UniformRandomProvider & RandomLongSource) rng, source64);
316         }
317         if (byteOrder == ByteOrder.LITTLE_ENDIAN) {
318             convertedRng = RNGUtils.createReverseBytesProvider(convertedRng);
319         }
320         return convertedRng;
321     }
322 
323     /**
324      * Creates the output stream. This will not be buffered.
325      *
326      * @return the output stream
327      */
328     private OutputStream createOutputStream() {
329         if (fileOutput != null) {
330             try {
331                 return Files.newOutputStream(fileOutput.toPath());
332             } catch (IOException ex) {
333                 throw new ApplicationException("Failed to create output: " + fileOutput, ex);
334             }
335         }
336         return new FilterOutputStream(System.out) {
337             @Override
338             public void close() {
339                 // Do not close stdout
340             }
341         };
342     }
343 
344     /**
345      * Check the count is positive, otherwise create an error message for the provided format.
346      *
347      * @param count The count of numbers to output.
348      * @param format The format.
349      * @throws ApplicationException If the count is not positive.
350      */
351     private static void checkCount(long count,
352                                    OutputFormat format) {
353         if (count <= 0) {
354             throw new ApplicationException(format + " format requires a positive count: " + count);
355         }
356     }
357 
358     /**
359      * Write int data to the specified output using the dieharder text format.
360      *
361      * @param rng The random generator.
362      * @param out The output.
363      * @throws IOException Signals that an I/O exception has occurred.
364      * @throws ApplicationException If the count is not positive.
365      */
366     private void writeDieharder(final UniformRandomProvider rng,
367                                 final OutputStream out) throws IOException {
368         checkCount(count, OutputFormat.DIEHARDER);
369 
370         // Use dieharder output, e.g.
371         //#==================================================================
372         //# generator mt19937  seed = 1
373         //#==================================================================
374         //type: d
375         //count: 1
376         //numbit: 32
377         //1791095845
378         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
379             writeHeaderLine(output);
380             output.write("# generator ");
381             output.write(rng.toString());
382             output.write("  seed = ");
383             output.write(createSeedString());
384             output.write(NEW_LINE);
385             writeHeaderLine(output);
386             output.write("type: d");
387             output.write(NEW_LINE);
388             output.write("count: ");
389             output.write(Long.toString(count));
390             output.write(NEW_LINE);
391             output.write("numbit: 32");
392             output.write(NEW_LINE);
393             for (long c = 0; c < count; c++) {
394                 // Unsigned integers
395                 final String text = Long.toString(rng.nextInt() & 0xffffffffL);
396                 // Left pad with spaces
397                 for (int i = 10 - text.length(); i > 0; i--) {
398                     output.write(' ');
399                 }
400                 output.write(text);
401                 output.write(NEW_LINE);
402             }
403         }
404     }
405 
406     /**
407      * Write a header line to the output.
408      *
409      * @param output the output
410      * @throws IOException Signals that an I/O exception has occurred.
411      */
412     private static void writeHeaderLine(Writer output) throws IOException {
413         output.write("#==================================================================");
414         output.write(NEW_LINE);
415     }
416 
417     /**
418      * Write raw binary data to the output.
419      *
420      * @param rng The random generator.
421      * @param out The output.
422      * @throws IOException Signals that an I/O exception has occurred.
423      */
424     private void writeBinaryData(final UniformRandomProvider rng,
425                                  final OutputStream out) throws IOException {
426         // If count is not positive use max value.
427         // This is effectively unlimited: program must be killed.
428         final long limit = (count < 1) ? Long.MAX_VALUE : count;
429         try (RngDataOutput data = RNGUtils.createDataOutput(rng, source64, out, bufferSize, byteOrder)) {
430             for (long c = 0; c < limit; c++) {
431                 data.write(rng);
432             }
433         }
434     }
435 
436     /**
437      * Write binary bit data to the specified file.
438      *
439      * @param rng The random generator.
440      * @param out The output.
441      * @throws IOException Signals that an I/O exception has occurred.
442      * @throws ApplicationException If the count is not positive.
443      */
444     private void writeBitData(final UniformRandomProvider rng,
445                               final OutputStream out) throws IOException {
446         checkCount(count, OutputFormat.BITS);
447 
448         final boolean asLong = rng instanceof RandomLongSource;
449 
450         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
451             for (long c = 0; c < count; c++) {
452                 if (asLong) {
453                     writeLong(output, rng.nextLong());
454                 } else {
455                     writeInt(output, rng.nextInt());
456                 }
457             }
458         }
459     }
460 
461     /**
462      * Write an {@code long} value to the output. The native Java value will be
463      * written to the writer on a single line using: a binary string representation
464      * of the bytes; the unsigned integer; and the signed integer.
465      *
466      * <pre>
467      * 10011010 01010011 01011010 11100100 01000111 00010000 01000011 11000101  11120331841399178181 -7326412232310373435
468      * </pre>
469      *
470      * @param out The output.
471      * @param value The value.
472      * @throws IOException Signals that an I/O exception has occurred.
473      */
474     @SuppressWarnings("resource")
475     static void writeLong(Writer out,
476                           long value) throws IOException {
477 
478         // Write out as 8 bytes with spaces between them, high byte first.
479         writeByte(out, (int)(value >>> 56) & 0xff);
480         out.write(' ');
481         writeByte(out, (int)(value >>> 48) & 0xff);
482         out.write(' ');
483         writeByte(out, (int)(value >>> 40) & 0xff);
484         out.write(' ');
485         writeByte(out, (int)(value >>> 32) & 0xff);
486         out.write(' ');
487         writeByte(out, (int)(value >>> 24) & 0xff);
488         out.write(' ');
489         writeByte(out, (int)(value >>> 16) & 0xff);
490         out.write(' ');
491         writeByte(out, (int)(value >>>  8) & 0xff);
492         out.write(' ');
493         writeByte(out, (int)(value >>>  0) & 0xff);
494 
495         // Write the unsigned and signed int value
496         new Formatter(out).format("  %20s %20d%n", Long.toUnsignedString(value), value);
497     }
498 
499     /**
500      * Write an {@code int} value to the output. The native Java value will be
501      * written to the writer on a single line using: a binary string representation
502      * of the bytes; the unsigned integer; and the signed integer.
503      *
504      * <pre>
505      * 11001101 00100011 01101111 01110000   3441651568  -853315728
506      * </pre>
507      *
508      * @param out The output.
509      * @param value The value.
510      * @throws IOException Signals that an I/O exception has occurred.
511      */
512     @SuppressWarnings("resource")
513     static void writeInt(Writer out,
514                          int value) throws IOException {
515 
516         // Write out as 4 bytes with spaces between them, high byte first.
517         writeByte(out, (value >>> 24) & 0xff);
518         out.write(' ');
519         writeByte(out, (value >>> 16) & 0xff);
520         out.write(' ');
521         writeByte(out, (value >>>  8) & 0xff);
522         out.write(' ');
523         writeByte(out, (value >>>  0) & 0xff);
524 
525         // Write the unsigned and signed int value
526         new Formatter(out).format("  %10d %11d%n", value & 0xffffffffL, value);
527     }
528 
529     /**
530      * Write the lower 8 bits of an {@code int} value to the buffered writer using a
531      * binary string representation. This is left-filled with zeros if applicable.
532      *
533      * <pre>
534      * 11001101
535      * </pre>
536      *
537      * @param out The output.
538      * @param value The value.
539      * @throws IOException Signals that an I/O exception has occurred.
540      */
541     private static void writeByte(Writer out,
542                                   int value) throws IOException {
543         // This matches the functionality of:
544         // data.write(String.format("%8s", Integer.toBinaryString(value & 0xff)).replace(' ', '0'))
545         out.write(BIT_REP[value >>> 4]);
546         out.write(BIT_REP[value & 0x0F]);
547     }
548 }