1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.commons.compress.compressors;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.security.AccessController;
25 import java.security.PrivilegedAction;
26 import java.util.Collections;
27 import java.util.Locale;
28 import java.util.ServiceLoader;
29 import java.util.Set;
30 import java.util.SortedMap;
31 import java.util.TreeMap;
32
33 import org.apache.commons.compress.compressors.brotli.BrotliCompressorInputStream;
34 import org.apache.commons.compress.compressors.brotli.BrotliUtils;
35 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
36 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
37 import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
38 import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
39 import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream;
40 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
41 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
42 import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream;
43 import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorOutputStream;
44 import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
45 import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorOutputStream;
46 import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream;
47 import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream;
48 import org.apache.commons.compress.compressors.lzma.LZMAUtils;
49 import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
50 import org.apache.commons.compress.compressors.pack200.Pack200CompressorOutputStream;
51 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
52 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
53 import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
54 import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
55 import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
56 import org.apache.commons.compress.compressors.xz.XZUtils;
57 import org.apache.commons.compress.compressors.z.ZCompressorInputStream;
58 import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
59 import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
60 import org.apache.commons.compress.compressors.zstandard.ZstdUtils;
61 import org.apache.commons.compress.utils.IOUtils;
62 import org.apache.commons.compress.utils.Sets;
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91 public class CompressorStreamFactory implements CompressorStreamProvider {
92
93 private static final CompressorStreamFactory SINGLETON = new CompressorStreamFactory();
94
95
96
97
98
99
100 public static final String BROTLI = "br";
101
102
103
104
105
106
107 public static final String BZIP2 = "bzip2";
108
109
110
111
112
113
114 public static final String GZIP = "gz";
115
116
117
118
119
120
121 public static final String PACK200 = "pack200";
122
123
124
125
126
127
128 public static final String XZ = "xz";
129
130
131
132
133
134
135 public static final String LZMA = "lzma";
136
137
138
139
140
141
142 public static final String SNAPPY_FRAMED = "snappy-framed";
143
144
145
146
147
148
149 public static final String SNAPPY_RAW = "snappy-raw";
150
151
152
153
154
155
156 public static final String Z = "z";
157
158
159
160
161
162
163 public static final String DEFLATE = "deflate";
164
165
166
167
168
169
170 public static final String DEFLATE64 = "deflate64";
171
172
173
174
175
176
177 public static final String LZ4_BLOCK = "lz4-block";
178
179
180
181
182
183
184 public static final String LZ4_FRAMED = "lz4-framed";
185
186
187
188
189
190
191 public static final String ZSTANDARD = "zstd";
192
193 private static final String YOU_NEED_BROTLI_DEC = youNeed("Google Brotli Dec", "https://github.com/google/brotli/");
194 private static final String YOU_NEED_XZ_JAVA = youNeed("XZ for Java", "https://tukaani.org/xz/java.html");
195 private static final String YOU_NEED_ZSTD_JNI = youNeed("Zstd JNI", "https://github.com/luben/zstd-jni");
196
197 private static final Set<String> ALL_NAMES = Sets.newHashSet(BZIP2, GZIP, PACK200, SNAPPY_FRAMED, Z, DEFLATE, XZ, LZMA, LZ4_FRAMED, ZSTANDARD);
198
199 private static Iterable<CompressorStreamProvider> archiveStreamProviderIterable() {
200 return ServiceLoader.load(CompressorStreamProvider.class, ClassLoader.getSystemClassLoader());
201 }
202
203
204
205
206
207
208
209
210
211
212
213 public static String detect(final InputStream inputStream) throws CompressorException {
214 return detect(inputStream, ALL_NAMES);
215 }
216
217
218
219
220
221
222
223
224
225
226 static String detect(final InputStream inputStream, final Set<String> compressorNames) throws CompressorException {
227 if (inputStream == null) {
228 throw new IllegalArgumentException("Stream must not be null.");
229 }
230
231 if (compressorNames == null || compressorNames.isEmpty()) {
232 throw new IllegalArgumentException("Compressor names cannot be null or empty");
233 }
234
235 if (!inputStream.markSupported()) {
236 throw new IllegalArgumentException("Mark is not supported.");
237 }
238
239 final byte[] signature = new byte[12];
240 inputStream.mark(signature.length);
241 int signatureLength = -1;
242 try {
243 signatureLength = IOUtils.readFully(inputStream, signature);
244 inputStream.reset();
245 } catch (final IOException e) {
246 throw new CompressorException("IOException while reading signature.", e);
247 }
248
249 if (compressorNames.contains(BZIP2) && BZip2CompressorInputStream.matches(signature, signatureLength)) {
250 return BZIP2;
251 }
252
253 if (compressorNames.contains(GZIP) && GzipCompressorInputStream.matches(signature, signatureLength)) {
254 return GZIP;
255 }
256
257 if (compressorNames.contains(PACK200) && Pack200CompressorInputStream.matches(signature, signatureLength)) {
258 return PACK200;
259 }
260
261 if (compressorNames.contains(SNAPPY_FRAMED) && FramedSnappyCompressorInputStream.matches(signature, signatureLength)) {
262 return SNAPPY_FRAMED;
263 }
264
265 if (compressorNames.contains(Z) && ZCompressorInputStream.matches(signature, signatureLength)) {
266 return Z;
267 }
268
269 if (compressorNames.contains(DEFLATE) && DeflateCompressorInputStream.matches(signature, signatureLength)) {
270 return DEFLATE;
271 }
272
273 if (compressorNames.contains(XZ) && XZUtils.matches(signature, signatureLength)) {
274 return XZ;
275 }
276
277 if (compressorNames.contains(LZMA) && LZMAUtils.matches(signature, signatureLength)) {
278 return LZMA;
279 }
280
281 if (compressorNames.contains(LZ4_FRAMED) && FramedLZ4CompressorInputStream.matches(signature, signatureLength)) {
282 return LZ4_FRAMED;
283 }
284
285 if (compressorNames.contains(ZSTANDARD) && ZstdUtils.matches(signature, signatureLength)) {
286 return ZSTANDARD;
287 }
288
289 throw new CompressorException("No Compressor found for the stream signature.");
290 }
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312 public static SortedMap<String, CompressorStreamProvider> findAvailableCompressorInputStreamProviders() {
313 return AccessController.doPrivileged((PrivilegedAction<SortedMap<String, CompressorStreamProvider>>) () -> {
314 final TreeMap<String, CompressorStreamProvider> map = new TreeMap<>();
315 putAll(SINGLETON.getInputStreamCompressorNames(), SINGLETON, map);
316 archiveStreamProviderIterable().forEach(provider -> putAll(provider.getInputStreamCompressorNames(), provider, map));
317 return map;
318 });
319 }
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341 public static SortedMap<String, CompressorStreamProvider> findAvailableCompressorOutputStreamProviders() {
342 return AccessController.doPrivileged((PrivilegedAction<SortedMap<String, CompressorStreamProvider>>) () -> {
343 final TreeMap<String, CompressorStreamProvider> map = new TreeMap<>();
344 putAll(SINGLETON.getOutputStreamCompressorNames(), SINGLETON, map);
345 archiveStreamProviderIterable().forEach(provider -> putAll(provider.getOutputStreamCompressorNames(), provider, map));
346 return map;
347 });
348 }
349
350 public static String getBrotli() {
351 return BROTLI;
352 }
353
354 public static String getBzip2() {
355 return BZIP2;
356 }
357
358 public static String getDeflate() {
359 return DEFLATE;
360 }
361
362
363
364
365
366 public static String getDeflate64() {
367 return DEFLATE64;
368 }
369
370 public static String getGzip() {
371 return GZIP;
372 }
373
374 public static String getLZ4Block() {
375 return LZ4_BLOCK;
376 }
377
378 public static String getLZ4Framed() {
379 return LZ4_FRAMED;
380 }
381
382 public static String getLzma() {
383 return LZMA;
384 }
385
386 public static String getPack200() {
387 return PACK200;
388 }
389
390 public static CompressorStreamFactory getSingleton() {
391 return SINGLETON;
392 }
393
394 public static String getSnappyFramed() {
395 return SNAPPY_FRAMED;
396 }
397
398 public static String getSnappyRaw() {
399 return SNAPPY_RAW;
400 }
401
402 public static String getXz() {
403 return XZ;
404 }
405
406 public static String getZ() {
407 return Z;
408 }
409
410 public static String getZstandard() {
411 return ZSTANDARD;
412 }
413
414 static void putAll(final Set<String> names, final CompressorStreamProvider provider, final TreeMap<String, CompressorStreamProvider> map) {
415 names.forEach(name -> map.put(toKey(name), provider));
416 }
417
418 private static String toKey(final String name) {
419 return name.toUpperCase(Locale.ROOT);
420 }
421
422 private static String youNeed(final String name, final String url) {
423 return " In addition to Apache Commons Compress you need the " + name + " library - see " + url;
424 }
425
426
427
428
429
430 private final Boolean decompressUntilEOF;
431
432
433
434
435
436 private SortedMap<String, CompressorStreamProvider> compressorInputStreamProviders;
437
438 private SortedMap<String, CompressorStreamProvider> compressorOutputStreamProviders;
439
440
441
442
443
444 private volatile boolean decompressConcatenated;
445
446 private final int memoryLimitInKb;
447
448
449
450
451 public CompressorStreamFactory() {
452 this.decompressUntilEOF = null;
453 this.memoryLimitInKb = -1;
454 }
455
456
457
458
459
460
461
462
463 public CompressorStreamFactory(final boolean decompressUntilEOF) {
464 this(decompressUntilEOF, -1);
465 }
466
467
468
469
470
471
472
473
474
475
476
477 public CompressorStreamFactory(final boolean decompressUntilEOF, final int memoryLimitInKb) {
478 this.decompressUntilEOF = decompressUntilEOF;
479
480
481 this.decompressConcatenated = decompressUntilEOF;
482 this.memoryLimitInKb = memoryLimitInKb;
483 }
484
485
486
487
488
489
490
491
492
493
494
495 public CompressorInputStream createCompressorInputStream(final InputStream in) throws CompressorException {
496 return createCompressorInputStream(detect(in), in);
497 }
498
499
500
501
502
503
504
505
506
507
508
509
510 public CompressorInputStream createCompressorInputStream(final InputStream in, final Set<String> compressorNames) throws CompressorException {
511 return createCompressorInputStream(detect(in, compressorNames), in);
512 }
513
514
515
516
517
518
519
520
521
522
523
524
525
526 public CompressorInputStream createCompressorInputStream(final String name, final InputStream in) throws CompressorException {
527 return createCompressorInputStream(name, in, decompressConcatenated);
528 }
529
530 @Override
531 public CompressorInputStream createCompressorInputStream(final String name, final InputStream in, final boolean actualDecompressConcatenated)
532 throws CompressorException {
533 if (name == null || in == null) {
534 throw new IllegalArgumentException("Compressor name and stream must not be null.");
535 }
536
537 try {
538
539 if (GZIP.equalsIgnoreCase(name)) {
540 return new GzipCompressorInputStream(in, actualDecompressConcatenated);
541 }
542
543 if (BZIP2.equalsIgnoreCase(name)) {
544 return new BZip2CompressorInputStream(in, actualDecompressConcatenated);
545 }
546
547 if (BROTLI.equalsIgnoreCase(name)) {
548 if (!BrotliUtils.isBrotliCompressionAvailable()) {
549 throw new CompressorException("Brotli compression is not available." + YOU_NEED_BROTLI_DEC);
550 }
551 return new BrotliCompressorInputStream(in);
552 }
553
554 if (XZ.equalsIgnoreCase(name)) {
555 if (!XZUtils.isXZCompressionAvailable()) {
556 throw new CompressorException("XZ compression is not available." + YOU_NEED_XZ_JAVA);
557 }
558 return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb);
559 }
560
561 if (ZSTANDARD.equalsIgnoreCase(name)) {
562 if (!ZstdUtils.isZstdCompressionAvailable()) {
563 throw new CompressorException("Zstandard compression is not available." + YOU_NEED_ZSTD_JNI);
564 }
565 return new ZstdCompressorInputStream(in);
566 }
567
568 if (LZMA.equalsIgnoreCase(name)) {
569 if (!LZMAUtils.isLZMACompressionAvailable()) {
570 throw new CompressorException("LZMA compression is not available" + YOU_NEED_XZ_JAVA);
571 }
572 return new LZMACompressorInputStream(in, memoryLimitInKb);
573 }
574
575 if (PACK200.equalsIgnoreCase(name)) {
576 return new Pack200CompressorInputStream(in);
577 }
578
579 if (SNAPPY_RAW.equalsIgnoreCase(name)) {
580 return new SnappyCompressorInputStream(in);
581 }
582
583 if (SNAPPY_FRAMED.equalsIgnoreCase(name)) {
584 return new FramedSnappyCompressorInputStream(in);
585 }
586
587 if (Z.equalsIgnoreCase(name)) {
588 return new ZCompressorInputStream(in, memoryLimitInKb);
589 }
590
591 if (DEFLATE.equalsIgnoreCase(name)) {
592 return new DeflateCompressorInputStream(in);
593 }
594
595 if (DEFLATE64.equalsIgnoreCase(name)) {
596 return new Deflate64CompressorInputStream(in);
597 }
598
599 if (LZ4_BLOCK.equalsIgnoreCase(name)) {
600 return new BlockLZ4CompressorInputStream(in);
601 }
602
603 if (LZ4_FRAMED.equalsIgnoreCase(name)) {
604 return new FramedLZ4CompressorInputStream(in, actualDecompressConcatenated);
605 }
606
607 } catch (final IOException e) {
608 throw new CompressorException("Could not create CompressorInputStream.", e);
609 }
610 final CompressorStreamProvider compressorStreamProvider = getCompressorInputStreamProviders().get(toKey(name));
611 if (compressorStreamProvider != null) {
612 return compressorStreamProvider.createCompressorInputStream(name, in, actualDecompressConcatenated);
613 }
614
615 throw new CompressorException("Compressor: " + name + " not found.");
616 }
617
618
619
620
621
622
623
624
625
626
627
628 @Override
629 public CompressorOutputStream<?> createCompressorOutputStream(final String name, final OutputStream out) throws CompressorException {
630 if (name == null || out == null) {
631 throw new IllegalArgumentException("Compressor name and stream must not be null.");
632 }
633
634 try {
635
636 if (GZIP.equalsIgnoreCase(name)) {
637 return new GzipCompressorOutputStream(out);
638 }
639
640 if (BZIP2.equalsIgnoreCase(name)) {
641 return new BZip2CompressorOutputStream(out);
642 }
643
644 if (XZ.equalsIgnoreCase(name)) {
645 return new XZCompressorOutputStream(out);
646 }
647
648 if (PACK200.equalsIgnoreCase(name)) {
649 return new Pack200CompressorOutputStream(out);
650 }
651
652 if (LZMA.equalsIgnoreCase(name)) {
653 return new LZMACompressorOutputStream(out);
654 }
655
656 if (DEFLATE.equalsIgnoreCase(name)) {
657 return new DeflateCompressorOutputStream(out);
658 }
659
660 if (SNAPPY_FRAMED.equalsIgnoreCase(name)) {
661 return new FramedSnappyCompressorOutputStream(out);
662 }
663
664 if (LZ4_BLOCK.equalsIgnoreCase(name)) {
665 return new BlockLZ4CompressorOutputStream(out);
666 }
667
668 if (LZ4_FRAMED.equalsIgnoreCase(name)) {
669 return new FramedLZ4CompressorOutputStream(out);
670 }
671
672 if (ZSTANDARD.equalsIgnoreCase(name)) {
673 return new ZstdCompressorOutputStream(out);
674 }
675 } catch (final IOException e) {
676 throw new CompressorException("Could not create CompressorOutputStream", e);
677 }
678 final CompressorStreamProvider compressorStreamProvider = getCompressorOutputStreamProviders().get(toKey(name));
679 if (compressorStreamProvider != null) {
680 return compressorStreamProvider.createCompressorOutputStream(name, out);
681 }
682 throw new CompressorException("Compressor: " + name + " not found.");
683 }
684
685 public SortedMap<String, CompressorStreamProvider> getCompressorInputStreamProviders() {
686 if (compressorInputStreamProviders == null) {
687 compressorInputStreamProviders = Collections.unmodifiableSortedMap(findAvailableCompressorInputStreamProviders());
688 }
689 return compressorInputStreamProviders;
690 }
691
692 public SortedMap<String, CompressorStreamProvider> getCompressorOutputStreamProviders() {
693 if (compressorOutputStreamProviders == null) {
694 compressorOutputStreamProviders = Collections.unmodifiableSortedMap(findAvailableCompressorOutputStreamProviders());
695 }
696 return compressorOutputStreamProviders;
697 }
698
699
700 boolean getDecompressConcatenated() {
701 return decompressConcatenated;
702 }
703
704 public Boolean getDecompressUntilEOF() {
705 return decompressUntilEOF;
706 }
707
708 @Override
709 public Set<String> getInputStreamCompressorNames() {
710 return Sets.newHashSet(GZIP, BROTLI, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, Z, LZ4_BLOCK, LZ4_FRAMED, ZSTANDARD, DEFLATE64);
711 }
712
713 @Override
714 public Set<String> getOutputStreamCompressorNames() {
715 return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_FRAMED, LZ4_BLOCK, LZ4_FRAMED, ZSTANDARD);
716 }
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731 @Deprecated
732 public void setDecompressConcatenated(final boolean decompressConcatenated) {
733 if (this.decompressUntilEOF != null) {
734 throw new IllegalStateException("Cannot override the setting defined by the constructor");
735 }
736 this.decompressConcatenated = decompressConcatenated;
737 }
738
739 }