View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   https://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.commons.compress.compressors;
20  
21  import static org.apache.commons.compress.AbstractTest.getFile;
22  import static org.junit.jupiter.api.Assertions.assertEquals;
23  import static org.junit.jupiter.api.Assertions.assertFalse;
24  import static org.junit.jupiter.api.Assertions.assertInstanceOf;
25  import static org.junit.jupiter.api.Assertions.assertNotNull;
26  import static org.junit.jupiter.api.Assertions.assertThrows;
27  import static org.junit.jupiter.api.Assertions.assertTrue;
28  
29  import java.io.BufferedInputStream;
30  import java.io.ByteArrayInputStream;
31  import java.io.IOException;
32  import java.io.InputStream;
33  import java.nio.file.Files;
34  import java.util.Collections;
35  import java.util.HashSet;
36  import java.util.Set;
37  import java.util.stream.Stream;
38  
39  import org.apache.commons.compress.MemoryLimitException;
40  import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
41  import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
42  import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
43  import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
44  import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
45  import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
46  import org.apache.commons.compress.utils.ByteUtils;
47  import org.apache.commons.io.input.BrokenInputStream;
48  import org.junit.jupiter.api.Test;
49  import org.junit.jupiter.params.ParameterizedTest;
50  import org.junit.jupiter.params.provider.Arguments;
51  import org.junit.jupiter.params.provider.MethodSource;
52  
53  @SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated
54  public final class DetectCompressorTest {
55  
56      static class TestData {
57  
58          final String fileName; // The multiple file name
59          final char[] entryNames; // expected entries ...
60          final CompressorStreamFactory factory; // ... when using this factory
61          final boolean concat; // expected value for decompressConcatenated
62  
63          TestData(final String name, final char[] names, final CompressorStreamFactory factory, final boolean concat) {
64              this.fileName = name;
65              this.entryNames = names;
66              this.factory = factory;
67              this.concat = concat;
68          }
69      }
70  
71      private static final CompressorStreamFactory factoryTrue = new CompressorStreamFactory(true);
72      private static final CompressorStreamFactory factoryFalse = new CompressorStreamFactory(false);
73  
74      // Must be static to allow use in the TestData entries
75      private static final CompressorStreamFactory factorySetTrue;
76      private static final CompressorStreamFactory factorySetFalse;
77  
78      static {
79          factorySetTrue = new CompressorStreamFactory();
80          factorySetTrue.setDecompressConcatenated(true);
81          factorySetFalse = new CompressorStreamFactory();
82          factorySetFalse.setDecompressConcatenated(false);
83      }
84  
85      public static Stream<Arguments> getDetectLimitedByNameParams() {
86          // @formatter:off
87          return Stream.of(
88              Arguments.of("bla.txt.bz2", CompressorStreamFactory.BZIP2),
89              Arguments.of("bla.tgz", CompressorStreamFactory.GZIP),
90              Arguments.of("bla.pack", CompressorStreamFactory.PACK200),
91              Arguments.of("bla.tar.xz", CompressorStreamFactory.XZ),
92              Arguments.of("bla.tar.deflatez", CompressorStreamFactory.DEFLATE),
93              Arguments.of("bla.tar.lz4", CompressorStreamFactory.LZ4_FRAMED),
94              Arguments.of("bla.tar.lzma", CompressorStreamFactory.LZMA),
95              Arguments.of("bla.tar.sz", CompressorStreamFactory.SNAPPY_FRAMED),
96              Arguments.of("bla.tar.Z", CompressorStreamFactory.Z),
97              Arguments.of("bla.tar.zst", CompressorStreamFactory.ZSTANDARD)
98          );
99          // @formatter:on
100     }
101 
102     final CompressorStreamFactory factory = new CompressorStreamFactory();
103 
104     private final TestData[] tests = {
105         // @formatter:off
106         new TestData("multiple.bz2", new char[] { 'a', 'b' }, factoryTrue, true),
107         new TestData("multiple.bz2", new char[] { 'a', 'b' }, factorySetTrue, true),
108         new TestData("multiple.bz2", new char[] { 'a' }, factoryFalse, false),
109         new TestData("multiple.bz2", new char[] { 'a' }, factorySetFalse, false),
110         new TestData("multiple.bz2", new char[] { 'a' }, factory, false),
111 
112         new TestData("multiple.gz", new char[] { 'a', 'b' }, factoryTrue, true),
113         new TestData("multiple.gz", new char[] { 'a', 'b' }, factorySetTrue, true),
114         new TestData("multiple.gz", new char[] { 'a' }, factoryFalse, false),
115         new TestData("multiple.gz", new char[] { 'a' }, factorySetFalse, false),
116         new TestData("multiple.gz", new char[] { 'a' }, factory, false),
117 
118         new TestData("multiple.xz", new char[] { 'a', 'b' }, factoryTrue, true),
119         new TestData("multiple.xz", new char[] { 'a', 'b' }, factorySetTrue, true),
120         new TestData("multiple.xz", new char[] { 'a' }, factoryFalse, false),
121         new TestData("multiple.xz", new char[] { 'a' }, factorySetFalse, false),
122         new TestData("multiple.xz", new char[] { 'a' }, factory, false),
123         // @formatter:on
124     };
125 
126     @SuppressWarnings("resource") // Caller closes.
127     private CompressorInputStream createCompressorInputStream(final String resource) throws CompressorException, IOException {
128         return factory.createCompressorInputStream(new BufferedInputStream(Files.newInputStream(getFile(resource).toPath())));
129     }
130 
131     @SuppressWarnings("resource") // Caller closes.
132     private CompressorInputStream createCompressorInputStream(final String resource, final CompressorStreamFactory factory)
133             throws CompressorException, IOException {
134         return factory.createCompressorInputStream(new BufferedInputStream(Files.newInputStream(getFile(resource).toPath())));
135     }
136 
137     @SuppressWarnings("resource") // Caller closes.
138     private CompressorInputStream createCompressorInputStream(final String resource, final Set<String> compressorNames)
139             throws CompressorException, IOException {
140         return factory.createCompressorInputStream(new BufferedInputStream(Files.newInputStream(getFile(resource).toPath())), compressorNames);
141     }
142 
143     private InputStream createInputStream(final String fileName, final int memoryLimitInKb) throws Exception {
144         final CompressorStreamFactory fac = new CompressorStreamFactory(true, memoryLimitInKb);
145         final InputStream is = new BufferedInputStream(Files.newInputStream(getFile(fileName).toPath()));
146         try {
147             return fac.createCompressorInputStream(is);
148         } catch (final CompressorException e) {
149             if (e.getCause() != null && e.getCause() instanceof Exception) {
150                 // unwrap cause to reveal MemoryLimitException
151                 throw (Exception) e.getCause();
152             }
153             throw e;
154         }
155 
156     }
157 
158     private String detect(final String testFileName) throws IOException, CompressorException {
159         return detect(testFileName, null);
160     }
161 
162     private String detect(final String testFileName, final Set<String> compressorNames) throws IOException, CompressorException {
163         try (InputStream is = new BufferedInputStream(Files.newInputStream(getFile(testFileName).toPath()))) {
164             return compressorNames != null ? CompressorStreamFactory.detect(is, compressorNames) : CompressorStreamFactory.detect(is);
165         }
166     }
167 
168     @Test
169     void testCreateLimitedByName() throws Exception {
170         try (CompressorInputStream bzip2 = createCompressorInputStream("bla.txt.bz2", Collections.singleton(CompressorStreamFactory.BZIP2))) {
171             assertNotNull(bzip2);
172             assertInstanceOf(BZip2CompressorInputStream.class, bzip2);
173         }
174 
175         try (CompressorInputStream gzip = createCompressorInputStream("bla.tgz", Collections.singleton(CompressorStreamFactory.GZIP))) {
176             assertNotNull(gzip);
177             assertInstanceOf(GzipCompressorInputStream.class, gzip);
178         }
179 
180         try (CompressorInputStream pack200 = createCompressorInputStream("bla.pack", Collections.singleton(CompressorStreamFactory.PACK200))) {
181             assertNotNull(pack200);
182             assertInstanceOf(Pack200CompressorInputStream.class, pack200);
183         }
184 
185         try (CompressorInputStream xz = createCompressorInputStream("bla.tar.xz", Collections.singleton(CompressorStreamFactory.XZ))) {
186             assertNotNull(xz);
187             assertInstanceOf(XZCompressorInputStream.class, xz);
188         }
189 
190         try (CompressorInputStream zlib = createCompressorInputStream("bla.tar.deflatez", Collections.singleton(CompressorStreamFactory.DEFLATE))) {
191             assertNotNull(zlib);
192             assertInstanceOf(DeflateCompressorInputStream.class, zlib);
193         }
194 
195         try (CompressorInputStream zstd = createCompressorInputStream("bla.tar.zst", Collections.singleton(CompressorStreamFactory.ZSTANDARD))) {
196             assertNotNull(zstd);
197             assertInstanceOf(ZstdCompressorInputStream.class, zstd);
198         }
199     }
200 
201     @Test
202     void testCreateLimitedByNameNotFound() throws Exception {
203         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.txt.bz2", Collections.singleton(CompressorStreamFactory.BROTLI)));
204         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tgz", Collections.singleton(CompressorStreamFactory.Z)));
205         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.pack", Collections.singleton(CompressorStreamFactory.SNAPPY_FRAMED)));
206         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tar.xz", Collections.singleton(CompressorStreamFactory.GZIP)));
207         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tar.deflatez", Collections.singleton(CompressorStreamFactory.PACK200)));
208         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tar.zst", Collections.singleton(CompressorStreamFactory.LZ4_FRAMED)));
209     }
210 
211     @Test
212     void testCreateWithAutoDetection() throws Exception {
213         try (CompressorInputStream bzip2 = createCompressorInputStream("bla.txt.bz2")) {
214             assertNotNull(bzip2);
215             assertInstanceOf(BZip2CompressorInputStream.class, bzip2);
216         }
217 
218         try (CompressorInputStream gzip = createCompressorInputStream("bla.tgz")) {
219             assertNotNull(gzip);
220             assertInstanceOf(GzipCompressorInputStream.class, gzip);
221         }
222 
223         try (CompressorInputStream pack200 = createCompressorInputStream("bla.pack")) {
224             assertNotNull(pack200);
225             assertInstanceOf(Pack200CompressorInputStream.class, pack200);
226         }
227 
228         try (CompressorInputStream xz = createCompressorInputStream("bla.tar.xz")) {
229             assertNotNull(xz);
230             assertInstanceOf(XZCompressorInputStream.class, xz);
231         }
232 
233         try (CompressorInputStream zlib = createCompressorInputStream("bla.tar.deflatez")) {
234             assertNotNull(zlib);
235             assertInstanceOf(DeflateCompressorInputStream.class, zlib);
236         }
237 
238         try (CompressorInputStream zstd = createCompressorInputStream("bla.tar.zst")) {
239             assertNotNull(zstd);
240             assertInstanceOf(ZstdCompressorInputStream.class, zstd);
241         }
242 
243         assertThrows(CompressorException.class, () -> factory.createCompressorInputStream(new ByteArrayInputStream(ByteUtils.EMPTY_BYTE_ARRAY)));
244     }
245 
246     @Test
247     void testDetect() throws Exception {
248         assertEquals(CompressorStreamFactory.BZIP2, detect("bla.txt.bz2"));
249         assertEquals(CompressorStreamFactory.GZIP, detect("bla.tgz"));
250         assertEquals(CompressorStreamFactory.PACK200, detect("bla.pack"));
251         assertEquals(CompressorStreamFactory.XZ, detect("bla.tar.xz"));
252         assertEquals(CompressorStreamFactory.DEFLATE, detect("bla.tar.deflatez"));
253         assertEquals(CompressorStreamFactory.LZ4_FRAMED, detect("bla.tar.lz4"));
254         assertEquals(CompressorStreamFactory.LZMA, detect("bla.tar.lzma"));
255         assertEquals(CompressorStreamFactory.SNAPPY_FRAMED, detect("bla.tar.sz"));
256         assertEquals(CompressorStreamFactory.Z, detect("bla.tar.Z"));
257         assertEquals(CompressorStreamFactory.ZSTANDARD, detect("bla.tar.zst"));
258 
259         // make sure we don't oom on detect
260         assertEquals(CompressorStreamFactory.Z, detect("COMPRESS-386"));
261         assertEquals(CompressorStreamFactory.LZMA, detect("COMPRESS-382"));
262 
263         assertThrows(CompressorException.class,
264                 () -> CompressorStreamFactory.detect(new BufferedInputStream(new ByteArrayInputStream(ByteUtils.EMPTY_BYTE_ARRAY))));
265 
266         final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> CompressorStreamFactory.detect(null),
267                 "shouldn't be able to detect null stream");
268         assertEquals("Stream must not be null.", e.getMessage());
269 
270         final CompressorException ce = assertThrows(CompressorException.class,
271                 () -> CompressorStreamFactory.detect(new BufferedInputStream(new BrokenInputStream())), "Expected IOException");
272         assertEquals("Failed to read signature.", ce.getMessage());
273     }
274 
275     @ParameterizedTest
276     @MethodSource("getDetectLimitedByNameParams")
277     void testDetectLimitedByName(final String filename, final String compressorName) throws Exception {
278         assertEquals(compressorName, detect(filename, Collections.singleton(compressorName)));
279     }
280 
281     @Test
282     void testDetectLimitedByNameNotFound() throws Exception {
283         final Set<String> compressorNames = Collections.singleton(CompressorStreamFactory.DEFLATE);
284 
285         assertThrows(CompressorException.class, () -> detect("bla.txt.bz2", compressorNames));
286         assertThrows(CompressorException.class, () -> detect("bla.tgz", compressorNames));
287         assertThrows(CompressorException.class, () -> detect("bla.pack", compressorNames));
288         assertThrows(CompressorException.class, () -> detect("bla.tar.xz", compressorNames));
289         assertThrows(CompressorException.class, () -> detect("bla.tar.deflatez", Collections.singleton(CompressorStreamFactory.BZIP2)));
290         assertThrows(CompressorException.class, () -> detect("bla.tar.lz4", compressorNames));
291         assertThrows(CompressorException.class, () -> detect("bla.tar.lzma", compressorNames));
292         assertThrows(CompressorException.class, () -> detect("bla.tar.sz", compressorNames));
293         assertThrows(CompressorException.class, () -> detect("bla.tar.Z", compressorNames));
294         assertThrows(CompressorException.class, () -> detect("bla.tar.zst", compressorNames));
295     }
296 
297     @Test
298     void testDetectNullOrEmptyCompressorNames() throws Exception {
299         assertThrows(IllegalArgumentException.class, () -> CompressorStreamFactory.detect(createCompressorInputStream("bla.txt.bz2"), (Set<String>) null));
300         assertThrows(IllegalArgumentException.class, () -> CompressorStreamFactory.detect(createCompressorInputStream("bla.tgz"), new HashSet<>()));
301     }
302 
303     @Test
304     void testLZMAMemoryLimit() throws Exception {
305         assertThrows(MemoryLimitException.class, () -> createInputStream("COMPRESS-382", 100));
306     }
307 
308     @Test
309     void testMultiples() throws Exception {
310         for (int i = 0; i < tests.length; i++) {
311             final TestData test = tests[i];
312             final CompressorStreamFactory fac = test.factory;
313             assertNotNull(fac, "Test entry " + i);
314             assertEquals(test.concat, fac.getDecompressConcatenated(), "Test entry " + i);
315             try (CompressorInputStream in = createCompressorInputStream(test.fileName, fac)) {
316                 assertNotNull(in, "Test entry " + i);
317                 for (final char entry : test.entryNames) {
318                     assertEquals(entry, in.read(), "Test entry" + i);
319                 }
320                 assertEquals(0, in.available());
321                 assertEquals(-1, in.read());
322             }
323         }
324     }
325 
326     @Test
327     void testOverride() {
328         final CompressorStreamFactory fac1 = new CompressorStreamFactory();
329         assertFalse(fac1.getDecompressConcatenated());
330         fac1.setDecompressConcatenated(true);
331         assertTrue(fac1.getDecompressConcatenated());
332 
333         final CompressorStreamFactory fac2 = new CompressorStreamFactory(false);
334         assertFalse(fac2.getDecompressConcatenated());
335         assertThrows(IllegalStateException.class, () -> fac2.setDecompressConcatenated(true), "Expected IllegalStateException");
336 
337         final CompressorStreamFactory fac3 = new CompressorStreamFactory(true);
338         assertTrue(fac3.getDecompressConcatenated());
339         assertThrows(IllegalStateException.class, () -> fac3.setDecompressConcatenated(true), "Expected IllegalStateException");
340     }
341 
342     @Test
343     void testXZMemoryLimitOnRead() throws Exception {
344         // Even though the file is very small, the memory limit
345         // has to be quite large (8296 KiB) because of the dictionary size
346 
347         // This is triggered on read(); not during initialization.
348         // This test is here instead of the xz unit test to make sure
349         // that the parameter is properly passed via the CompressorStreamFactory
350         try (InputStream compressorIs = createInputStream("bla.tar.xz", 100)) {
351             assertThrows(MemoryLimitException.class, () -> compressorIs.read());
352         }
353     }
354 
355     @Test
356     void testXZMemoryLimitOnSkip() throws Exception {
357         try (InputStream compressorIs = createInputStream("bla.tar.xz", 100)) {
358             assertThrows(MemoryLimitException.class, () -> compressorIs.skip(10));
359         }
360     }
361 
362     @Test
363     void testZMemoryLimit() throws Exception {
364         assertThrows(MemoryLimitException.class, () -> createInputStream("COMPRESS-386", 100));
365     }
366 }