View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.commons.compress.compressors;
20  
21  import static org.apache.commons.compress.AbstractTest.getFile;
22  import static org.junit.jupiter.api.Assertions.assertEquals;
23  import static org.junit.jupiter.api.Assertions.assertFalse;
24  import static org.junit.jupiter.api.Assertions.assertNotNull;
25  import static org.junit.jupiter.api.Assertions.assertThrows;
26  import static org.junit.jupiter.api.Assertions.assertTrue;
27  
28  import java.io.BufferedInputStream;
29  import java.io.ByteArrayInputStream;
30  import java.io.IOException;
31  import java.io.InputStream;
32  import java.nio.file.Files;
33  import java.util.Collections;
34  import java.util.HashSet;
35  import java.util.Set;
36  import java.util.stream.Stream;
37  
38  import org.apache.commons.compress.MemoryLimitException;
39  import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
40  import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
41  import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
42  import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
43  import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
44  import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
45  import org.apache.commons.compress.utils.ByteUtils;
46  import org.apache.commons.io.input.BrokenInputStream;
47  import org.junit.jupiter.api.Test;
48  import org.junit.jupiter.params.ParameterizedTest;
49  import org.junit.jupiter.params.provider.Arguments;
50  import org.junit.jupiter.params.provider.MethodSource;
51  
52  @SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated
53  public final class DetectCompressorTest {
54  
55      static class TestData {
56  
57          final String fileName; // The multiple file name
58          final char[] entryNames; // expected entries ...
59          final CompressorStreamFactory factory; // ... when using this factory
60          final boolean concat; // expected value for decompressConcatenated
61  
62          TestData(final String name, final char[] names, final CompressorStreamFactory factory, final boolean concat) {
63              this.fileName = name;
64              this.entryNames = names;
65              this.factory = factory;
66              this.concat = concat;
67          }
68      }
69  
70      private static final CompressorStreamFactory factoryTrue = new CompressorStreamFactory(true);
71      private static final CompressorStreamFactory factoryFalse = new CompressorStreamFactory(false);
72  
73      // Must be static to allow use in the TestData entries
74      private static final CompressorStreamFactory factorySetTrue;
75      private static final CompressorStreamFactory factorySetFalse;
76  
77      static {
78          factorySetTrue = new CompressorStreamFactory();
79          factorySetTrue.setDecompressConcatenated(true);
80          factorySetFalse = new CompressorStreamFactory();
81          factorySetFalse.setDecompressConcatenated(false);
82      }
83  
84      public static Stream<Arguments> getDetectLimitedByNameParams() {
85          // @formatter:off
86          return Stream.of(
87              Arguments.of("bla.txt.bz2", CompressorStreamFactory.BZIP2),
88              Arguments.of("bla.tgz", CompressorStreamFactory.GZIP),
89              Arguments.of("bla.pack", CompressorStreamFactory.PACK200),
90              Arguments.of("bla.tar.xz", CompressorStreamFactory.XZ),
91              Arguments.of("bla.tar.deflatez", CompressorStreamFactory.DEFLATE),
92              Arguments.of("bla.tar.lz4", CompressorStreamFactory.LZ4_FRAMED),
93              Arguments.of("bla.tar.lzma", CompressorStreamFactory.LZMA),
94              Arguments.of("bla.tar.sz", CompressorStreamFactory.SNAPPY_FRAMED),
95              Arguments.of("bla.tar.Z", CompressorStreamFactory.Z),
96              Arguments.of("bla.tar.zst", CompressorStreamFactory.ZSTANDARD)
97          );
98          // @formatter:on
99      }
100 
101     final CompressorStreamFactory factory = new CompressorStreamFactory();
102 
103     private final TestData[] tests = {
104         // @formatter:off
105         new TestData("multiple.bz2", new char[] { 'a', 'b' }, factoryTrue, true),
106         new TestData("multiple.bz2", new char[] { 'a', 'b' }, factorySetTrue, true),
107         new TestData("multiple.bz2", new char[] { 'a' }, factoryFalse, false),
108         new TestData("multiple.bz2", new char[] { 'a' }, factorySetFalse, false),
109         new TestData("multiple.bz2", new char[] { 'a' }, factory, false),
110 
111         new TestData("multiple.gz", new char[] { 'a', 'b' }, factoryTrue, true),
112         new TestData("multiple.gz", new char[] { 'a', 'b' }, factorySetTrue, true),
113         new TestData("multiple.gz", new char[] { 'a' }, factoryFalse, false),
114         new TestData("multiple.gz", new char[] { 'a' }, factorySetFalse, false),
115         new TestData("multiple.gz", new char[] { 'a' }, factory, false),
116 
117         new TestData("multiple.xz", new char[] { 'a', 'b' }, factoryTrue, true),
118         new TestData("multiple.xz", new char[] { 'a', 'b' }, factorySetTrue, true),
119         new TestData("multiple.xz", new char[] { 'a' }, factoryFalse, false),
120         new TestData("multiple.xz", new char[] { 'a' }, factorySetFalse, false),
121         new TestData("multiple.xz", new char[] { 'a' }, factory, false),
122         // @formatter:on
123     };
124 
125     @SuppressWarnings("resource") // Caller closes.
126     private CompressorInputStream createCompressorInputStream(final String resource) throws CompressorException, IOException {
127         return factory.createCompressorInputStream(new BufferedInputStream(Files.newInputStream(getFile(resource).toPath())));
128     }
129 
130     @SuppressWarnings("resource") // Caller closes.
131     private CompressorInputStream createCompressorInputStream(final String resource, final CompressorStreamFactory factory)
132             throws CompressorException, IOException {
133         return factory.createCompressorInputStream(new BufferedInputStream(Files.newInputStream(getFile(resource).toPath())));
134     }
135 
136     @SuppressWarnings("resource") // Caller closes.
137     private CompressorInputStream createCompressorInputStream(final String resource, final Set<String> compressorNames)
138             throws CompressorException, IOException {
139         return factory.createCompressorInputStream(new BufferedInputStream(Files.newInputStream(getFile(resource).toPath())), compressorNames);
140     }
141 
142     private InputStream createInputStream(final String fileName, final int memoryLimitInKb) throws Exception {
143         final CompressorStreamFactory fac = new CompressorStreamFactory(true, memoryLimitInKb);
144         final InputStream is = new BufferedInputStream(Files.newInputStream(getFile(fileName).toPath()));
145         try {
146             return fac.createCompressorInputStream(is);
147         } catch (final CompressorException e) {
148             if (e.getCause() != null && e.getCause() instanceof Exception) {
149                 // unwrap cause to reveal MemoryLimitException
150                 throw (Exception) e.getCause();
151             }
152             throw e;
153         }
154 
155     }
156 
157     private String detect(final String testFileName) throws IOException, CompressorException {
158         return detect(testFileName, null);
159     }
160 
161     private String detect(final String testFileName, final Set<String> compressorNames) throws IOException, CompressorException {
162         try (InputStream is = new BufferedInputStream(Files.newInputStream(getFile(testFileName).toPath()))) {
163             return compressorNames != null ? CompressorStreamFactory.detect(is, compressorNames) : CompressorStreamFactory.detect(is);
164         }
165     }
166 
167     @Test
168     public void testCreateLimitedByName() throws Exception {
169         try (CompressorInputStream bzip2 = createCompressorInputStream("bla.txt.bz2", Collections.singleton(CompressorStreamFactory.BZIP2))) {
170             assertNotNull(bzip2);
171             assertTrue(bzip2 instanceof BZip2CompressorInputStream);
172         }
173 
174         try (CompressorInputStream gzip = createCompressorInputStream("bla.tgz", Collections.singleton(CompressorStreamFactory.GZIP))) {
175             assertNotNull(gzip);
176             assertTrue(gzip instanceof GzipCompressorInputStream);
177         }
178 
179         try (CompressorInputStream pack200 = createCompressorInputStream("bla.pack", Collections.singleton(CompressorStreamFactory.PACK200))) {
180             assertNotNull(pack200);
181             assertTrue(pack200 instanceof Pack200CompressorInputStream);
182         }
183 
184         try (CompressorInputStream xz = createCompressorInputStream("bla.tar.xz", Collections.singleton(CompressorStreamFactory.XZ))) {
185             assertNotNull(xz);
186             assertTrue(xz instanceof XZCompressorInputStream);
187         }
188 
189         try (CompressorInputStream zlib = createCompressorInputStream("bla.tar.deflatez", Collections.singleton(CompressorStreamFactory.DEFLATE))) {
190             assertNotNull(zlib);
191             assertTrue(zlib instanceof DeflateCompressorInputStream);
192         }
193 
194         try (CompressorInputStream zstd = createCompressorInputStream("bla.tar.zst", Collections.singleton(CompressorStreamFactory.ZSTANDARD))) {
195             assertNotNull(zstd);
196             assertTrue(zstd instanceof ZstdCompressorInputStream);
197         }
198     }
199 
200     @Test
201     public void testCreateLimitedByNameNotFound() throws Exception {
202         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.txt.bz2", Collections.singleton(CompressorStreamFactory.BROTLI)));
203         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tgz", Collections.singleton(CompressorStreamFactory.Z)));
204         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.pack", Collections.singleton(CompressorStreamFactory.SNAPPY_FRAMED)));
205         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tar.xz", Collections.singleton(CompressorStreamFactory.GZIP)));
206         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tar.deflatez", Collections.singleton(CompressorStreamFactory.PACK200)));
207         assertThrows(CompressorException.class, () -> createCompressorInputStream("bla.tar.zst", Collections.singleton(CompressorStreamFactory.LZ4_FRAMED)));
208     }
209 
210     @Test
211     public void testCreateWithAutoDetection() throws Exception {
212         try (CompressorInputStream bzip2 = createCompressorInputStream("bla.txt.bz2")) {
213             assertNotNull(bzip2);
214             assertTrue(bzip2 instanceof BZip2CompressorInputStream);
215         }
216 
217         try (CompressorInputStream gzip = createCompressorInputStream("bla.tgz")) {
218             assertNotNull(gzip);
219             assertTrue(gzip instanceof GzipCompressorInputStream);
220         }
221 
222         try (CompressorInputStream pack200 = createCompressorInputStream("bla.pack")) {
223             assertNotNull(pack200);
224             assertTrue(pack200 instanceof Pack200CompressorInputStream);
225         }
226 
227         try (CompressorInputStream xz = createCompressorInputStream("bla.tar.xz")) {
228             assertNotNull(xz);
229             assertTrue(xz instanceof XZCompressorInputStream);
230         }
231 
232         try (CompressorInputStream zlib = createCompressorInputStream("bla.tar.deflatez")) {
233             assertNotNull(zlib);
234             assertTrue(zlib instanceof DeflateCompressorInputStream);
235         }
236 
237         try (CompressorInputStream zstd = createCompressorInputStream("bla.tar.zst")) {
238             assertNotNull(zstd);
239             assertTrue(zstd instanceof ZstdCompressorInputStream);
240         }
241 
242         assertThrows(CompressorException.class, () -> factory.createCompressorInputStream(new ByteArrayInputStream(ByteUtils.EMPTY_BYTE_ARRAY)));
243     }
244 
245     @Test
246     public void testDetect() throws Exception {
247         assertEquals(CompressorStreamFactory.BZIP2, detect("bla.txt.bz2"));
248         assertEquals(CompressorStreamFactory.GZIP, detect("bla.tgz"));
249         assertEquals(CompressorStreamFactory.PACK200, detect("bla.pack"));
250         assertEquals(CompressorStreamFactory.XZ, detect("bla.tar.xz"));
251         assertEquals(CompressorStreamFactory.DEFLATE, detect("bla.tar.deflatez"));
252         assertEquals(CompressorStreamFactory.LZ4_FRAMED, detect("bla.tar.lz4"));
253         assertEquals(CompressorStreamFactory.LZMA, detect("bla.tar.lzma"));
254         assertEquals(CompressorStreamFactory.SNAPPY_FRAMED, detect("bla.tar.sz"));
255         assertEquals(CompressorStreamFactory.Z, detect("bla.tar.Z"));
256         assertEquals(CompressorStreamFactory.ZSTANDARD, detect("bla.tar.zst"));
257 
258         // make sure we don't oom on detect
259         assertEquals(CompressorStreamFactory.Z, detect("COMPRESS-386"));
260         assertEquals(CompressorStreamFactory.LZMA, detect("COMPRESS-382"));
261 
262         assertThrows(CompressorException.class,
263                 () -> CompressorStreamFactory.detect(new BufferedInputStream(new ByteArrayInputStream(ByteUtils.EMPTY_BYTE_ARRAY))));
264 
265         final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> CompressorStreamFactory.detect(null),
266                 "shouldn't be able to detect null stream");
267         assertEquals("Stream must not be null.", e.getMessage());
268 
269         final CompressorException ce = assertThrows(CompressorException.class,
270                 () -> CompressorStreamFactory.detect(new BufferedInputStream(new BrokenInputStream())), "Expected IOException");
271         assertEquals("IOException while reading signature.", ce.getMessage());
272     }
273 
274     @ParameterizedTest
275     @MethodSource("getDetectLimitedByNameParams")
276     public void testDetectLimitedByName(final String filename, final String compressorName) throws Exception {
277         assertEquals(compressorName, detect(filename, Collections.singleton(compressorName)));
278     }
279 
280     @Test
281     public void testDetectLimitedByNameNotFound() throws Exception {
282         final Set<String> compressorNames = Collections.singleton(CompressorStreamFactory.DEFLATE);
283 
284         assertThrows(CompressorException.class, () -> detect("bla.txt.bz2", compressorNames));
285         assertThrows(CompressorException.class, () -> detect("bla.tgz", compressorNames));
286         assertThrows(CompressorException.class, () -> detect("bla.pack", compressorNames));
287         assertThrows(CompressorException.class, () -> detect("bla.tar.xz", compressorNames));
288         assertThrows(CompressorException.class, () -> detect("bla.tar.deflatez", Collections.singleton(CompressorStreamFactory.BZIP2)));
289         assertThrows(CompressorException.class, () -> detect("bla.tar.lz4", compressorNames));
290         assertThrows(CompressorException.class, () -> detect("bla.tar.lzma", compressorNames));
291         assertThrows(CompressorException.class, () -> detect("bla.tar.sz", compressorNames));
292         assertThrows(CompressorException.class, () -> detect("bla.tar.Z", compressorNames));
293         assertThrows(CompressorException.class, () -> detect("bla.tar.zst", compressorNames));
294     }
295 
296     @Test
297     public void testDetectNullOrEmptyCompressorNames() throws Exception {
298         assertThrows(IllegalArgumentException.class, () -> CompressorStreamFactory.detect(createCompressorInputStream("bla.txt.bz2"), (Set<String>) null));
299         assertThrows(IllegalArgumentException.class, () -> CompressorStreamFactory.detect(createCompressorInputStream("bla.tgz"), new HashSet<>()));
300     }
301 
302     @Test
303     public void testLZMAMemoryLimit() throws Exception {
304         assertThrows(MemoryLimitException.class, () -> createInputStream("COMPRESS-382", 100));
305     }
306 
307     @Test
308     public void testMultiples() throws Exception {
309         for (int i = 0; i < tests.length; i++) {
310             final TestData test = tests[i];
311             final CompressorStreamFactory fac = test.factory;
312             assertNotNull(fac, "Test entry " + i);
313             assertEquals(test.concat, fac.getDecompressConcatenated(), "Test entry " + i);
314             try (CompressorInputStream in = createCompressorInputStream(test.fileName, fac)) {
315                 assertNotNull(in, "Test entry " + i);
316                 for (final char entry : test.entryNames) {
317                     assertEquals(entry, in.read(), "Test entry" + i);
318                 }
319                 assertEquals(0, in.available());
320                 assertEquals(-1, in.read());
321             }
322         }
323     }
324 
325     @Test
326     public void testOverride() {
327         final CompressorStreamFactory fac1 = new CompressorStreamFactory();
328         assertFalse(fac1.getDecompressConcatenated());
329         fac1.setDecompressConcatenated(true);
330         assertTrue(fac1.getDecompressConcatenated());
331 
332         final CompressorStreamFactory fac2 = new CompressorStreamFactory(false);
333         assertFalse(fac2.getDecompressConcatenated());
334         assertThrows(IllegalStateException.class, () -> fac2.setDecompressConcatenated(true), "Expected IllegalStateException");
335 
336         final CompressorStreamFactory fac3 = new CompressorStreamFactory(true);
337         assertTrue(fac3.getDecompressConcatenated());
338         assertThrows(IllegalStateException.class, () -> fac3.setDecompressConcatenated(true), "Expected IllegalStateException");
339     }
340 
341     @Test
342     public void testXZMemoryLimitOnRead() throws Exception {
343         // Even though the file is very small, the memory limit
344         // has to be quite large (8296 KiB) because of the dictionary size
345 
346         // This is triggered on read(); not during initialization.
347         // This test is here instead of the xz unit test to make sure
348         // that the parameter is properly passed via the CompressorStreamFactory
349         try (InputStream compressorIs = createInputStream("bla.tar.xz", 100)) {
350             assertThrows(MemoryLimitException.class, () -> compressorIs.read());
351         }
352     }
353 
354     @Test
355     public void testXZMemoryLimitOnSkip() throws Exception {
356         try (InputStream compressorIs = createInputStream("bla.tar.xz", 100)) {
357             assertThrows(MemoryLimitException.class, () -> compressorIs.skip(10));
358         }
359     }
360 
361     @Test
362     public void testZMemoryLimit() throws Exception {
363         assertThrows(MemoryLimitException.class, () -> createInputStream("COMPRESS-386", 100));
364     }
365 }