1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.commons.compress.compressors.lz4;
20
21 import static org.hamcrest.CoreMatchers.containsString;
22 import static org.hamcrest.MatcherAssert.assertThat;
23 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
24 import static org.junit.jupiter.api.Assertions.assertEquals;
25 import static org.junit.jupiter.api.Assertions.assertFalse;
26 import static org.junit.jupiter.api.Assertions.assertThrows;
27 import static org.junit.jupiter.api.Assertions.assertTrue;
28 import static org.junit.jupiter.api.Assertions.fail;
29
30 import java.io.BufferedInputStream;
31 import java.io.ByteArrayInputStream;
32 import java.io.File;
33 import java.io.IOException;
34 import java.io.InputStream;
35 import java.nio.file.Files;
36 import java.util.Arrays;
37
38 import org.apache.commons.compress.AbstractTest;
39 import org.apache.commons.compress.compressors.CompressorStreamFactory;
40 import org.apache.commons.io.IOUtils;
41 import org.junit.jupiter.api.Test;
42
43 public final class FramedLZ4CompressorInputStreamTest extends AbstractTest {
44
45 interface StreamWrapper {
46 InputStream wrap(InputStream in) throws Exception;
47 }
48
49 private static byte[] duplicate(final byte[] from) {
50 final byte[] to = Arrays.copyOf(from, 2 * from.length);
51 System.arraycopy(from, 0, to, from.length, from.length);
52 return to;
53 }
54
55 private void expectIOException(final String fileName) {
56 assertThrows(IOException.class, () -> {
57 try (InputStream is = Files.newInputStream(getFile(fileName).toPath());
58 FramedLZ4CompressorInputStream in = new FramedLZ4CompressorInputStream(is)) {
59 IOUtils.toByteArray(in);
60 }
61 });
62 }
63
64 private void readDoubledBlaLz4(final StreamWrapper wrapper, final boolean expectDuplicateOutput) throws Exception {
65 byte[] singleInput;
66 try (InputStream i = newInputStream("bla.tar.lz4")) {
67 singleInput = IOUtils.toByteArray(i);
68 }
69 final byte[] input = duplicate(singleInput);
70 try (InputStream a = wrapper.wrap(new ByteArrayInputStream(input));
71 InputStream e = newInputStream("bla.tar")) {
72 final byte[] expected = IOUtils.toByteArray(e);
73 final byte[] actual = IOUtils.toByteArray(a);
74 assertArrayEquals(expectDuplicateOutput ? duplicate(expected) : expected, actual);
75 }
76 }
77
78 @Test
79 public void testBackreferenceAtStartCausesIOException() {
80 expectIOException("COMPRESS-490/ArrayIndexOutOfBoundsException1.lz4");
81 }
82
83 @Test
84 public void testBackreferenceOfSize0CausesIOException() {
85 expectIOException("COMPRESS-490/ArithmeticException.lz4");
86 }
87
88 @Test
89 public void testBackreferenceWithOffsetTooBigCausesIOException() {
90 expectIOException("COMPRESS-490/ArrayIndexOutOfBoundsException2.lz4");
91 }
92
93 @Test
94 public void testMatches() throws IOException {
95 assertFalse(FramedLZ4CompressorInputStream.matches(new byte[10], 4));
96 final byte[] expected = readAllBytes("bla.tar.lz4");
97 assertFalse(FramedLZ4CompressorInputStream.matches(expected, 3));
98 assertTrue(FramedLZ4CompressorInputStream.matches(expected, 4));
99 assertTrue(FramedLZ4CompressorInputStream.matches(expected, 5));
100 }
101
102 @Test
103 public void testMultiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException {
104 final File input = getFile("bla.tar.lz4");
105 final byte[] buf = new byte[2];
106 try (InputStream is = Files.newInputStream(input.toPath());
107 FramedLZ4CompressorInputStream in = new FramedLZ4CompressorInputStream(is)) {
108 IOUtils.toByteArray(in);
109 assertEquals(-1, in.read(buf));
110 assertEquals(-1, in.read(buf));
111 }
112 }
113
114 @Test
115 public void testReadBlaDumpLz4() throws IOException {
116 try (InputStream a = new FramedLZ4CompressorInputStream(newInputStream("bla.dump.lz4"));
117 InputStream e = newInputStream("bla.dump")) {
118 final byte[] expected = IOUtils.toByteArray(e);
119 final byte[] actual = IOUtils.toByteArray(a);
120 assertArrayEquals(expected, actual);
121 }
122 }
123
124 @Test
125 public void testReadBlaLz4() throws IOException {
126 try (InputStream a = new FramedLZ4CompressorInputStream(newInputStream("bla.tar.lz4"));
127 InputStream e = newInputStream("bla.tar")) {
128 final byte[] expected = IOUtils.toByteArray(e);
129 final byte[] actual = IOUtils.toByteArray(a);
130 assertArrayEquals(expected, actual);
131 }
132 }
133
134 @Test
135 public void testReadBlaLz4ViaFactory() throws Exception {
136 try (InputStream a = new CompressorStreamFactory().createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), newInputStream("bla.tar.lz4"));
137 InputStream e = newInputStream("bla.tar")) {
138 final byte[] expected = IOUtils.toByteArray(e);
139 final byte[] actual = IOUtils.toByteArray(a);
140 assertArrayEquals(expected, actual);
141 }
142 }
143
144 @Test
145 public void testReadBlaLz4ViaFactoryAutoDetection() throws Exception {
146 try (InputStream a = new CompressorStreamFactory().createCompressorInputStream(new BufferedInputStream(newInputStream("bla.tar.lz4")));
147 InputStream e = newInputStream("bla.tar")) {
148 final byte[] expected = IOUtils.toByteArray(e);
149 final byte[] actual = IOUtils.toByteArray(a);
150 assertArrayEquals(expected, actual);
151 }
152 }
153
154 @Test
155 public void testReadBlaLz4ViaFactoryWithDecompressConcatenated() throws Exception {
156 try (InputStream a = new CompressorStreamFactory().createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), newInputStream("bla.tar.lz4"),
157 true);
158 InputStream e = newInputStream("bla.tar")) {
159 final byte[] expected = IOUtils.toByteArray(e);
160 final byte[] actual = IOUtils.toByteArray(a);
161 assertArrayEquals(expected, actual);
162 }
163 }
164
165 @Test
166 public void testReadBlaLz4WithDecompressConcatenated() throws IOException {
167 try (InputStream a = new FramedLZ4CompressorInputStream(newInputStream("bla.tar.lz4"), true);
168 InputStream e = newInputStream("bla.tar")) {
169 final byte[] expected = IOUtils.toByteArray(e);
170 final byte[] actual = IOUtils.toByteArray(a);
171 assertArrayEquals(expected, actual);
172 }
173 }
174
175 @Test
176 public void testReadDoubledBlaLz4ViaFactoryWithDecompressConcatenatedFalse() throws Exception {
177 readDoubledBlaLz4(in -> new CompressorStreamFactory().createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, false), false);
178 }
179
180 @Test
181 public void testReadDoubledBlaLz4ViaFactoryWithDecompressConcatenatedTrue() throws Exception {
182 readDoubledBlaLz4(in -> new CompressorStreamFactory().createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, true), true);
183 }
184
185 @Test
186 public void testReadDoubledBlaLz4ViaFactoryWithoutExplicitDecompressConcatenated() throws Exception {
187 readDoubledBlaLz4(in -> new CompressorStreamFactory().createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in), false);
188 }
189
190 @Test
191 public void testReadDoubledBlaLz4WithDecompressConcatenatedFalse() throws Exception {
192 readDoubledBlaLz4(in -> new FramedLZ4CompressorInputStream(in, false), false);
193 }
194
195 @Test
196 public void testReadDoubledBlaLz4WithDecompressConcatenatedTrue() throws Exception {
197 readDoubledBlaLz4(in -> new FramedLZ4CompressorInputStream(in, true), true);
198 }
199
200 @Test
201 public void testReadDoubledBlaLz4WithoutExplicitDecompressConcatenated() throws Exception {
202 readDoubledBlaLz4(FramedLZ4CompressorInputStream::new, false);
203 }
204
205 @Test
206 public void testReadsUncompressedBlocks() throws IOException {
207 final byte[] input = { 4, 0x22, 0x4d, 0x18,
208 0x60,
209 0x70,
210 115,
211 13, 0, 0, (byte) 0x80,
212 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
213 0, 0, 0, 0,
214 };
215 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
216 final byte[] actual = IOUtils.toByteArray(a);
217 assertArrayEquals(new byte[] { 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!' }, actual);
218 }
219 }
220
221 @Test
222 public void testReadsUncompressedBlocksUsingSingleByteRead() throws IOException {
223 final byte[] input = { 4, 0x22, 0x4d, 0x18,
224 0x60,
225 0x70,
226 115,
227 13, 0, 0, (byte) 0x80,
228 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
229 0, 0, 0, 0,
230 };
231 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
232 final int h = a.read();
233 assertEquals('H', h);
234 }
235 }
236
237 @Test
238 public void testRejectsBlocksWithoutChecksum() {
239 final byte[] input = { 4, 0x22, 0x4d, 0x18,
240 0x70,
241 0x70,
242 114,
243 13, 0, 0, (byte) 0x80,
244 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
245 };
246 final IOException ex = assertThrows(IOException.class, () -> {
247 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
248 IOUtils.toByteArray(a);
249 }
250 }, "expected exception");
251 assertThat(ex.getMessage(), containsString("block checksum"));
252 }
253
254 @Test
255 public void testRejectsFileWithBadHeaderChecksum() {
256 final byte[] input = { 4, 0x22, 0x4d, 0x18,
257 0x64,
258 0x70,
259 0, };
260 final IOException ex = assertThrows(IOException.class, () -> {
261 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
262 }
263 }, "expected exception");
264 assertThat(ex.getMessage(), containsString("header checksum mismatch"));
265 }
266
267 @Test
268 public void testRejectsFileWithInsufficientContentSize() {
269 final byte[] input = { 4, 0x22, 0x4d, 0x18,
270 0x6C,
271 0x70,
272 };
273 final IOException ex = assertThrows(IOException.class, () -> {
274 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
275 }
276 }, "expected exception");
277 assertThat(ex.getMessage(), containsString("content size"));
278 }
279
280 @Test
281 public void testRejectsFileWithoutBlockSizeByte() {
282 final byte[] input = { 4, 0x22, 0x4d, 0x18,
283 0x64,
284 };
285 final IOException ex = assertThrows(IOException.class, () -> {
286 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
287 }
288 }, "expected exception");
289 assertThat(ex.getMessage(), containsString("BD byte"));
290 }
291
292 @Test
293 public void testRejectsFileWithoutFrameDescriptor() {
294 final byte[] input = { 4, 0x22, 0x4d, 0x18
295 };
296 final IOException ex = assertThrows(IOException.class, () -> {
297 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
298 }
299 }, "expected exception");
300 assertThat(ex.getMessage(), containsString("frame flags"));
301 }
302
303 @Test
304 public void testRejectsFileWithoutHeaderChecksum() {
305 final byte[] input = { 4, 0x22, 0x4d, 0x18,
306 0x64,
307 0x70,
308 };
309 final IOException ex = assertThrows(IOException.class, () -> {
310 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
311 }
312 }, "expected exception");
313 assertThat(ex.getMessage(), containsString("header checksum"));
314 }
315
316 @Test
317 public void testRejectsFileWithWrongVersion() {
318 final byte[] input = { 4, 0x22, 0x4d, 0x18,
319 0x24,
320 };
321 final IOException ex = assertThrows(IOException.class, () -> {
322 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
323 }
324 }, "expected exception");
325 assertThat(ex.getMessage(), containsString("version"));
326 }
327
328 @Test
329 public void testRejectsNonLZ4Stream() {
330 assertThrows(IOException.class, () -> new FramedLZ4CompressorInputStream(newInputStream("bla.tar")));
331 }
332
333 @Test
334 public void testRejectsSkippableFrameFollowedByJunk() {
335 final byte[] input = { 4, 0x22, 0x4d, 0x18,
336 0x60,
337 0x70,
338 115,
339 13, 0, 0, (byte) 0x80,
340 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
341 0, 0, 0, 0,
342 0x50, 0x2a, 0x4d, 0x18,
343 2, 0, 0, 0,
344 1, 2,
345 1, 0x22, 0x4d, 0x18,
346 };
347 final IOException ex = assertThrows(IOException.class, () -> {
348 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
349 IOUtils.toByteArray(a);
350 }
351 }, "expected exception");
352 assertThat(ex.getMessage(), containsString("garbage"));
353 }
354
355 @Test
356 public void testRejectsSkippableFrameFollowedByTooFewBytes() {
357 final byte[] input = { 4, 0x22, 0x4d, 0x18,
358 0x60,
359 0x70,
360 115,
361 13, 0, 0, (byte) 0x80,
362 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
363 0, 0, 0, 0,
364 0x52, 0x2a, 0x4d, 0x18,
365 2, 0, 0, 0,
366 1, 2,
367 4,
368 };
369 final IOException ex = assertThrows(IOException.class, () -> {
370 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
371 IOUtils.toByteArray(a);
372 }
373 }, "expected exception");
374 assertThat(ex.getMessage(), containsString("garbage"));
375 }
376
377 @Test
378 public void testRejectsSkippableFrameWithBadSignaturePrefix() {
379 final byte[] input = { 4, 0x22, 0x4d, 0x18,
380 0x60,
381 0x70,
382 115,
383 13, 0, 0, (byte) 0x80,
384 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
385 0, 0, 0, 0,
386 0x60, 0x2a, 0x4d, 0x18,
387 };
388 final IOException ex = assertThrows(IOException.class, () -> {
389 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
390 IOUtils.toByteArray(a);
391 fail();
392 }
393 }, "expected exception");
394 assertThat(ex.getMessage(), containsString("garbage"));
395 }
396
397 @Test
398 public void testRejectsSkippableFrameWithBadSignatureTrailer() {
399 final byte[] input = { 4, 0x22, 0x4d, 0x18,
400 0x60,
401 0x70,
402 115,
403 13, 0, 0, (byte) 0x80,
404 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
405 0, 0, 0, 0,
406 0x51, 0x2a, 0x4d, 0x17,
407 };
408 final IOException ex = assertThrows(IOException.class, () -> {
409 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
410 IOUtils.toByteArray(a);
411 }
412 }, "expected exception");
413 assertThat(ex.getMessage(), containsString("garbage"));
414 }
415
416 @Test
417 public void testRejectsSkippableFrameWithPrematureEnd() {
418 final byte[] input = { 4, 0x22, 0x4d, 0x18,
419 0x60,
420 0x70,
421 115,
422 13, 0, 0, (byte) 0x80,
423 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
424 0, 0, 0, 0,
425 0x50, 0x2a, 0x4d, 0x18,
426 2, 0, 0, 0,
427 1,
428 };
429 final IOException ex = assertThrows(IOException.class, () -> {
430 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
431 IOUtils.toByteArray(a);
432 }
433 }, "expected exception");
434 assertThat(ex.getMessage(), containsString("Premature end of stream while skipping frame"));
435 }
436
437 @Test
438 public void testRejectsSkippableFrameWithPrematureEndInLengthBytes() {
439 final byte[] input = { 4, 0x22, 0x4d, 0x18,
440 0x60,
441 0x70,
442 115,
443 13, 0, 0, (byte) 0x80,
444 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
445 0, 0, 0, 0,
446 0x55, 0x2a, 0x4d, 0x18,
447 2, 0, 0,
448 };
449 final IOException ex = assertThrows(IOException.class, () -> {
450 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
451 IOUtils.toByteArray(a);
452 }
453 }, "expected exception");
454 assertThat(ex.getMessage(), containsString("Premature end of data"));
455 }
456
457 @Test
458 public void testRejectsStreamsWithBadContentChecksum() {
459 final byte[] input = { 4, 0x22, 0x4d, 0x18,
460 0x64,
461 0x70,
462 (byte) 185,
463 13, 0, 0, (byte) 0x80,
464 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
465 0, 0, 0, 0,
466 1, 2, 3, 4, };
467 final IOException ex = assertThrows(IOException.class, () -> {
468 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
469 IOUtils.toByteArray(a);
470 }
471 }, "expected exception");
472 assertThat(ex.getMessage(), containsString("content checksum mismatch"));
473 }
474
475 @Test
476 public void testRejectsStreamsWithoutContentChecksum() {
477 final byte[] input = { 4, 0x22, 0x4d, 0x18,
478 0x64,
479 0x70,
480 (byte) 185,
481 13, 0, 0, (byte) 0x80,
482 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
483 0, 0, 0, 0,
484 };
485 final IOException ex = assertThrows(IOException.class, () -> {
486 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
487 IOUtils.toByteArray(a);
488 }
489 }, "expected exception");
490 assertThat(ex.getMessage(), containsString("content checksum"));
491 }
492
493 @Test
494 public void testRejectsTrailingBytesAfterValidFrame() {
495 final byte[] input = { 4, 0x22, 0x4d, 0x18,
496 0x60,
497 0x70,
498 115,
499 13, 0, 0, (byte) 0x80,
500 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
501 0, 0, 0, 0,
502 0x56, 0x2a, 0x4d,
503 };
504 final IOException ex = assertThrows(IOException.class, () -> {
505 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
506 IOUtils.toByteArray(a);
507 }
508 }, "expected exception");
509 assertThat(ex.getMessage(), containsString("garbage"));
510 }
511
512 @Test
513 public void testSingleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException {
514 final File input = getFile("bla.tar.lz4");
515 try (InputStream is = Files.newInputStream(input.toPath());
516 FramedLZ4CompressorInputStream in = new FramedLZ4CompressorInputStream(is);) {
517 IOUtils.toByteArray(in);
518 assertEquals(-1, in.read());
519 assertEquals(-1, in.read());
520 }
521 }
522
523 @Test
524 public void testSkipsOverSkippableFrames() throws IOException {
525 final byte[] input = { 4, 0x22, 0x4d, 0x18,
526 0x60,
527 0x70,
528 115,
529 13, 0, 0, (byte) 0x80,
530 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
531 0, 0, 0, 0,
532 0x5f, 0x2a, 0x4d, 0x18,
533 2, 0, 0, 0,
534 1, 2,
535 4, 0x22, 0x4d, 0x18,
536 0x60,
537 0x70,
538 115,
539 1, 0, 0, (byte) 0x80,
540 '!',
541 0, 0, 0, 0,
542 };
543 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
544 final byte[] actual = IOUtils.toByteArray(a);
545 assertArrayEquals(new byte[] { 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', '!' }, actual);
546 }
547 }
548
549 @Test
550 public void testSkipsOverTrailingSkippableFrames() throws IOException {
551 final byte[] input = { 4, 0x22, 0x4d, 0x18,
552 0x60,
553 0x70,
554 115,
555 13, 0, 0, (byte) 0x80,
556 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!',
557 0, 0, 0, 0,
558 0x51, 0x2a, 0x4d, 0x18,
559 2, 0, 0, 0,
560 1, 2,
561 };
562 try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
563 final byte[] actual = IOUtils.toByteArray(a);
564 assertArrayEquals(new byte[] { 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!' }, actual);
565 }
566 }
567
568 }