View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.commons.compress.compressors.snappy;
20  
21  import static org.junit.jupiter.api.Assertions.assertArrayEquals;
22  
23  import java.io.ByteArrayInputStream;
24  import java.io.ByteArrayOutputStream;
25  import java.io.File;
26  import java.io.IOException;
27  import java.io.OutputStream;
28  import java.nio.file.Files;
29  import java.nio.file.Path;
30  import java.util.Random;
31  
32  import org.apache.commons.compress.AbstractTest;
33  import org.apache.commons.compress.compressors.lz77support.Parameters;
34  import org.apache.commons.io.IOUtils;
35  import org.junit.jupiter.api.Test;
36  
37  public final class SnappyRoundtripTest extends AbstractTest {
38  
39      private static Parameters newParameters(final int windowSize, final int minBackReferenceLength, final int maxBackReferenceLength, final int maxOffset,
40              final int maxLiteralLength) {
41          return Parameters.builder(windowSize).withMinBackReferenceLength(minBackReferenceLength).withMaxBackReferenceLength(maxBackReferenceLength)
42                  .withMaxOffset(maxOffset).withMaxLiteralLength(maxLiteralLength).build();
43      }
44  
45      private void roundTripTest(final byte[] input, final Parameters params) throws IOException {
46          final ByteArrayOutputStream os = new ByteArrayOutputStream();
47          try (SnappyCompressorOutputStream sos = new SnappyCompressorOutputStream(os, input.length, params)) {
48              sos.write(input);
49          }
50          try (SnappyCompressorInputStream sis = new SnappyCompressorInputStream(new ByteArrayInputStream(os.toByteArray()), params.getWindowSize())) {
51              final byte[] actual = IOUtils.toByteArray(sis);
52              assertArrayEquals(input, actual);
53          }
54      }
55  
56      private void roundTripTest(final Path input, final Parameters params) throws IOException {
57          final File outputSz = newTempFile(input.getFileName() + ".raw.sz");
58          try (OutputStream os = Files.newOutputStream(outputSz.toPath());
59                  SnappyCompressorOutputStream sos = new SnappyCompressorOutputStream(os, Files.size(input), params)) {
60              Files.copy(input, sos);
61          }
62          try (SnappyCompressorInputStream sis = new SnappyCompressorInputStream(Files.newInputStream(outputSz.toPath()), params.getWindowSize())) {
63              final byte[] expected = Files.readAllBytes(input);
64              final byte[] actual = IOUtils.toByteArray(sis);
65              assertArrayEquals(expected, actual);
66          }
67      }
68  
69      private void roundTripTest(final String testFile) throws IOException {
70          roundTripTest(getPath(testFile), SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE).build());
71      }
72  
73      // yields no compression at all
74      @Test
75      public void testBiggerFileRoundtrip() throws IOException {
76          roundTripTest("COMPRESS-256.7z");
77      }
78  
79      // should yield decent compression
80      @Test
81      public void testBlaTarRoundtrip() throws IOException {
82          // System.err.println("Configuration: default");
83          roundTripTest("bla.tar");
84      }
85  
86      @Test
87      public void testBlaTarRoundtripTunedForCompressionRatio() throws IOException {
88          // System.err.println("Configuration: tuned for compression ratio");
89          roundTripTest(getPath("bla.tar"),
90                  SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE).tunedForCompressionRatio().build());
91      }
92  
93      @Test
94      public void testBlaTarRoundtripTunedForSpeed() throws IOException {
95          // System.err.println("Configuration: tuned for speed");
96          roundTripTest(getPath("bla.tar"),
97                  SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE).tunedForSpeed().build());
98      }
99  
100     // yields no compression at all
101     @Test
102     public void testGzippedLoremIpsumRoundtrip() throws IOException {
103         roundTripTest("lorem-ipsum.txt.gz");
104     }
105 
106     @Test
107     public void testTryReallyBigOffset() throws IOException {
108         // "normal" Snappy files will never reach offsets beyond
109         // 16bits (i.e. those using four bytes to encode the length)
110         // as the block size is only 32k. This means we never execute
111         // the code for four-byte length copies in either stream class
112         // using real-world Snappy files.
113         // This is an artificial stream using a bigger block size that
114         // may not even be expandable by other Snappy implementations.
115         // Start with the four byte sequence 0000 after that add > 64k
116         // of random noise that doesn't contain any 0000 at all, then
117         // add 0000.
118         final ByteArrayOutputStream fs = new ByteArrayOutputStream((1 << 16) + 1024);
119         fs.write(0);
120         fs.write(0);
121         fs.write(0);
122         fs.write(0);
123         final int cnt = 1 << 16 + 5;
124         final Random r = new Random();
125         for (int i = 0; i < cnt; i++) {
126             fs.write(r.nextInt(255) + 1);
127         }
128         fs.write(0);
129         fs.write(0);
130         fs.write(0);
131         fs.write(0);
132 
133         roundTripTest(fs.toByteArray(), newParameters(1 << 17, 4, 64, 1 << 17 - 1, 1 << 17 - 1));
134     }
135 
136     @Test
137     public void testTryReallyLongLiterals() throws IOException {
138         // "normal" Snappy files will never reach literal blocks with
139         // length beyond 16bits (i.e. those using three or four bytes
140         // to encode the length) as the block size is only 32k. This
141         // means we never execute the code for the three/four byte
142         // length literals in either stream class using real-world
143         // Snappy files.
144         // What we'd need would be a sequence of bytes with no four
145         // byte subsequence repeated that is longer than 64k, we try
146         // our best with random, but will probably only hit the three byte
147         // methods in a few lucky cases.
148         // The four byte methods would require even more luck and a
149         // buffer (and a file written to disk) that was 2^5 bigger
150         // than the buffer used here.
151         final Path path = newTempPath("reallyBigLiteralTest");
152         try (OutputStream outputStream = Files.newOutputStream(path)) {
153             final int cnt = 1 << 19;
154             final Random r = new Random();
155             for (int i = 0; i < cnt; i++) {
156                 outputStream.write(r.nextInt(256));
157             }
158         }
159         roundTripTest(path, newParameters(1 << 18, 4, 64, 1 << 16 - 1, 1 << 18 - 1));
160     }
161 }