001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * https://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, 013 * software distributed under the License is distributed on an 014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 * KIND, either express or implied. See the License for the 016 * specific language governing permissions and limitations 017 * under the License. 018 */ 019package org.apache.commons.compress.compressors.snappy; 020 021import java.io.IOException; 022import java.io.InputStream; 023import java.io.PushbackInputStream; 024import java.util.Arrays; 025 026import org.apache.commons.codec.digest.PureJavaCrc32C; 027import org.apache.commons.compress.compressors.CompressorInputStream; 028import org.apache.commons.compress.utils.ByteUtils; 029import org.apache.commons.compress.utils.IOUtils; 030import org.apache.commons.compress.utils.InputStreamStatistics; 031import org.apache.commons.io.input.BoundedInputStream; 032 033/** 034 * CompressorInputStream for the framing Snappy format. 035 * 036 * <p> 037 * Based on the "spec" in the version "Last revised: 2013-10-25" 038 * </p> 039 * 040 * @see <a href="https://github.com/google/snappy/blob/master/framing_format.txt">Snappy framing format description</a> 041 * @since 1.7 042 */ 043public class FramedSnappyCompressorInputStream extends CompressorInputStream implements InputStreamStatistics { 044 045 /** 046 * package private for tests only. 047 */ 048 static final long MASK_OFFSET = 0xa282ead8L; 049 050 private static final int STREAM_IDENTIFIER_TYPE = 0xff; 051 static final int COMPRESSED_CHUNK_TYPE = 0; 052 private static final int UNCOMPRESSED_CHUNK_TYPE = 1; 053 private static final int PADDING_CHUNK_TYPE = 0xfe; 054 private static final int MIN_UNSKIPPABLE_TYPE = 2; 055 private static final int MAX_UNSKIPPABLE_TYPE = 0x7f; 056 private static final int MAX_SKIPPABLE_TYPE = 0xfd; 057 058 // used by FramedSnappyCompressorOutputStream as well 059 static final byte[] SZ_SIGNATURE = { // NOSONAR 060 (byte) STREAM_IDENTIFIER_TYPE, // tag 061 6, 0, 0, // length 062 's', 'N', 'a', 'P', 'p', 'Y' }; 063 064 /** 065 * Checks if the signature matches what is expected for a .sz file. 066 * 067 * <p> 068 * .sz files start with a chunk with tag 0xff and content sNaPpY. 069 * </p> 070 * 071 * @param signature the bytes to check 072 * @param length the number of bytes to check 073 * @return true if this is a .sz stream, false otherwise 074 */ 075 public static boolean matches(final byte[] signature, final int length) { 076 077 if (length < SZ_SIGNATURE.length) { 078 return false; 079 } 080 081 byte[] shortenedSig = signature; 082 if (signature.length > SZ_SIGNATURE.length) { 083 shortenedSig = Arrays.copyOf(signature, SZ_SIGNATURE.length); 084 } 085 086 return Arrays.equals(shortenedSig, SZ_SIGNATURE); 087 } 088 089 static long unmask(long x) { 090 // ugly, maybe we should just have used ints and deal with the 091 // overflow 092 x -= MASK_OFFSET; 093 x &= 0xffffFFFFL; 094 return (x >> 17 | x << 15) & 0xffffFFFFL; 095 } 096 097 private long unreadBytes; 098 099 private final BoundedInputStream countingStream; 100 101 /** The underlying stream to read compressed data from */ 102 private final PushbackInputStream inputStream; 103 104 /** The dialect to expect */ 105 private final FramedSnappyDialect dialect; 106 107 private SnappyCompressorInputStream currentCompressedChunk; 108 109 // used in no-arg read method 110 private final byte[] oneByte = new byte[1]; 111 private boolean endReached; 112 private boolean inUncompressedChunk; 113 private int uncompressedBytesRemaining; 114 private long expectedChecksum = -1; 115 116 private final int blockSize; 117 118 private final PureJavaCrc32C checksum = new PureJavaCrc32C(); 119 120 private final ByteUtils.ByteSupplier supplier = this::readOneByte; 121 122 /** 123 * Constructs a new input stream that decompresses snappy-framed-compressed data from the specified input stream using the 124 * {@link FramedSnappyDialect#STANDARD} dialect. 125 * 126 * @param in the InputStream from which to read the compressed data 127 * @throws IOException if reading fails 128 */ 129 public FramedSnappyCompressorInputStream(final InputStream in) throws IOException { 130 this(in, FramedSnappyDialect.STANDARD); 131 } 132 133 /** 134 * Constructs a new input stream that decompresses snappy-framed-compressed data from the specified input stream. 135 * 136 * @param in the InputStream from which to read the compressed data 137 * @param dialect the dialect used by the compressed stream 138 * @throws IOException if reading fails 139 */ 140 public FramedSnappyCompressorInputStream(final InputStream in, final FramedSnappyDialect dialect) throws IOException { 141 this(in, SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE, dialect); 142 } 143 144 /** 145 * Constructs a new input stream that decompresses snappy-framed-compressed data from the specified input stream. 146 * 147 * @param in the InputStream from which to read the compressed data 148 * @param blockSize the block size to use for the compressed stream 149 * @param dialect the dialect used by the compressed stream 150 * @throws IOException if reading fails 151 * @throws IllegalArgumentException if blockSize is not bigger than 0 152 * @since 1.14 153 */ 154 public FramedSnappyCompressorInputStream(final InputStream in, final int blockSize, final FramedSnappyDialect dialect) throws IOException { 155 if (blockSize <= 0) { 156 throw new IllegalArgumentException("blockSize must be bigger than 0"); 157 } 158 countingStream = BoundedInputStream.builder().setInputStream(in).get(); 159 this.inputStream = new PushbackInputStream(countingStream, 1); 160 this.blockSize = blockSize; 161 this.dialect = dialect; 162 if (dialect.hasStreamIdentifier()) { 163 readStreamIdentifier(); 164 } 165 } 166 167 /** {@inheritDoc} */ 168 @Override 169 public int available() throws IOException { 170 if (inUncompressedChunk) { 171 return Math.min(uncompressedBytesRemaining, inputStream.available()); 172 } 173 if (currentCompressedChunk != null) { 174 return currentCompressedChunk.available(); 175 } 176 return 0; 177 } 178 179 /** {@inheritDoc} */ 180 @Override 181 public void close() throws IOException { 182 try { 183 org.apache.commons.io.IOUtils.close(currentCompressedChunk); 184 currentCompressedChunk = null; 185 } finally { 186 inputStream.close(); 187 } 188 } 189 190 /** 191 * @since 1.17 192 */ 193 @Override 194 public long getCompressedCount() { 195 return countingStream.getCount() - unreadBytes; 196 } 197 198 /** {@inheritDoc} */ 199 @Override 200 public int read() throws IOException { 201 return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF; 202 } 203 204 /** {@inheritDoc} */ 205 @Override 206 public int read(final byte[] b, final int off, final int len) throws IOException { 207 if (len == 0) { 208 return 0; 209 } 210 int read = readOnce(b, off, len); 211 if (read == -1) { 212 readNextBlock(); 213 if (endReached) { 214 return -1; 215 } 216 read = readOnce(b, off, len); 217 } 218 return read; 219 } 220 221 private long readCrc() throws IOException { 222 final byte[] b = new byte[4]; 223 final int read = IOUtils.readFully(inputStream, b); 224 count(read); 225 if (read != 4) { 226 throw new IOException("Premature end of stream"); 227 } 228 return ByteUtils.fromLittleEndian(b); 229 } 230 231 private void readNextBlock() throws IOException { 232 verifyLastChecksumAndReset(); 233 inUncompressedChunk = false; 234 final int type = readOneByte(); 235 if (type == -1) { 236 endReached = true; 237 } else if (type == STREAM_IDENTIFIER_TYPE) { 238 inputStream.unread(type); 239 unreadBytes++; 240 pushedBackBytes(1); 241 readStreamIdentifier(); 242 readNextBlock(); 243 } else if (type == PADDING_CHUNK_TYPE || type > MAX_UNSKIPPABLE_TYPE && type <= MAX_SKIPPABLE_TYPE) { 244 skipBlock(); 245 readNextBlock(); 246 } else if (type >= MIN_UNSKIPPABLE_TYPE && type <= MAX_UNSKIPPABLE_TYPE) { 247 throw new IOException("Unskippable chunk with type " + type + " (hex " + Integer.toHexString(type) + ") detected."); 248 } else if (type == UNCOMPRESSED_CHUNK_TYPE) { 249 inUncompressedChunk = true; 250 uncompressedBytesRemaining = readSize() - 4 /* CRC */; 251 if (uncompressedBytesRemaining < 0) { 252 throw new IOException("Found illegal chunk with negative size"); 253 } 254 expectedChecksum = unmask(readCrc()); 255 } else if (type == COMPRESSED_CHUNK_TYPE) { 256 final boolean expectChecksum = dialect.usesChecksumWithCompressedChunks(); 257 final long size = readSize() - (expectChecksum ? 4L : 0L); 258 if (size < 0) { 259 throw new IOException("Found illegal chunk with negative size"); 260 } 261 if (expectChecksum) { 262 expectedChecksum = unmask(readCrc()); 263 } else { 264 expectedChecksum = -1; 265 } 266 // @formatter:off 267 currentCompressedChunk = new SnappyCompressorInputStream(BoundedInputStream.builder() 268 .setInputStream(inputStream) 269 .setMaxCount(size) 270 .setPropagateClose(false) 271 .get(), 272 blockSize); 273 // @formatter:on 274 // constructor reads uncompressed size 275 count(currentCompressedChunk.getBytesRead()); 276 } else { 277 // impossible as all potential byte values have been covered 278 throw new IOException("Unknown chunk type " + type + " detected."); 279 } 280 } 281 282 /** 283 * Reads from the current chunk into the given array. 284 * 285 * @return -1 if there is no current chunk or the number of bytes read from the current chunk (which may be -1 if the end of the chunk is reached). 286 */ 287 private int readOnce(final byte[] b, final int off, final int len) throws IOException { 288 int read = -1; 289 if (inUncompressedChunk) { 290 final int amount = Math.min(uncompressedBytesRemaining, len); 291 if (amount == 0) { 292 return -1; 293 } 294 read = inputStream.read(b, off, amount); 295 if (read != -1) { 296 uncompressedBytesRemaining -= read; 297 count(read); 298 } 299 } else if (currentCompressedChunk != null) { 300 final long before = currentCompressedChunk.getBytesRead(); 301 read = currentCompressedChunk.read(b, off, len); 302 if (read == -1) { 303 currentCompressedChunk.close(); 304 currentCompressedChunk = null; 305 } else { 306 count(currentCompressedChunk.getBytesRead() - before); 307 } 308 } 309 if (read > 0) { 310 checksum.update(b, off, read); 311 } 312 return read; 313 } 314 315 private int readOneByte() throws IOException { 316 final int b = inputStream.read(); 317 if (b != -1) { 318 count(1); 319 return b & 0xFF; 320 } 321 return -1; 322 } 323 324 private int readSize() throws IOException { 325 return (int) ByteUtils.fromLittleEndian(supplier, 3); 326 } 327 328 private void readStreamIdentifier() throws IOException { 329 final byte[] b = new byte[10]; 330 final int read = IOUtils.readFully(inputStream, b); 331 count(read); 332 if (10 != read || !matches(b, 10)) { 333 throw new IOException("Not a framed Snappy stream"); 334 } 335 } 336 337 private void skipBlock() throws IOException { 338 final int size = readSize(); 339 if (size < 0) { 340 throw new IOException("Found illegal chunk with negative size"); 341 } 342 final long read = org.apache.commons.io.IOUtils.skip(inputStream, size); 343 count(read); 344 if (read != size) { 345 throw new IOException("Premature end of stream"); 346 } 347 } 348 349 private void verifyLastChecksumAndReset() throws IOException { 350 if (expectedChecksum >= 0 && expectedChecksum != checksum.getValue()) { 351 throw new IOException("Checksum verification failed"); 352 } 353 expectedChecksum = -1; 354 checksum.reset(); 355 } 356 357}