Skip to content

Commit 9f8c3cf

Browse files
BDxerial
BD
andauthored
Merge pull request from GHSA-55g7-9cwv-5qfv
* Validate chunk size to be within a configured maximum * Add constructors to have max size configurable * Code cleanup * Use 512MB for consistency --------- Co-authored-by: Taro L. Saito <[email protected]>
1 parent 49d7001 commit 9f8c3cf

File tree

5 files changed

+60
-3
lines changed

5 files changed

+60
-3
lines changed

src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package org.xerial.snappy;
22

3-
import java.io.OutputStream;
4-
53
import org.xerial.snappy.buffer.CachedBufferAllocator;
64

5+
import java.io.OutputStream;
6+
77
public class SnappyHadoopCompatibleOutputStream extends SnappyOutputStream
88
{
99
public SnappyHadoopCompatibleOutputStream(OutputStream out)

src/main/java/org/xerial/snappy/SnappyInputStream.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,11 @@
3636
public class SnappyInputStream
3737
extends InputStream
3838
{
39+
public static final int MAX_CHUNK_SIZE = 512 * 1024 * 1024; // 512 MiB
40+
3941
private boolean finishedReading = false;
4042
protected final InputStream in;
43+
private final int maxChunkSize;
4144

4245
private byte[] compressed;
4346
private byte[] uncompressed;
@@ -55,6 +58,21 @@ public class SnappyInputStream
5558
public SnappyInputStream(InputStream input)
5659
throws IOException
5760
{
61+
this(input, MAX_CHUNK_SIZE);
62+
}
63+
64+
65+
/**
66+
* Create a filter for reading compressed data as a uncompressed stream with provided maximum chunk size
67+
*
68+
* @param input
69+
* @param maxChunkSize
70+
* @throws IOException
71+
*/
72+
public SnappyInputStream(InputStream input, int maxChunkSize)
73+
throws IOException
74+
{
75+
this.maxChunkSize = maxChunkSize;
5876
this.in = input;
5977
readHeader();
6078
}
@@ -422,6 +440,11 @@ protected boolean hasNextChunk()
422440
throw new SnappyError(SnappyErrorCode.INVALID_CHUNK_SIZE, "chunkSize is too big or negative : " + chunkSize);
423441
}
424442

443+
// chunkSize is big
444+
if (chunkSize > maxChunkSize) {
445+
throw new SnappyError(SnappyErrorCode.FAILED_TO_UNCOMPRESS, String.format("Received chunkSize %,d is greater than max configured chunk size %,d", chunkSize, maxChunkSize));
446+
}
447+
425448
// extend the compressed data buffer size
426449
if (compressed == null || chunkSize > compressed.length) {
427450
// chunkSize exceeds limit

src/main/java/org/xerial/snappy/SnappyOutputStream.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
public class SnappyOutputStream
6060
extends OutputStream
6161
{
62+
public static final int MAX_BLOCK_SIZE = 512 * 1024 * 1024; // 512 MiB
6263
static final int MIN_BLOCK_SIZE = 1 * 1024;
6364
static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size
6465

@@ -84,7 +85,7 @@ public SnappyOutputStream(OutputStream out)
8485
/**
8586
* @param out
8687
* @param blockSize byte size of the internal buffer size
87-
* @throws IOException
88+
* @throws IllegalArgumentException when blockSize is larger than 512 MiB
8889
*/
8990
public SnappyOutputStream(OutputStream out, int blockSize)
9091
{
@@ -95,6 +96,9 @@ public SnappyOutputStream(OutputStream out, int blockSize, BufferAllocatorFactor
9596
{
9697
this.out = out;
9798
this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize);
99+
if (this.blockSize > MAX_BLOCK_SIZE){
100+
throw new IllegalArgumentException(String.format("Provided chunk size %,d larger than max %,d", this.blockSize, MAX_BLOCK_SIZE));
101+
}
98102
int inputSize = blockSize;
99103
int outputSize = SnappyCodec.HEADER_SIZE + 4 + Snappy.maxCompressedLength(blockSize);
100104

src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.nio.ByteOrder;
3535

3636
import org.junit.Test;
37+
import org.junit.Assert;
3738
import org.xerial.snappy.buffer.BufferAllocatorFactory;
3839
import org.xerial.snappy.buffer.CachedBufferAllocator;
3940
import org.xerial.snappy.buffer.DefaultBufferAllocator;
@@ -106,6 +107,17 @@ public void bufferSize()
106107
is.close();
107108
}
108109

110+
@Test(expected = IllegalArgumentException.class)
111+
public void invalidBlockSize()
112+
throws Exception
113+
{
114+
// We rely on catch below, if there is no error this test will pass
115+
// This can be done better with Assertions.assertThrows
116+
Boolean exceptionThrown = false;
117+
ByteArrayOutputStream b = new ByteArrayOutputStream();
118+
SnappyOutputStream os = new SnappyOutputStream(b, 1024 * 1024 * 1024);
119+
}
120+
109121
@Test
110122
public void smallWrites()
111123
throws Exception

src/test/java/org/xerial/snappy/SnappyTest.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,24 @@ public void isInvalidChunkLengthForSnappyInputStreamOutOfMemory()
379379
}
380380
}
381381

382+
/*
383+
Tests sad cases for SnappyInputStream.read method
384+
- Expects a failed to compress exception due to upper bounds chunk size
385+
- {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff}
386+
*/
387+
@Test
388+
public void isInvalidChunkLengthForSnappyInputStream()
389+
throws Exception {
390+
byte[] data = {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff};
391+
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data));
392+
byte[] out = new byte[50];
393+
try {
394+
in.read(out);
395+
} catch (SnappyError error) {
396+
Assert.assertEquals(error.errorCode, SnappyErrorCode.FAILED_TO_UNCOMPRESS);
397+
}
398+
}
399+
382400
/*
383401
Tests happy cases for BitShuffle.shuffle method
384402
- double: 0, 10

0 commit comments

Comments
 (0)