Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions dump/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,6 @@
<artifactId>trove4j</artifactId>
<version>3.0.3</version>
</dependency>
<dependency>
<groupId>org.iq80.snappy</groupId>
<artifactId>snappy</artifactId>
<version>0.4</version>
</dependency>
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
Expand Down
33 changes: 1 addition & 32 deletions dump/src/util/dump/stream/Compression.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
/**
* Using Compression enum values in StreamProviders you can compress your dumps transparently.
*
* With <code>Compression.LZ4</code> and <code>Compression.Snappy</code> there are two options for very fast
* With <code>Compression.LZ4</code> there is an option for very fast
* compression, where one might expect, that performance improves overall, simply because you do less IO.
* Unfortunately, in a single-threaded use-case with no other IO load this is not the case, even when using
* the fastest option, LZ4. Externalization creates high load on CPU, compression increases that load.
Expand All @@ -49,7 +49,6 @@ public enum Compression implements ByteArrayPacker {
GZipLevel7,
GZipLevel8,
GZipLevel9,
Snappy,
LZ4,
Zstd1,
Zstd5,
Expand Down Expand Up @@ -101,7 +100,6 @@ public <E extends Externalizable> byte[] initDictionary( Iterable<E> dictInputPr
@Override
public boolean isPackedSizeInFirstFourBytes() {
switch ( this ) {
case Snappy:
case LZ4:
case Zstd1:
case Zstd5:
Expand Down Expand Up @@ -136,8 +134,6 @@ public byte[] pack( byte[] bytes, int bytesLength, @Nullable byte[] target, @Nul
return gzip(8, bytes, bytesLength);
case GZipLevel9:
return gzip(9, bytes, bytesLength);
case Snappy:
return snappy(bytes, bytesLength, target);
case LZ4:
return lz4(bytes, bytesLength, target);
case Zstd1:
Expand Down Expand Up @@ -168,8 +164,6 @@ public byte[] unpack( byte[] source, int sourceLength, @Nullable byte[] target,
case GZipLevel8:
case GZipLevel9:
return gunzip(source, sourceLength);
case Snappy:
return unsnappy(source, sourceLength, target);
case LZ4:
return unLZ4(source, target);
case Zstd1:
Expand Down Expand Up @@ -256,19 +250,6 @@ private byte[] lz4( byte[] bytes, int bytesLength, byte[] target ) {
return target;
}

private byte[] snappy( byte[] data, int dataLength, byte[] target ) {
int length = org.iq80.snappy.Snappy.maxCompressedLength(dataLength) + 4;
if ( target == null || target.length < length ) {
target = new byte[length];
}
int compressedSize = org.iq80.snappy.Snappy.compress(data, 0, dataLength, target, 4);
target[0] = (byte)((compressedSize >>> 24) & 0xFF);
target[1] = (byte)((compressedSize >>> 16) & 0xFF);
target[2] = (byte)((compressedSize >>> 8) & 0xFF);
target[3] = (byte)((compressedSize) & 0xFF);
return target;
}

private byte[] unLZ4( byte[] bytes, byte[] target ) {
LZ4FastDecompressor lz4Decompressor = getLZ4Decompressor();
int length = (((bytes[0] & 0xff) << 24) + ((bytes[1] & 0xff) << 16) + ((bytes[2] & 0xff) << 8) + (bytes[3] & 0xff));
Expand Down Expand Up @@ -298,18 +279,6 @@ private byte[] unZstd( byte[] source, int sourceLength, byte[] target, byte[] di
return target;
}

private byte[] unsnappy( byte[] bytes, int sourceLength, byte[] target ) {
int length = org.iq80.snappy.Snappy.getUncompressedLength(bytes, 0);
if ( length > 100_000_000 ) {
throw new RuntimeException("insane size for decompressed length:" + length + " - failing now to prevent OutOfMemoryErrors");
}
if ( target == null || target.length < length ) {
target = new byte[length];
}
org.iq80.snappy.Snappy.uncompress(bytes, 0, sourceLength, target, 0);
return target;
}

private byte[] zstd( int compressionLevel, byte[] source, int sourceLength, byte[] target, byte[] dict ) {
int length = sourceLength * 2;
if ( target == null || target.length < length ) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* - the {@link Externalizable}s may not use readObject() during readExternal() or writeObject(o) during writeExternal()<br><br>
* - if you put an instance twice into the dump, you will have two instances after deserialization in memory<br><br>
*
* This ObjectStreamProvider can compress the streams using Gzip, Snappy, LZ4 or ZStd. Use the appropriate constructor with a CompressionType.
* This ObjectStreamProvider can compress the streams using Gzip, LZ4 or ZStd. Use the appropriate constructor with a CompressionType.
* Use compression only if you have limited storage space on your server, an IO bottleneck on your server, or if you access the dumps via
* network and have a network bottleneck.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@ public void testCompression() throws Exception {
test(provider, testBeans);
provider = new ExternalizableObjectStreamProvider(Compression.GZipLevel9);
test(provider, testBeans);
provider = new ExternalizableObjectStreamProvider(Compression.Snappy);
test(provider, testBeans);
provider = new ExternalizableObjectStreamProvider(Compression.LZ4);
test(provider, testBeans);

Expand Down Expand Up @@ -83,7 +81,6 @@ public void testCompression() throws Exception {
test(stProvider, testExternalizableBeans);
stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.GZipLevel9);
test(stProvider, testExternalizableBeans);
stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.Snappy);
test(stProvider, testExternalizableBeans);
stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.LZ4);
test(stProvider, testExternalizableBeans);
Expand Down