Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions dump/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,6 @@
<artifactId>trove4j</artifactId>
<version>3.0.3</version>
</dependency>
<dependency>
<groupId>org.iq80.snappy</groupId>
<artifactId>snappy</artifactId>
<version>0.4</version>
</dependency>
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
Expand Down
33 changes: 1 addition & 32 deletions dump/src/util/dump/stream/Compression.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
/**
* Using Compression enum values in StreamProviders you can compress your dumps transparently.
*
* With <code>Compression.LZ4</code> and <code>Compression.Snappy</code> there are two options for very fast
* With <code>Compression.LZ4</code> there is a option for very fast
Comment thread
mrtnrdl marked this conversation as resolved.
Outdated
* compression, where one might expect, that performance improves overall, simply because you do less IO.
* Unfortunately, in a single-threaded use-case with no other IO load this is not the case, even when using
* the fastest option, LZ4. Externalization creates high load on CPU, compression increases that load.
Expand All @@ -49,7 +49,6 @@ public enum Compression implements ByteArrayPacker {
GZipLevel7,
GZipLevel8,
GZipLevel9,
Snappy,
LZ4,
Zstd1,
Zstd5,
Expand Down Expand Up @@ -101,7 +100,6 @@ public <E extends Externalizable> byte[] initDictionary( Iterable<E> dictInputPr
@Override
public boolean isPackedSizeInFirstFourBytes() {
switch ( this ) {
case Snappy:
case LZ4:
case Zstd1:
case Zstd5:
Expand Down Expand Up @@ -136,8 +134,6 @@ public byte[] pack( byte[] bytes, int bytesLength, @Nullable byte[] target, @Nul
return gzip(8, bytes, bytesLength);
case GZipLevel9:
return gzip(9, bytes, bytesLength);
case Snappy:
return snappy(bytes, bytesLength, target);
case LZ4:
return lz4(bytes, bytesLength, target);
case Zstd1:
Expand Down Expand Up @@ -168,8 +164,6 @@ public byte[] unpack( byte[] source, int sourceLength, @Nullable byte[] target,
case GZipLevel8:
case GZipLevel9:
return gunzip(source, sourceLength);
case Snappy:
return unsnappy(source, sourceLength, target);
case LZ4:
return unLZ4(source, target);
case Zstd1:
Expand Down Expand Up @@ -256,19 +250,6 @@ private byte[] lz4( byte[] bytes, int bytesLength, byte[] target ) {
return target;
}

private byte[] snappy( byte[] data, int dataLength, byte[] target ) {
int length = org.iq80.snappy.Snappy.maxCompressedLength(dataLength) + 4;
if ( target == null || target.length < length ) {
target = new byte[length];
}
int compressedSize = org.iq80.snappy.Snappy.compress(data, 0, dataLength, target, 4);
target[0] = (byte)((compressedSize >>> 24) & 0xFF);
target[1] = (byte)((compressedSize >>> 16) & 0xFF);
target[2] = (byte)((compressedSize >>> 8) & 0xFF);
target[3] = (byte)((compressedSize) & 0xFF);
return target;
}

private byte[] unLZ4( byte[] bytes, byte[] target ) {
LZ4FastDecompressor lz4Decompressor = getLZ4Decompressor();
int length = (((bytes[0] & 0xff) << 24) + ((bytes[1] & 0xff) << 16) + ((bytes[2] & 0xff) << 8) + (bytes[3] & 0xff));
Expand Down Expand Up @@ -298,18 +279,6 @@ private byte[] unZstd( byte[] source, int sourceLength, byte[] target, byte[] di
return target;
}

private byte[] unsnappy( byte[] bytes, int sourceLength, byte[] target ) {
int length = org.iq80.snappy.Snappy.getUncompressedLength(bytes, 0);
if ( length > 100_000_000 ) {
throw new RuntimeException("insane size for decompressed length:" + length + " - failing now to prevent OutOfMemoryErrors");
}
if ( target == null || target.length < length ) {
target = new byte[length];
}
org.iq80.snappy.Snappy.uncompress(bytes, 0, sourceLength, target, 0);
return target;
}

private byte[] zstd( int compressionLevel, byte[] source, int sourceLength, byte[] target, byte[] dict ) {
int length = sourceLength * 2;
if ( target == null || target.length < length ) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* - the {@link Externalizable}s may not use readObject() during readExternal() or writeObject(o) during writeExternal()<br><br>
* - if you put an instance twice into the dump, you will have two instances after deserialization in memory<br><br>
*
* This ObjectStreamProvider can compress the streams using Gzip, Snappy, LZ4 or ZStd. Use the appropriate constructor with a CompressionType.
* This ObjectStreamProvider can compress the streams using Gzip, LZ4 or ZStd. Use the appropriate constructor with a CompressionType.
* Use compression only if you have limited storage space on your server, an IO bottleneck on your server, or if you access the dumps via
* network and have a network bottleneck.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@ public void testCompression() throws Exception {
test(provider, testBeans);
provider = new ExternalizableObjectStreamProvider(Compression.GZipLevel9);
test(provider, testBeans);
provider = new ExternalizableObjectStreamProvider(Compression.Snappy);
test(provider, testBeans);
provider = new ExternalizableObjectStreamProvider(Compression.LZ4);
test(provider, testBeans);

Expand Down Expand Up @@ -83,7 +81,6 @@ public void testCompression() throws Exception {
test(stProvider, testExternalizableBeans);
stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.GZipLevel9);
test(stProvider, testExternalizableBeans);
stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.Snappy);
test(stProvider, testExternalizableBeans);
stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.LZ4);
test(stProvider, testExternalizableBeans);
Expand Down