From 5d09a9804b39f7fa9901dc31d7999dcf7cf31c9f Mon Sep 17 00:00:00 2001 From: "martin.riedel" Date: Thu, 27 Nov 2025 14:22:00 +0100 Subject: [PATCH 1/3] Updated snappy to 0.5 --- dump/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/pom.xml b/dump/pom.xml index 0018809..de3ca36 100644 --- a/dump/pom.xml +++ b/dump/pom.xml @@ -26,7 +26,7 @@ org.iq80.snappy snappy - 0.4 + 0.5 org.lz4 From 92c846eb5ea044e078a3f3160e0251f3a29b408d Mon Sep 17 00:00:00 2001 From: "martin.riedel" Date: Thu, 27 Nov 2025 15:07:53 +0100 Subject: [PATCH 2/3] Remove snappy --- dump/pom.xml | 5 --- dump/src/util/dump/stream/Compression.java | 33 +------------------ .../SingleTypeObjectStreamProvider.java | 2 +- ...xternalizableObjectStreamProviderTest.java | 3 -- 4 files changed, 2 insertions(+), 41 deletions(-) diff --git a/dump/pom.xml b/dump/pom.xml index de3ca36..9dcd99d 100644 --- a/dump/pom.xml +++ b/dump/pom.xml @@ -23,11 +23,6 @@ trove4j 3.0.3 - - org.iq80.snappy - snappy - 0.5 - org.lz4 lz4-java diff --git a/dump/src/util/dump/stream/Compression.java b/dump/src/util/dump/stream/Compression.java index 4be6ee1..589e200 100644 --- a/dump/src/util/dump/stream/Compression.java +++ b/dump/src/util/dump/stream/Compression.java @@ -28,7 +28,7 @@ /** * Using Compression enum values in StreamProviders you can compress your dumps transparently. * - * With Compression.LZ4 and Compression.Snappy there are two options for very fast + * With Compression.LZ4 there is a option for very fast * compression, where one might expect, that performance improves overall, simply because you do less IO. * Unfortunately, in a single-threaded use-case with no other IO load this is not the case, even when using * the fastest option, LZ4. Externalization creates high load on CPU, compression increases that load. @@ -49,7 +49,6 @@ public enum Compression implements ByteArrayPacker { GZipLevel7, GZipLevel8, GZipLevel9, - Snappy, LZ4, Zstd1, Zstd5, @@ -101,7 +100,6 @@ public byte[] initDictionary( Iterable dictInputPr @Override public boolean isPackedSizeInFirstFourBytes() { switch ( this ) { - case Snappy: case LZ4: case Zstd1: case Zstd5: @@ -136,8 +134,6 @@ public byte[] pack( byte[] bytes, int bytesLength, @Nullable byte[] target, @Nul return gzip(8, bytes, bytesLength); case GZipLevel9: return gzip(9, bytes, bytesLength); - case Snappy: - return snappy(bytes, bytesLength, target); case LZ4: return lz4(bytes, bytesLength, target); case Zstd1: @@ -168,8 +164,6 @@ public byte[] unpack( byte[] source, int sourceLength, @Nullable byte[] target, case GZipLevel8: case GZipLevel9: return gunzip(source, sourceLength); - case Snappy: - return unsnappy(source, sourceLength, target); case LZ4: return unLZ4(source, target); case Zstd1: @@ -256,19 +250,6 @@ private byte[] lz4( byte[] bytes, int bytesLength, byte[] target ) { return target; } - private byte[] snappy( byte[] data, int dataLength, byte[] target ) { - int length = org.iq80.snappy.Snappy.maxCompressedLength(dataLength) + 4; - if ( target == null || target.length < length ) { - target = new byte[length]; - } - int compressedSize = org.iq80.snappy.Snappy.compress(data, 0, dataLength, target, 4); - target[0] = (byte)((compressedSize >>> 24) & 0xFF); - target[1] = (byte)((compressedSize >>> 16) & 0xFF); - target[2] = (byte)((compressedSize >>> 8) & 0xFF); - target[3] = (byte)((compressedSize) & 0xFF); - return target; - } - private byte[] unLZ4( byte[] bytes, byte[] target ) { LZ4FastDecompressor lz4Decompressor = getLZ4Decompressor(); int length = (((bytes[0] & 0xff) << 24) + ((bytes[1] & 0xff) << 16) + ((bytes[2] & 0xff) << 8) + (bytes[3] & 0xff)); @@ -298,18 +279,6 @@ private byte[] unZstd( byte[] source, int sourceLength, byte[] target, byte[] di return target; } - private byte[] unsnappy( byte[] bytes, int sourceLength, byte[] target ) { - int length = org.iq80.snappy.Snappy.getUncompressedLength(bytes, 0); - if ( length > 100_000_000 ) { - throw new RuntimeException("insane size for decompressed length:" + length + " - failing now to prevent OutOfMemoryErrors"); - } - if ( target == null || target.length < length ) { - target = new byte[length]; - } - org.iq80.snappy.Snappy.uncompress(bytes, 0, sourceLength, target, 0); - return target; - } - private byte[] zstd( int compressionLevel, byte[] source, int sourceLength, byte[] target, byte[] dict ) { int length = sourceLength * 2; if ( target == null || target.length < length ) { diff --git a/dump/src/util/dump/stream/SingleTypeObjectStreamProvider.java b/dump/src/util/dump/stream/SingleTypeObjectStreamProvider.java index 0279240..7ff053e 100644 --- a/dump/src/util/dump/stream/SingleTypeObjectStreamProvider.java +++ b/dump/src/util/dump/stream/SingleTypeObjectStreamProvider.java @@ -15,7 +15,7 @@ * - the {@link Externalizable}s may not use readObject() during readExternal() or writeObject(o) during writeExternal()

* - if you put an instance twice into the dump, you will have two instances after deserialization in memory

* - * This ObjectStreamProvider can compress the streams using Gzip, Snappy, LZ4 or ZStd. Use the appropriate constructor with a CompressionType. + * This ObjectStreamProvider can compress the streams using Gzip, LZ4 or ZStd. Use the appropriate constructor with a CompressionType. * Use compression only if you have limited storage space on your server, an IO bottleneck on your server, or if you access the dumps via * network and have a network bottleneck. * diff --git a/dump/test/util/dump/stream/ExternalizableObjectStreamProviderTest.java b/dump/test/util/dump/stream/ExternalizableObjectStreamProviderTest.java index e9261c4..6a75cb9 100644 --- a/dump/test/util/dump/stream/ExternalizableObjectStreamProviderTest.java +++ b/dump/test/util/dump/stream/ExternalizableObjectStreamProviderTest.java @@ -51,8 +51,6 @@ public void testCompression() throws Exception { test(provider, testBeans); provider = new ExternalizableObjectStreamProvider(Compression.GZipLevel9); test(provider, testBeans); - provider = new ExternalizableObjectStreamProvider(Compression.Snappy); - test(provider, testBeans); provider = new ExternalizableObjectStreamProvider(Compression.LZ4); test(provider, testBeans); @@ -83,7 +81,6 @@ public void testCompression() throws Exception { test(stProvider, testExternalizableBeans); stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.GZipLevel9); test(stProvider, testExternalizableBeans); - stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.Snappy); test(stProvider, testExternalizableBeans); stProvider = new SingleTypeObjectStreamProvider<>(TestExternalizableBean.class, Compression.LZ4); test(stProvider, testExternalizableBeans); From 9e2120828802a7f22002d5ea09f5dd2517443663 Mon Sep 17 00:00:00 2001 From: Martin Riedel <1167062+mrtnrdl@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:45:09 +0100 Subject: [PATCH 3/3] Update dump/src/util/dump/stream/Compression.java Co-authored-by: Niklas Keller --- dump/src/util/dump/stream/Compression.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/util/dump/stream/Compression.java b/dump/src/util/dump/stream/Compression.java index 589e200..a6a62e8 100644 --- a/dump/src/util/dump/stream/Compression.java +++ b/dump/src/util/dump/stream/Compression.java @@ -28,7 +28,7 @@ /** * Using Compression enum values in StreamProviders you can compress your dumps transparently. * - * With Compression.LZ4 there is a option for very fast + * With Compression.LZ4 there is an option for very fast * compression, where one might expect, that performance improves overall, simply because you do less IO. * Unfortunately, in a single-threaded use-case with no other IO load this is not the case, even when using * the fastest option, LZ4. Externalization creates high load on CPU, compression increases that load.