From 7c4b32a501d079d79b92c9ae53d00bbe2f7d8066 Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Sun, 2 Aug 2026 13:27:38 +0300 Subject: [PATCH 1/2] Reject LZ4 match offsets that point before the start of the output LZ4#decompress read a match offset without checking it against the number of bytes written so far. An offset larger than dOff makes the match reference bytes before dest[0], which this call never wrote, so corrupt input failed with ArrayIndexOutOfBoundsException from System.arraycopy or from the incremental copy loop rather than with a checked IOException. The bound is dOff and not the number of bytes decompressed by this call, because a preset dictionary is placed in dest[dOff-dictLen:dOff] and a match may legitimately reference it. The compressor already asserts the same invariant when writing a match (matchDec > 0 && matchDec < 1 << 16 at LZ4.java:183), so this only affects input that was not produced by LZ4#compress. --- lucene/CHANGES.txt | 4 ++ .../org/apache/lucene/util/compress/LZ4.java | 5 ++ .../util/compress/TestDecompressLZ4.java | 66 +++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 92e7fed208e2..34bcd959a4e9 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -407,6 +407,10 @@ Bug Fixes which made the join set degenerate to the whole graph on hub-and-spoke shaped HNSW graphs. (Mayya Sharipova) +* GITHUB#PENDING: Reject LZ4 match offsets that point before the start of the output in + LZ4#decompress. Corrupt input previously threw ArrayIndexOutOfBoundsException instead of + a checked IOException. (Serhiy Bzhezytskyy) + * GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward) * GITHUB#16378: Accumulate join Total/Avg scores in double precision in diff --git a/lucene/core/src/java/org/apache/lucene/util/compress/LZ4.java b/lucene/core/src/java/org/apache/lucene/util/compress/LZ4.java index df9c5e6fc25f..ae9f126b8a2f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/compress/LZ4.java +++ b/lucene/core/src/java/org/apache/lucene/util/compress/LZ4.java @@ -114,6 +114,11 @@ public static int decompress(DataInput compressed, int decompressedLen, byte[] d if (matchDec == 0) { throw new IOException("offset 0 is invalid"); } + if (matchDec > dOff) { + // The match would reference bytes before dest[0], which this call never wrote. + throw new IOException( + "match offset " + matchDec + " is invalid, only " + dOff + " bytes are available"); + } int matchLen = token & 0x0F; if (matchLen == 0x0F) { diff --git a/lucene/core/src/test/org/apache/lucene/util/compress/TestDecompressLZ4.java b/lucene/core/src/test/org/apache/lucene/util/compress/TestDecompressLZ4.java index 9e4aeff5002b..8cd63baf1b7a 100644 --- a/lucene/core/src/test/org/apache/lucene/util/compress/TestDecompressLZ4.java +++ b/lucene/core/src/test/org/apache/lucene/util/compress/TestDecompressLZ4.java @@ -50,4 +50,70 @@ public void testDecompressOffset0() { () -> LZ4.decompress(new ByteArrayDataInput(input), output.length, output, 0)); assertEquals("offset 0 is invalid", e.getMessage()); } + + public void testDecompressOffsetBeyondOutput() { + // A match offset must not point before the start of the output, otherwise the match would + // reference bytes that this call never decompressed. + byte[] input = + new byte[] { + // token: 0 literals, match length 4 (MIN_MATCH) + 0x0, + // offset 8, which is greater than the 0 bytes decompressed so far + 8, + 0, + // last literals + // token + 7 << 4, + // literals + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }; + + byte[] output = new byte[18]; + + var e = + assertThrows( + IOException.class, + () -> LZ4.decompress(new ByteArrayDataInput(input), output.length, output, 0)); + assertEquals("match offset 8 is invalid, only 0 bytes are available", e.getMessage()); + } + + public void testDecompressOffsetBeyondOutputWithDictionary() { + // With a preset dictionary the match offset may reach into the dictionary that the caller + // placed in dest[dOff-dictLen:dOff], so the bound is dOff, not the number of bytes that this + // call decompressed. + byte[] input = + new byte[] { + // token: 0 literals, match length 4 (MIN_MATCH) + 0x0, + // offset 5, one byte past the 4-byte dictionary + 5, + 0, + // last literals + // token + 7 << 4, + // literals + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }; + + byte[] output = new byte[22]; + final int dictLen = 4; + + var e = + assertThrows( + IOException.class, + () -> LZ4.decompress(new ByteArrayDataInput(input), 18, output, dictLen)); + assertEquals("match offset 5 is invalid, only 4 bytes are available", e.getMessage()); + } } From 15dd3e9e3c3295e306a4ffc5abde4b3b5067e07f Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Sun, 2 Aug 2026 18:00:21 +0300 Subject: [PATCH 2/2] Use the PR number in the CHANGES entry --- lucene/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 34bcd959a4e9..ed0a59ca38bc 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -407,7 +407,7 @@ Bug Fixes which made the join set degenerate to the whole graph on hub-and-spoke shaped HNSW graphs. (Mayya Sharipova) -* GITHUB#PENDING: Reject LZ4 match offsets that point before the start of the output in +* GITHUB#16478: Reject LZ4 match offsets that point before the start of the output in LZ4#decompress. Corrupt input previously threw ArrayIndexOutOfBoundsException instead of a checked IOException. (Serhiy Bzhezytskyy)