From 6ba562a6f99b167b585fe2bfecfba0559ad1fb47 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Wed, 1 Apr 2026 01:04:50 +0100 Subject: [PATCH 1/6] Make ES and Filebeat extraction incremental to avoid redundant downloads copyEs and copyFilebeat had no inputs/outputs declared and depended on deleteLocalEs/deleteLocalFilebeat, which wiped the extracted directories unconditionally, forcing re-extraction of multi-gigabyte archives on every integration test run. Declare inputs.files and outputs.dir on both tasks so Gradle can skip extraction when the tar.gz is unchanged. Move the delete inside doLast so it only runs when the task actually executes. Add mustRunAfter unpackTarDistribution to satisfy Gradle implicit dependency validation for the shared build/ output directory. Use inputs.files(tasks.named(...)) instead of a closure reading project.ext to avoid configuration-phase property access errors. --- build.gradle | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index dd72f66136c..164a6c0d452 100644 --- a/build.gradle +++ b/build.gradle @@ -564,16 +564,18 @@ tasks.register("deleteLocalFilebeat", Delete) { } tasks.register("copyFilebeat") { - dependsOn = [downloadFilebeat, deleteLocalFilebeat] + dependsOn downloadFilebeat + inputs.files(tasks.named("downloadFilebeat")) + outputs.dir('./build/filebeat') + mustRunAfter tasks.named("unpackTarDistribution") doLast { + delete('./build/filebeat') copy { from tarTree(resources.gzip(project.ext.filebeatDownloadLocation)) into "./build/" } file("./build/${project.ext.unpackedFilebeatName}").renameTo('./build/filebeat') System.out.println "Unzipped ${project.ext.filebeatDownloadLocation} to ./build/filebeat" - System.out.println "Deleting ${project.ext.filebeatDownloadLocation}" - delete(project.ext.filebeatDownloadLocation) } } @@ -659,17 +661,19 @@ tasks.register("deleteLocalEs", Delete) { } tasks.register("copyEs") { - dependsOn = [downloadEs, deleteLocalEs] + dependsOn downloadEs + inputs.files(tasks.named("downloadEs")) + outputs.dir('./build/elasticsearch') + mustRunAfter tasks.named("unpackTarDistribution") doLast { println "copyEs executing.." + delete('./build/elasticsearch') copy { from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation)) into "./build/" } - file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch') println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch" - println "Deleting ${project.ext.elasticsearchDownloadLocation}" } } From 50e8e6010fb480aebcc37a84e637d26d747fe69a Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Wed, 1 Apr 2026 01:05:11 +0100 Subject: [PATCH 2/6] feat: support running a single xpack integration test via -PrubyIntegrationSpecs Pass -PrubyIntegrationSpecs= to target a single spec file instead of the entire qa/integration suite: ./gradlew :logstash-xpack:rubyIntegrationTests \ -PrubyIntegrationSpecs=qa/integration/management/multiple_pipelines_spec.rb RSpecIntegrationTests reads org.logstash.xpack.integration.specs system property with qa/integration as default. x-pack/build.gradle forwards the Gradle property to the JVM system property when present. Also adds outputs.upToDateWhen { false } so rubyIntegrationTests always executes when invoked, regardless of Gradle UP-TO-DATE state. --- x-pack/build.gradle | 4 ++++ .../java/org/logstash/xpack/test/RSpecIntegrationTests.java | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/build.gradle b/x-pack/build.gradle index b9e311f1a2f..f3456971a68 100644 --- a/x-pack/build.gradle +++ b/x-pack/build.gradle @@ -70,6 +70,10 @@ tasks.register("rubyIntegrationTests", Test) { inputs.files fileTree("${rootProject.projectDir}/Gemfile.lock") inputs.files fileTree("${rootProject.projectDir}/logstash-core/lib") systemProperty 'logstash.root.dir', projectDir.parent + if (project.hasProperty('rubyIntegrationSpecs')) { + systemProperty 'org.logstash.xpack.integration.specs', project.property('rubyIntegrationSpecs') + } + outputs.upToDateWhen { false } include '/org/logstash/xpack/test/RSpecIntegrationTests.class' } diff --git a/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java b/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java index c62f29f17a0..8b7bd32b952 100644 --- a/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java +++ b/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java @@ -16,7 +16,8 @@ public class RSpecIntegrationTests extends RSpecTests { @Override protected List rspecArgs() { - return Arrays.asList("-fd", "qa/integration"); + String specs = System.getProperty("org.logstash.xpack.integration.specs", "qa/integration"); + return Arrays.asList("-fd", specs); } @Test From 33c227e3b308dd8514f62fd22f1035be8d01cd31 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Wed, 1 Apr 2026 01:32:39 +0100 Subject: [PATCH 3/6] docs: document single xpack integration spec command in x-pack/AGENTS.md --- x-pack/AGENTS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/AGENTS.md b/x-pack/AGENTS.md index a4b01850824..e889f9e568d 100644 --- a/x-pack/AGENTS.md +++ b/x-pack/AGENTS.md @@ -62,6 +62,10 @@ The `LicenseManager` (`lib/license_checker/license_manager.rb`) polls Elasticsea # Integration tests (requires running Elasticsearch) ./gradlew :logstash-xpack:rubyIntegrationTests + +# Single integration spec +./gradlew :logstash-xpack:rubyIntegrationTests \ + -PrubyIntegrationSpecs=qa/integration/management/multiple_pipelines_spec.rb ``` ### Test Structure From cb245478c701e90eab5b5e2981fab16a050a48cd Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 2 Apr 2026 10:50:58 +0100 Subject: [PATCH 4/6] feat: add SHA verification for Filebeat download Add checkFilebeatSHA task mirroring checkEsSHA. Without it, daily snapshot rebuilds of the same version go undetected because Gradle skips downloadFilebeat when versions.yml is unchanged. --- build.gradle | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 164a6c0d452..1ab527b51ae 100644 --- a/build.gradle +++ b/build.gradle @@ -538,8 +538,40 @@ tasks.register("prepareFilebeatDownload") { } } +tasks.register("checkFilebeatSHA") { + dependsOn configureArtifactInfo + + description = "Download Filebeat version remote's fingerprint file" + + def projectRef = project + doLast { + String beatsVersion = projectRef.ext.get("artifactApiVersion") + String downloadedFilebeatName = "filebeat-${beatsVersion}-${projectRef.ext.get("beatsArchitecture")}" + String remoteSHA + + def res = SnapshotArtifactURLs.packageUrls("beats", beatsVersion, downloadedFilebeatName) + remoteSHA = res.packageShaUrl.toURL().text + + def localFilebeatArchive = new File("${projectDir}/build/${downloadedFilebeatName}.tar.gz") + if (localFilebeatArchive.exists()) { + ant.checksum(file: localFilebeatArchive, algorithm: "SHA-512", forceoverwrite: true) + + File localFilebeatCalculatedSHAFile = new File("${projectDir}/build/${downloadedFilebeatName}.tar.gz.SHA-512") + String localFilebeatCalculatedSHA = localFilebeatCalculatedSHAFile.text.trim() + def splitted = remoteSHA.split(' ') + String remoteSHACode = splitted[0] + if (localFilebeatCalculatedSHA != remoteSHACode) { + println "Filebeat package calculated fingerprint is different from remote, deleting local archive" + delete(localFilebeatArchive) + } else { + println "Local Filebeat package is already the latest" + } + } + } +} + tasks.register("downloadFilebeat", Download) { - dependsOn prepareFilebeatDownload + dependsOn prepareFilebeatDownload, checkFilebeatSHA description = "Download Filebeat Snapshot for current branch version: ${version}" project.ext.set("versionFound", true) From 4b10ed8f31310a8123bb3b0c3189faa49e15c14f Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 2 Apr 2026 11:03:37 +0100 Subject: [PATCH 5/6] refactor: extract verifyPackageSHA helper to deduplicate checkEsSHA and checkFilebeatSHA Both tasks share identical logic: fetch remote SHA, compare with local archive, delete local if different. Extract into a single method. --- build.gradle | 67 +++++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 48 deletions(-) diff --git a/build.gradle b/build.gradle index 1ab527b51ae..b8d02254316 100644 --- a/build.gradle +++ b/build.gradle @@ -538,6 +538,23 @@ tasks.register("prepareFilebeatDownload") { } } +def verifyPackageSHA(String artifactProject, String version, String packageName, String label) { + def res = SnapshotArtifactURLs.packageUrls(artifactProject, version, packageName) + String remoteSHACode = res.packageShaUrl.toURL().text.split(' ')[0] + + def localArchive = new File("${projectDir}/build/${packageName}.tar.gz") + if (localArchive.exists()) { + ant.checksum(file: localArchive, algorithm: "SHA-512", forceoverwrite: true) + String localSHA = new File("${projectDir}/build/${packageName}.tar.gz.SHA-512").text.trim() + if (localSHA != remoteSHACode) { + println "${label} package calculated fingerprint is different from remote, deleting local archive" + delete(localArchive) + } else { + println "Local ${label} package is already the latest" + } + } +} + tasks.register("checkFilebeatSHA") { dependsOn configureArtifactInfo @@ -547,26 +564,7 @@ tasks.register("checkFilebeatSHA") { doLast { String beatsVersion = projectRef.ext.get("artifactApiVersion") String downloadedFilebeatName = "filebeat-${beatsVersion}-${projectRef.ext.get("beatsArchitecture")}" - String remoteSHA - - def res = SnapshotArtifactURLs.packageUrls("beats", beatsVersion, downloadedFilebeatName) - remoteSHA = res.packageShaUrl.toURL().text - - def localFilebeatArchive = new File("${projectDir}/build/${downloadedFilebeatName}.tar.gz") - if (localFilebeatArchive.exists()) { - ant.checksum(file: localFilebeatArchive, algorithm: "SHA-512", forceoverwrite: true) - - File localFilebeatCalculatedSHAFile = new File("${projectDir}/build/${downloadedFilebeatName}.tar.gz.SHA-512") - String localFilebeatCalculatedSHA = localFilebeatCalculatedSHAFile.text.trim() - def splitted = remoteSHA.split(' ') - String remoteSHACode = splitted[0] - if (localFilebeatCalculatedSHA != remoteSHACode) { - println "Filebeat package calculated fingerprint is different from remote, deleting local archive" - delete(localFilebeatArchive) - } else { - println "Local Filebeat package is already the latest" - } - } + verifyPackageSHA("beats", beatsVersion, downloadedFilebeatName, "Filebeat") } } @@ -620,34 +618,7 @@ tasks.register("checkEsSHA") { doLast { String esVersion = projectRef.ext.get("artifactApiVersion") String downloadedElasticsearchName = "elasticsearch-${esVersion}-${projectRef.ext.get("esArchitecture")}" - String remoteSHA - - def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName) - remoteSHA = res.packageShaUrl.toURL().text - - def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz") - if (localESArchive.exists()) { - // this create a file named localESArchive with ".SHA-512" postfix - ant.checksum(file: localESArchive, algorithm: "SHA-512", forceoverwrite: true) - - File localESCalculatedSHAFile = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz.SHA-512") - String localESCalculatedSHA = localESCalculatedSHAFile.text.trim() - def splitted = remoteSHA.split(' ') - String remoteSHACode = splitted[0] - if (localESCalculatedSHA != remoteSHACode) { - println "ES package calculated fingerprint is different from remote, deleting local archive" - delete(localESArchive) - } else { - println "Local ES package is already the latest" - } - }/* else { - mkdir project.buildDir - // touch the SHA file else downloadEs task doesn't start, this file his input for the other task - new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz.SHA-512").withWriter { w -> - w << "${downloadedElasticsearchName} not yet downloaded" - w.close() - } - }*/ + verifyPackageSHA("elasticsearch", esVersion, downloadedElasticsearchName, "ES") } } From b6fcf7eecaad39d53b5659277fc3088b2bc818dd Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 2 Apr 2026 11:29:58 +0100 Subject: [PATCH 6/6] refactor: remove redundant label param from verifyPackageSHA Use artifactProject as the log label instead. --- build.gradle | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index b8d02254316..a41412bcec7 100644 --- a/build.gradle +++ b/build.gradle @@ -538,7 +538,7 @@ tasks.register("prepareFilebeatDownload") { } } -def verifyPackageSHA(String artifactProject, String version, String packageName, String label) { +def verifyPackageSHA(String artifactProject, String version, String packageName) { def res = SnapshotArtifactURLs.packageUrls(artifactProject, version, packageName) String remoteSHACode = res.packageShaUrl.toURL().text.split(' ')[0] @@ -547,10 +547,10 @@ def verifyPackageSHA(String artifactProject, String version, String packageName, ant.checksum(file: localArchive, algorithm: "SHA-512", forceoverwrite: true) String localSHA = new File("${projectDir}/build/${packageName}.tar.gz.SHA-512").text.trim() if (localSHA != remoteSHACode) { - println "${label} package calculated fingerprint is different from remote, deleting local archive" + println "${artifactProject} package calculated fingerprint is different from remote, deleting local archive" delete(localArchive) } else { - println "Local ${label} package is already the latest" + println "Local ${artifactProject} package is already the latest" } } } @@ -564,7 +564,7 @@ tasks.register("checkFilebeatSHA") { doLast { String beatsVersion = projectRef.ext.get("artifactApiVersion") String downloadedFilebeatName = "filebeat-${beatsVersion}-${projectRef.ext.get("beatsArchitecture")}" - verifyPackageSHA("beats", beatsVersion, downloadedFilebeatName, "Filebeat") + verifyPackageSHA("beats", beatsVersion, downloadedFilebeatName) } } @@ -618,7 +618,7 @@ tasks.register("checkEsSHA") { doLast { String esVersion = projectRef.ext.get("artifactApiVersion") String downloadedElasticsearchName = "elasticsearch-${esVersion}-${projectRef.ext.get("esArchitecture")}" - verifyPackageSHA("elasticsearch", esVersion, downloadedElasticsearchName, "ES") + verifyPackageSHA("elasticsearch", esVersion, downloadedElasticsearchName) } }