diff --git a/build.gradle b/build.gradle index 4cd1cbb3e09..483d3b4a3bc 100644 --- a/build.gradle +++ b/build.gradle @@ -547,8 +547,38 @@ tasks.register("prepareFilebeatDownload") { } } +def verifyPackageSHA(String artifactProject, String version, String packageName) { + def res = SnapshotArtifactURLs.packageUrls(artifactProject, version, packageName) + String remoteSHACode = res.packageShaUrl.toURL().text.split(' ')[0] + + def localArchive = new File("${projectDir}/build/${packageName}.tar.gz") + if (localArchive.exists()) { + ant.checksum(file: localArchive, algorithm: "SHA-512", forceoverwrite: true) + String localSHA = new File("${projectDir}/build/${packageName}.tar.gz.SHA-512").text.trim() + if (localSHA != remoteSHACode) { + println "${artifactProject} package calculated fingerprint is different from remote, deleting local archive" + delete(localArchive) + } else { + println "Local ${artifactProject} package is already the latest" + } + } +} + +tasks.register("checkFilebeatSHA") { + dependsOn configureArtifactInfo + + description = "Download Filebeat version remote's fingerprint file" + + def projectRef = project + doLast { + String beatsVersion = projectRef.ext.get("artifactApiVersion") + String downloadedFilebeatName = "filebeat-${beatsVersion}-${projectRef.ext.get("beatsArchitecture")}" + verifyPackageSHA("beats", beatsVersion, downloadedFilebeatName) + } +} + tasks.register("downloadFilebeat", Download) { - dependsOn prepareFilebeatDownload + dependsOn prepareFilebeatDownload, checkFilebeatSHA description = "Download Filebeat Snapshot for current branch version: ${version}" project.ext.set("versionFound", true) @@ -573,16 +603,18 @@ tasks.register("deleteLocalFilebeat", Delete) { } tasks.register("copyFilebeat") { - dependsOn = [downloadFilebeat, deleteLocalFilebeat] + dependsOn downloadFilebeat + inputs.files(tasks.named("downloadFilebeat")) + outputs.dir('./build/filebeat') + mustRunAfter tasks.named("unpackTarDistribution") doLast { + delete('./build/filebeat') copy { from tarTree(resources.gzip(project.ext.filebeatDownloadLocation)) into "./build/" } file("./build/${project.ext.unpackedFilebeatName}").renameTo('./build/filebeat') System.out.println "Unzipped ${project.ext.filebeatDownloadLocation} to ./build/filebeat" - System.out.println "Deleting ${project.ext.filebeatDownloadLocation}" - delete(project.ext.filebeatDownloadLocation) } } @@ -595,34 +627,7 @@ tasks.register("checkEsSHA") { doLast { String esVersion = projectRef.ext.get("artifactApiVersion") String downloadedElasticsearchName = "elasticsearch-${esVersion}-${projectRef.ext.get("esArchitecture")}" - String remoteSHA - - def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName) - remoteSHA = res.packageShaUrl.toURL().text - - def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz") - if (localESArchive.exists()) { - // this create a file named localESArchive with ".SHA-512" postfix - ant.checksum(file: localESArchive, algorithm: "SHA-512", forceoverwrite: true) - - File localESCalculatedSHAFile = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz.SHA-512") - String localESCalculatedSHA = localESCalculatedSHAFile.text.trim() - def splitted = remoteSHA.split(' ') - String remoteSHACode = splitted[0] - if (localESCalculatedSHA != remoteSHACode) { - println "ES package calculated fingerprint is different from remote, deleting local archive" - delete(localESArchive) - } else { - println "Local ES package is already the latest" - } - }/* else { - mkdir project.buildDir - // touch the SHA file else downloadEs task doesn't start, this file his input for the other task - new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz.SHA-512").withWriter { w -> - w << "${downloadedElasticsearchName} not yet downloaded" - w.close() - } - }*/ + verifyPackageSHA("elasticsearch", esVersion, downloadedElasticsearchName) } } @@ -668,17 +673,19 @@ tasks.register("deleteLocalEs", Delete) { } tasks.register("copyEs") { - dependsOn = [downloadEs, deleteLocalEs] + dependsOn downloadEs + inputs.files(tasks.named("downloadEs")) + outputs.dir('./build/elasticsearch') + mustRunAfter tasks.named("unpackTarDistribution") doLast { println "copyEs executing.." + delete('./build/elasticsearch') copy { from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation)) into "./build/" } - file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch') println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch" - println "Deleting ${project.ext.elasticsearchDownloadLocation}" } } diff --git a/x-pack/AGENTS.md b/x-pack/AGENTS.md new file mode 100644 index 00000000000..e889f9e568d --- /dev/null +++ b/x-pack/AGENTS.md @@ -0,0 +1,87 @@ +# x-pack/AGENTS.md + +Guidance for coding agents working with X-Pack (Elastic-licensed) features. See also the root `AGENTS.md` for general project conventions. + +## Overview + +X-Pack provides commercial features that extend the Logstash core. Most features integrate via the **UniversalPlugin** extension pattern; License Checking is a standalone library used by the other features. X-Pack is a separate Ruby package under `x-pack/` with its own test suite. Build without X-Pack by setting `OSS=true`. + +## Features + +| Feature | Path | Purpose | +|---------|------|---------| +| **Monitoring** | `lib/monitoring/` | Collects JVM, system, and pipeline metrics; ships to Elasticsearch | +| **Config Management** | `lib/config_management/` | Fetches pipeline configs from Elasticsearch/Kibana | +| **GeoIP Database Management** | `lib/geoip_database_management/` | Auto-downloads and updates GeoIP databases from Elastic CDN | +| **License Checking** | `lib/license_checker/` | Validates Elasticsearch license for feature gating | + +## Extension Pattern + +Monitoring, Config Management, and GeoIP follow the same integration pattern. Each has an **extension class** inheriting from `LogStash::UniversalPlugin` that implements two methods: + +1. **`additionals_settings(settings)`** — Registers `xpack.*` configuration settings with the core settings registry. +2. **`register_hooks(hooks)`** — Registers lifecycle callbacks with `LogStash::Runner` (e.g. `before_bootstrap_checks`, `after_bootstrap_checks`). + +**Entry point:** `lib/x-pack/logstash_registry.rb` registers all three extensions plus built-in input/output plugins with `LogStash::PLUGIN_REGISTRY`. + +**Extension files:** +- `lib/config_management/extension.rb` +- `lib/geoip_database_management/extension.rb` +- `lib/monitoring/monitoring.rb` (extension at bottom of file) + +### Adding New Settings + +Use `LogStash::Setting::*` classes in `additionals_settings`: + +```ruby +def additionals_settings(settings) + settings.register(LogStash::Setting::BooleanSetting.new("xpack.feature.enabled", false)) + settings.register(LogStash::Setting::TimeValueSetting.new("xpack.feature.interval", "5s")) + settings.register(LogStash::Setting::ArrayCoercible.new("xpack.feature.hosts", String, ["localhost"])) + settings.register(LogStash::Setting::NullableStringSetting.new("xpack.feature.password")) +end +``` + +All X-Pack settings use the `xpack.` prefix. Elasticsearch connection options (hosts, SSL, auth, proxy) are shared across features via the `ElasticsearchOptions` helper mixin. + +## License Checking + +Features that require a commercial license use the `Licensed` mixin (`lib/license_checker/licensed.rb`): + +1. Call `setup_license_checker(FEATURE_NAME)` during initialization. +2. Wrap feature logic in `with_license_check(raise_on_error) { ... }`. +3. Override `populate_license_state(xpack_info, is_serverless)` to return `{ :state => :ok | :error, :log_level => ..., :log_message => ... }`. + +The `LicenseManager` (`lib/license_checker/license_manager.rb`) polls Elasticsearch every 30 seconds and notifies observers on state changes. License types: `trial`, `basic`, `standard`, `gold`, `platinum`, `enterprise`. Config management requires trial or above (not basic). + +## Running Tests + +```bash +# Unit tests (from repo root) +./gradlew :logstash-xpack:rubyTests + +# Integration tests (requires running Elasticsearch) +./gradlew :logstash-xpack:rubyIntegrationTests + +# Single integration spec +./gradlew :logstash-xpack:rubyIntegrationTests \ + -PrubyIntegrationSpecs=qa/integration/management/multiple_pipelines_spec.rb +``` + +### Test Structure + +- **Unit specs:** `x-pack/spec/` — Organized by feature (`spec/monitoring/`, `spec/config_management/`, `spec/geoip_database_management/`, `spec/license_checker/`). +- **Integration tests:** `x-pack/qa/integration/` — Subdirectories for `management/`, `monitoring/`, and `fips-validation/`. +- **Test helpers:** `x-pack/spec/support/helpers.rb` and `x-pack/spec/support/matchers.rb`. +- **Test runners:** JUnit-based RSpec invokers in `x-pack/src/test/java/org/logstash/xpack/test/` (`RSpecTests.java` for unit, `RSpecIntegrationTests.java` for integration). + +### GeoIP Test Data + +The `unzipGeolite` Gradle task downloads GeoLite2 test databases. Unit tests for GeoIP depend on this task running first (handled automatically by the `rubyTests` dependency chain). + +## Key Architectural Patterns + +- **Hook-based integration.** Config management intercepts bootstrap checks to swap the local config source with an Elasticsearch source. Monitoring adds an internal pipeline after the agent starts. Neither modifies core code directly. +- **Singleton managers.** `GeoipDatabaseManagement::Manager` and `LicenseChecker::LicenseManager` are singletons with thread-safe initialization via Mutex. +- **Observer pattern.** Database subscriptions and license managers notify observers on state changes, enabling features to react dynamically without polling. +- **Internal pipelines.** Monitoring generates its pipeline config from an ERB template (`lib/template.cfg.erb`) and injects it via `InternalPipelineSource`. These run alongside user pipelines but are hidden from the user-facing API. diff --git a/x-pack/build.gradle b/x-pack/build.gradle index b9e311f1a2f..f3456971a68 100644 --- a/x-pack/build.gradle +++ b/x-pack/build.gradle @@ -70,6 +70,10 @@ tasks.register("rubyIntegrationTests", Test) { inputs.files fileTree("${rootProject.projectDir}/Gemfile.lock") inputs.files fileTree("${rootProject.projectDir}/logstash-core/lib") systemProperty 'logstash.root.dir', projectDir.parent + if (project.hasProperty('rubyIntegrationSpecs')) { + systemProperty 'org.logstash.xpack.integration.specs', project.property('rubyIntegrationSpecs') + } + outputs.upToDateWhen { false } include '/org/logstash/xpack/test/RSpecIntegrationTests.class' } diff --git a/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java b/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java index c62f29f17a0..8b7bd32b952 100644 --- a/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java +++ b/x-pack/src/test/java/org/logstash/xpack/test/RSpecIntegrationTests.java @@ -16,7 +16,8 @@ public class RSpecIntegrationTests extends RSpecTests { @Override protected List rspecArgs() { - return Arrays.asList("-fd", "qa/integration"); + String specs = System.getProperty("org.logstash.xpack.integration.specs", "qa/integration"); + return Arrays.asList("-fd", specs); } @Test