diff --git a/.github/workflows/test-integration-unreleased.yml b/.github/workflows/test-integration-unreleased.yml index 965698f4c..e13cd447a 100644 --- a/.github/workflows/test-integration-unreleased.yml +++ b/.github/workflows/test-integration-unreleased.yml @@ -90,7 +90,7 @@ jobs: - name: Run Integration Test run: | cd opensearch-java - ./gradlew clean integrationTest -Dhttps=false + ./gradlew clean integrationTest -Dhttps=false -Dtests.opensearch.testcontainers.enabled=false - name: Upload Reports if: failure() diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 821b765f4..43baaf777 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -40,27 +40,8 @@ jobs: java-version: ${{ matrix.entry.java }} distribution: 'temurin' cache: 'gradle' - - name: Run Docker - run: | - echo "PASSWORD=admin" >> $GITHUB_ENV - docker info - docker compose --project-directory .ci/opensearch build --build-arg OPENSEARCH_VERSION=${{ matrix.entry.opensearch_version }} - docker compose --project-directory .ci/opensearch up -d - sleep 60 - - - name: Sets password (new versions) - run: | - VERSION_COMPONENTS=(${OPENSEARCH_VERSION//./ }) - MAJOR_VERSION=${VERSION_COMPONENTS[0]} - MINOR_VERSION=${VERSION_COMPONENTS[1]} - if (( $MAJOR_VERSION > 2 || ( $MAJOR_VERSION == 2 && $MINOR_VERSION >= 12 ) )); then - echo "PASSWORD=0_aD^min_0" >> $GITHUB_ENV - fi - env: - OPENSEARCH_VERSION: ${{ matrix.entry.opensearch_version }} - - name: Run Integration Test - run: ./gradlew clean integrationTest -Dpassword=${{ env.PASSWORD }} + run: ./gradlew clean integrationTest -Dtests.opensearch.version=${{ matrix.entry.opensearch_version }} - name: Upload Reports if: failure() @@ -69,7 +50,3 @@ jobs: name: test-reports-os${{ matrix.entry.opensearch_version }}-java${{ matrix.entry.java }} path: java-client/build/reports/ retention-days: 7 - - - name: Stop Docker - run: | - docker compose --project-directory .ci/opensearch down \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f85eaecf7..fa68c23cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Bump `org.apache.httpcomponents.client5:httpclient5` from 5.6 to 5.6.1 ([#1967](https://github.com/opensearch-project/opensearch-java/pull/1967)) ### Added +- Run Java client integration tests with a Testcontainers-managed OpenSearch instance by default ([#2033](https://github.com/opensearch-project/opensearch-java/pull/2033)) - Detect AWS SDK `Apache5HttpClient` in `AwsSdk2Transport` body-method guardrail ([#1903](https://github.com/opensearch-project/opensearch-java/pull/1970)) - Support Jackson 3.x release line ([#1810](https://github.com/opensearch-project/opensearch-java/pull/1810)) - Added `equals()` and `hashCode()` implementations to `FieldValue` ([#1998](https://github.com/opensearch-project/opensearch-java/pull/1998)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 6969aba85..208a6644e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -52,17 +52,28 @@ To run unit tests for the java-client: #### Integration Tests -To run integration tests for the java-client, start an OpenSearch cluster using docker and pass the OpenSearch version: +To run integration tests for the java-client: ``` -docker-compose --project-directory .ci/opensearch build --build-arg OPENSEARCH_VERSION=1.3.0 -docker-compose --project-directory .ci/opensearch up -d +./gradlew clean integrationTest ``` -Run integration tests after starting OpenSearch cluster: +By default, the integration test task starts a single OpenSearch test container for the test JVM. To test against a specific OpenSearch image version, pass the OpenSearch version: ``` -./gradlew clean integrationTest +./gradlew clean integrationTest -Dtests.opensearch.version=3.2.0 +``` + +To pass the full official OpenSearch image name, use: + +``` +./gradlew clean integrationTest -Dtests.opensearch.image=opensearchproject/opensearch:3.2.0 +``` + +To run against an already running cluster, disable the test container and pass the cluster endpoint if it is not `localhost:9200`: + +``` +./gradlew clean integrationTest -Dtests.opensearch.testcontainers.enabled=false -Dtests.rest.cluster=localhost:9200 ``` #### AWS Transport Integration Tests diff --git a/java-client/build.gradle.kts b/java-client/build.gradle.kts index 3f23f76b2..14ae23bc3 100644 --- a/java-client/build.gradle.kts +++ b/java-client/build.gradle.kts @@ -143,6 +143,9 @@ tasks.build { dependsOn("spotlessJavaCheck") } +val opensearchVersion = "3.5.0-SNAPSHOT" +val opensearchDockerVersion = opensearchVersion.removeSuffix("-SNAPSHOT") + tasks.test { systemProperty("tests.security.manager", "false") @@ -168,6 +171,16 @@ val integrationTest = task("integrationTest") { systemProperty("https", System.getProperty("https", "true")) systemProperty("user", System.getProperty("user", "admin")) systemProperty("password", System.getProperty("password", "admin")) + systemProperty( + "tests.opensearch.testcontainers.enabled", + System.getProperty("tests.opensearch.testcontainers.enabled", "true") + ) + systemProperty( + "tests.opensearch.version", + System.getProperty("tests.opensearch.version", opensearchDockerVersion) + ) + System.getProperty("tests.rest.cluster")?.let { systemProperty("tests.rest.cluster", it) } + System.getProperty("tests.opensearch.image")?.let { systemProperty("tests.opensearch.image", it) } systemProperty("tests.awsSdk2support.domainHost", System.getProperty("tests.awsSdk2support.domainHost", null)) systemProperty("tests.awsSdk2support.serviceName", @@ -176,8 +189,6 @@ val integrationTest = task("integrationTest") { System.getProperty("tests.awsSdk2support.domainRegion", "us-east-1")) } -val opensearchVersion = "3.5.0-SNAPSHOT" - dependencies { val jacksonVersion = "2.21.2" val jacksonDatabindVersion = "2.21.2" @@ -377,6 +388,7 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) { compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output runtimeClasspath += sourceSets.main.get().output + sourceSets.test.get().output srcDir("src/test/java11") + srcDir("src/test/java21") } } @@ -387,6 +399,8 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) { testImplementation("org.opensearch.test", "framework", opensearchVersion) { exclude(group = "org.hamcrest") } + testImplementation("org.opensearch:opensearch-testcontainers:4.1.0") + testImplementation("org.testcontainers:testcontainers:2.0.4") } tasks.named("compileJava21Java") { @@ -408,4 +422,4 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) { testClassesDirs += java21.output.classesDirs classpath = sourceSets["java21"].runtimeClasspath } -} \ No newline at end of file +} diff --git a/java-client/src/test/java11/org/opensearch/client/opensearch/integTest/OpenSearchJavaClientTestCase.java b/java-client/src/test/java11/org/opensearch/client/opensearch/integTest/OpenSearchJavaClientTestCase.java index e4825420a..9b23bb7fa 100644 --- a/java-client/src/test/java11/org/opensearch/client/opensearch/integTest/OpenSearchJavaClientTestCase.java +++ b/java-client/src/test/java11/org/opensearch/client/opensearch/integTest/OpenSearchJavaClientTestCase.java @@ -8,6 +8,7 @@ package org.opensearch.client.opensearch.integTest; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -30,6 +31,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; +import org.junit.ClassRule; import org.opensearch.Version; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; @@ -45,6 +47,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.test.rest.OpenSearchRestTestCase; +@ThreadLeakFilters(filters = TestcontainersThreadFilter.class) public abstract class OpenSearchJavaClientTestCase extends OpenSearchRestTestCase implements OpenSearchTransportSupport { private static final List systemIndices = List.of( ".opensearch-observability", @@ -59,6 +62,11 @@ public abstract class OpenSearchJavaClientTestCase extends OpenSearchRestTestCas private static TreeSet nodeVersions; private static List clusterHosts; + // The integration tests run through JUnit 4 (RandomizedRunner), so @ClassRule is the pre/post + // lifecycle hook; the rule starts a single container shared by the whole test JVM. + @ClassRule + public static final OpenSearchTestContainerRule testContainer = new OpenSearchTestContainerRule(); + @Before public void initJavaClient() throws IOException { if (javaClient == null) { diff --git a/java-client/src/test/java11/org/opensearch/client/opensearch/integTest/TestcontainersThreadFilter.java b/java-client/src/test/java11/org/opensearch/client/opensearch/integTest/TestcontainersThreadFilter.java new file mode 100644 index 000000000..727ab00ee --- /dev/null +++ b/java-client/src/test/java11/org/opensearch/client/opensearch/integTest/TestcontainersThreadFilter.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.opensearch.integTest; + +import com.carrotsearch.randomizedtesting.ThreadFilter; + +public final class TestcontainersThreadFilter implements ThreadFilter { + @Override + public boolean reject(Thread thread) { + String name = thread.getName(); + // Testcontainers owns these helper threads and Ryuk cleans them up after the JVM exits. + return "testcontainers-ryuk".equals(name) || name.startsWith("testcontainers-pull-watchdog-") || name.startsWith("ducttape-"); + } +} diff --git a/java-client/src/test/java21/org/opensearch/client/opensearch/integTest/OpenSearchTestContainerRule.java b/java-client/src/test/java21/org/opensearch/client/opensearch/integTest/OpenSearchTestContainerRule.java new file mode 100644 index 000000000..2cd8823d2 --- /dev/null +++ b/java-client/src/test/java21/org/opensearch/client/opensearch/integTest/OpenSearchTestContainerRule.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.opensearch.integTest; + +import java.net.URI; +import org.junit.rules.ExternalResource; +import org.opensearch.testcontainers.OpenSearchContainer; +import org.opensearch.testcontainers.OpenSearchDockerImage; + +/** + * Starts one OpenSearch test container for the whole test JVM and points the integration tests at + * it through system properties. The container is shared by every test class, so it is left running + * after each class; Testcontainers' Ryuk reaper removes it once the JVM exits. + */ +final class OpenSearchTestContainerRule extends ExternalResource { + static final String ENABLED_PROPERTY = "tests.opensearch.testcontainers.enabled"; + static final String VERSION_PROPERTY = "tests.opensearch.version"; + static final String IMAGE_PROPERTY = "tests.opensearch.image"; + static final String CLUSTER_PROPERTY = "tests.rest.cluster"; + static final String HTTPS_PROPERTY = "https"; + static final String USER_PROPERTY = "user"; + static final String PASSWORD_PROPERTY = "password"; + + private static final String DEFAULT_ADMIN_PASSWORD = "admin"; + + private static OpenSearchContainer container; + + @Override + protected void before() { + startIfNeeded(); + } + + static synchronized void startIfNeeded() { + if (hasText(System.getProperty(CLUSTER_PROPERTY)) || !testcontainersEnabled()) { + return; + } + + if (container == null) { + OpenSearchContainer openSearch = createContainer(); + openSearch.start(); + container = openSearch; + } + + // getHttpHostAddress() is scheme-prefixed, but tests.rest.cluster expects host:port. + URI httpHostAddress = URI.create(container.getHttpHostAddress()); + System.setProperty(CLUSTER_PROPERTY, httpHostAddress.getHost() + ":" + httpHostAddress.getPort()); + System.setProperty(HTTPS_PROPERTY, Boolean.toString(container.isSecurityEnabled())); + System.setProperty(USER_PROPERTY, container.getUsername()); + System.setProperty(PASSWORD_PROPERTY, container.getPassword()); + } + + private static OpenSearchContainer createContainer() { + String image = System.getProperty(IMAGE_PROPERTY); + OpenSearchContainer openSearch = hasText(image) + ? new OpenSearchContainer<>(image) + : new OpenSearchContainer<>(OpenSearchDockerImage.ofVersion(requiredVersion())); + + // Disk watermarks must stay disabled; constrained disks otherwise trip index_create_block_exception. + openSearch.withSecurityEnabled().withEnv("cluster.routing.allocation.disk.threshold_enabled", "false"); + + // The container has no password setter; OPENSEARCH_INITIAL_ADMIN_PASSWORD is its supported + // input and getPassword() reflects it. Only forward a non-default override: when the env is + // unset, the container substitutes its own strong default on images >= 2.12. + String configuredPassword = System.getProperty(PASSWORD_PROPERTY); + if (hasText(configuredPassword) && !DEFAULT_ADMIN_PASSWORD.equals(configuredPassword)) { + openSearch.withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", configuredPassword); + } + + return openSearch; + } + + private static String requiredVersion() { + String version = System.getProperty(VERSION_PROPERTY); + if (!hasText(version)) { + throw new IllegalStateException("Missing " + VERSION_PROPERTY + " for OpenSearch Testcontainers image"); + } + return version; + } + + private static boolean testcontainersEnabled() { + return Boolean.parseBoolean(System.getProperty(ENABLED_PROPERTY, "true")); + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } +}