Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test-integration-unreleased.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:
- name: Run Integration Test
run: |
cd opensearch-java
./gradlew clean integrationTest -Dhttps=false
./gradlew clean integrationTest -Dhttps=false -Dtests.opensearch.testcontainers.enabled=false

- name: Upload Reports
if: failure()
Expand Down
25 changes: 1 addition & 24 deletions .github/workflows/test-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,27 +40,8 @@ jobs:
java-version: ${{ matrix.entry.java }}
distribution: 'temurin'
cache: 'gradle'
- name: Run Docker
run: |
echo "PASSWORD=admin" >> $GITHUB_ENV
docker info
docker compose --project-directory .ci/opensearch build --build-arg OPENSEARCH_VERSION=${{ matrix.entry.opensearch_version }}
docker compose --project-directory .ci/opensearch up -d
sleep 60

- name: Sets password (new versions)
run: |
VERSION_COMPONENTS=(${OPENSEARCH_VERSION//./ })
MAJOR_VERSION=${VERSION_COMPONENTS[0]}
MINOR_VERSION=${VERSION_COMPONENTS[1]}
if (( $MAJOR_VERSION > 2 || ( $MAJOR_VERSION == 2 && $MINOR_VERSION >= 12 ) )); then
echo "PASSWORD=0_aD^min_0" >> $GITHUB_ENV
fi
env:
OPENSEARCH_VERSION: ${{ matrix.entry.opensearch_version }}

- name: Run Integration Test
run: ./gradlew clean integrationTest -Dpassword=${{ env.PASSWORD }}
run: ./gradlew clean integrationTest -Dtests.opensearch.version=${{ matrix.entry.opensearch_version }}

- name: Upload Reports
if: failure()
Expand All @@ -69,7 +50,3 @@ jobs:
name: test-reports-os${{ matrix.entry.opensearch_version }}-java${{ matrix.entry.java }}
path: java-client/build/reports/
retention-days: 7

- name: Stop Docker
run: |
docker compose --project-directory .ci/opensearch down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Bump `org.apache.httpcomponents.client5:httpclient5` from 5.6 to 5.6.1 ([#1967](https://github.com/opensearch-project/opensearch-java/pull/1967))

### Added
- Run Java client integration tests with a Testcontainers-managed OpenSearch instance by default ([#2033](https://github.com/opensearch-project/opensearch-java/pull/2033))
- Detect AWS SDK `Apache5HttpClient` in `AwsSdk2Transport` body-method guardrail ([#1903](https://github.com/opensearch-project/opensearch-java/pull/1970))
- Support Jackson 3.x release line ([#1810](https://github.com/opensearch-project/opensearch-java/pull/1810))
- Added `equals()` and `hashCode()` implementations to `FieldValue` ([#1998](https://github.com/opensearch-project/opensearch-java/pull/1998))
Expand Down
21 changes: 16 additions & 5 deletions DEVELOPER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,28 @@ To run unit tests for the java-client:

#### Integration Tests

To run integration tests for the java-client, start an OpenSearch cluster using docker and pass the OpenSearch version:
To run integration tests for the java-client:

```
docker-compose --project-directory .ci/opensearch build --build-arg OPENSEARCH_VERSION=1.3.0
docker-compose --project-directory .ci/opensearch up -d
./gradlew clean integrationTest
```

Run integration tests after starting OpenSearch cluster:
By default, the integration test task starts a single OpenSearch test container for the test JVM. To test against a specific OpenSearch image version, pass the OpenSearch version:

```
./gradlew clean integrationTest
./gradlew clean integrationTest -Dtests.opensearch.version=3.2.0
```

To use a custom security-enabled OpenSearch image, pass the full image name:

```
./gradlew clean integrationTest -Dtests.opensearch.image=opensearchproject/opensearch:3.2.0
```

To run against an already running cluster, disable the test container and pass the cluster endpoint if it is not `localhost:9200`:

```
./gradlew clean integrationTest -Dtests.opensearch.testcontainers.enabled=false -Dtests.rest.cluster=localhost:9200
```

#### AWS Transport Integration Tests
Expand Down
18 changes: 15 additions & 3 deletions java-client/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,9 @@ tasks.build {
dependsOn("spotlessJavaCheck")
}

val opensearchVersion = "3.5.0-SNAPSHOT"
val opensearchDockerVersion = opensearchVersion.removeSuffix("-SNAPSHOT")

tasks.test {
systemProperty("tests.security.manager", "false")

Expand All @@ -168,6 +171,16 @@ val integrationTest = task<Test>("integrationTest") {
systemProperty("https", System.getProperty("https", "true"))
systemProperty("user", System.getProperty("user", "admin"))
systemProperty("password", System.getProperty("password", "admin"))
systemProperty(
"tests.opensearch.testcontainers.enabled",
System.getProperty("tests.opensearch.testcontainers.enabled", "true")
)
systemProperty(
"tests.opensearch.version",
System.getProperty("tests.opensearch.version", opensearchDockerVersion)
)
System.getProperty("tests.rest.cluster")?.let { systemProperty("tests.rest.cluster", it) }
System.getProperty("tests.opensearch.image")?.let { systemProperty("tests.opensearch.image", it) }
systemProperty("tests.awsSdk2support.domainHost",
System.getProperty("tests.awsSdk2support.domainHost", null))
systemProperty("tests.awsSdk2support.serviceName",
Expand All @@ -176,8 +189,6 @@ val integrationTest = task<Test>("integrationTest") {
System.getProperty("tests.awsSdk2support.domainRegion", "us-east-1"))
}

val opensearchVersion = "3.5.0-SNAPSHOT"

dependencies {
val jacksonVersion = "2.21.2"
val jacksonDatabindVersion = "2.21.2"
Expand Down Expand Up @@ -387,6 +398,7 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) {
testImplementation("org.opensearch.test", "framework", opensearchVersion) {
exclude(group = "org.hamcrest")
}
"java21Implementation"("org.testcontainers:testcontainers:2.0.5")
Comment thread
reta marked this conversation as resolved.
Outdated
}

tasks.named<JavaCompile>("compileJava21Java") {
Expand All @@ -408,4 +420,4 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) {
testClassesDirs += java21.output.classesDirs
classpath = sourceSets["java21"].runtimeClasspath
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package org.opensearch.client.opensearch.integTest;

import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -30,6 +31,7 @@
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.opensearch.Version;
import org.opensearch.client.RestClient;
import org.opensearch.client.RestClientBuilder;
Expand All @@ -45,6 +47,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.test.rest.OpenSearchRestTestCase;

@ThreadLeakFilters(filters = TestcontainersThreadFilter.class)
public abstract class OpenSearchJavaClientTestCase extends OpenSearchRestTestCase implements OpenSearchTransportSupport {
private static final List<String> systemIndices = List.of(
".opensearch-observability",
Expand All @@ -59,6 +62,12 @@ public abstract class OpenSearchJavaClientTestCase extends OpenSearchRestTestCas
private static TreeSet<Version> nodeVersions;
private static List<HttpHost> clusterHosts;

// Superclass setup reads tests.rest.cluster before subclass @Before methods run.
@BeforeClass
public static void startOpenSearchTestContainer() {
OpenSearchTestContainer.startIfNeeded();
}

@Before
public void initJavaClient() throws IOException {
if (javaClient == null) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.client.opensearch.integTest;

import java.time.Duration;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
import org.testcontainers.utility.DockerImageName;

final class OpenSearchTestContainer {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please move under java-client/src/test/java21. Also, we should be using JUnit Jupiter extentions mechanism with proper pre / post lifycycle (see please examples here https://www.baeldung.com/junit-5-registerextension-annotation)

static final String ENABLED_PROPERTY = "tests.opensearch.testcontainers.enabled";
static final String VERSION_PROPERTY = "tests.opensearch.version";
static final String IMAGE_PROPERTY = "tests.opensearch.image";
static final String CLUSTER_PROPERTY = "tests.rest.cluster";
static final String HTTPS_PROPERTY = "https";
static final String USER_PROPERTY = "user";
static final String PASSWORD_PROPERTY = "password";

private static final String DEFAULT_IMAGE = "opensearchproject/opensearch";
private static final String ADMIN_USER = "admin";
private static final String DEFAULT_ADMIN_PASSWORD = "admin";
// OpenSearch 2.12+ rejects the old admin/admin demo password during container startup.
private static final String DEFAULT_INITIAL_ADMIN_PASSWORD = "OSClient-Test-2026-Password!";
private static final int HTTP_PORT = 9200;

private static GenericContainer<?> container;
private static String password;

private OpenSearchTestContainer() {}

static synchronized void startIfNeeded() {
if (hasText(System.getProperty(CLUSTER_PROPERTY)) || !testcontainersEnabled()) {
return;
}

if (container == null) {
String version = System.getProperty(VERSION_PROPERTY);
String imageName = resolveImageName(version);
String passwordCompatibilityVersion = passwordCompatibilityVersion(version, imageName);
password = passwordFor(passwordCompatibilityVersion);
GenericContainer<?> openSearch = createContainer(imageName, passwordCompatibilityVersion, password);
openSearch.start();
container = openSearch;
}

System.setProperty(CLUSTER_PROPERTY, container.getHost() + ":" + container.getMappedPort(HTTP_PORT));
System.setProperty(HTTPS_PROPERTY, "true");
System.setProperty(USER_PROPERTY, ADMIN_USER);
System.setProperty(PASSWORD_PROPERTY, password);
}

private static GenericContainer<?> createContainer(String imageName, String passwordCompatibilityVersion, String adminPassword) {
GenericContainer<?> openSearch = new GenericContainer<>(DockerImageName.parse(imageName)).withExposedPorts(HTTP_PORT)
.withEnv("discovery.type", "single-node")
.withEnv("bootstrap.memory_lock", "true")
.withEnv("cluster.routing.allocation.disk.threshold_enabled", "false")
.waitingFor(
new HttpWaitStrategy().forPort(HTTP_PORT)
.usingTls()
.allowInsecure()
.withBasicCredentials(ADMIN_USER, adminPassword)
.forStatusCode(200)
.withReadTimeout(Duration.ofSeconds(10))
.withStartupTimeout(Duration.ofMinutes(5))
);

if (requiresInitialAdminPassword(passwordCompatibilityVersion)) {
openSearch.withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", adminPassword);
}

return openSearch;
}

private static String resolveImageName(String version) {
String image = System.getProperty(IMAGE_PROPERTY);
if (hasText(image)) {
return image;
}
if (!hasText(version)) {
throw new IllegalStateException("Missing " + VERSION_PROPERTY + " for OpenSearch Testcontainers image");
}
return DEFAULT_IMAGE + ":" + version;
}

private static String passwordFor(String version) {
String configuredPassword = System.getProperty(PASSWORD_PROPERTY);
if (requiresInitialAdminPassword(version)) {
if (!hasText(configuredPassword) || DEFAULT_ADMIN_PASSWORD.equals(configuredPassword)) {
return DEFAULT_INITIAL_ADMIN_PASSWORD;
}
return configuredPassword;
}
return hasText(configuredPassword) ? configuredPassword : DEFAULT_ADMIN_PASSWORD;
}

static boolean requiresInitialAdminPassword(String version) {
if (!hasText(version) || "latest".equalsIgnoreCase(version)) {
return true;
}

String[] components = version.split("[-+]", 2)[0].split("\\.");
Integer major = parseInt(components, 0);
if (major == null) {
return true;
}
if (major > 2) {
return true;
}
if (major < 2) {
return false;
}
Integer minor = parseInt(components, 1);
if (minor == null) {
return true;
}
return minor >= 12;
}

private static Integer parseInt(String[] components, int index) {
if (index >= components.length) {
return null;
}
return parseInt(components[index]);
}

private static Integer parseInt(String component) {
try {
return Integer.parseInt(component);
} catch (NumberFormatException e) {
return null;
}
}

static String passwordCompatibilityVersion(String version, String imageName) {
String imageTag = imageTag(imageName);
if (isCompatibilityVersion(imageTag)) {
return imageTag;
}
return hasText(version) ? version : imageTag;
}

private static boolean isCompatibilityVersion(String version) {
if (!hasText(version)) {
return false;
}
if ("latest".equalsIgnoreCase(version)) {
return true;
}

String[] components = version.split("[-+]", 2)[0].split("\\.");
for (String component : components) {
if (parseInt(component) == null) {
return false;
}
}
return true;
}

private static String imageTag(String imageName) {
int lastSlash = imageName.lastIndexOf('/');
int lastColon = imageName.lastIndexOf(':');
if (lastColon > lastSlash && lastColon < imageName.length() - 1) {
return imageName.substring(lastColon + 1);
}
return null;
}

private static boolean testcontainersEnabled() {
return Boolean.parseBoolean(System.getProperty(ENABLED_PROPERTY, "true"));
}

private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
}
Loading
Loading