diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/README.md index 3565d0e067ea..6138a7b38305 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/README.md +++ b/hadoop-ozone/dist/src/main/compose/upgrade/README.md @@ -100,6 +100,7 @@ Docker compose cluster definitions to be used in upgrade testing are defined in - `SCM`: The name of the SCM container to run robot tests from. - This can be passed as the first argument to `execute_robot_test`. - This allows the same tests to work with and without SCM HA. + - `CLIENT`: The dedicated client container used to run Robot tests in rolling-upgrade HA. ### Testing New Versions diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml index 20c93493a14d..426ea3bbadad 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml @@ -281,6 +281,29 @@ services: - *krb5conf - *ozone-dir - *transformation + upgrade_client: + command: ["tail", "-f", "/dev/null"] + dns_search: . + env_file: + - docker-config + - ../../../common/security.conf + extra_hosts: + - "om1:10.9.0.11" + - "om2:10.9.0.12" + - "om3:10.9.0.13" + - "scm1.org:10.9.0.14" + - "scm2.org:10.9.0.15" + - "scm3.org:10.9.0.16" + image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + hostname: scm + networks: + net: + ipv4_address: 10.9.0.24 + volumes: + - *keytabs + - *krb5conf + - ../../../..:/opt/hadoop + - ../../../..:/opt/ozone networks: net: diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh index b1063db05440..39622ecd21db 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh @@ -24,6 +24,7 @@ set +u source "$TEST_DIR/testlib.sh" export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml" +export CLIENT=upgrade_client export OM_SERVICE_ID=omservice export SECURITY_ENABLED="true" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh index a418a96423a2..c83fc2850144 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh @@ -117,16 +117,18 @@ run_test() { ## @description Generates data on the cluster. ## @param The prefix to use for data generated. -## @param All parameters after the first one are passed directly to the robot command, +## @param The container to run the robot test in. +## @param All remaining parameters are passed directly to the robot command, ## see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options generate() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:2} upgrade/generate.robot + execute_robot_test "${2}" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:3} upgrade/generate.robot } ## @description Validates that data exists on the cluster. ## @param The prefix of the data to be validated. -## @param All parameters after the first one are passed directly to the robot command, +## @param The container to run the robot test in. +## @param All remaining parameters are passed directly to the robot command, ## see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options validate() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:2} upgrade/validate.robot + execute_robot_test "${2}" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:3} upgrade/validate.robot } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh index c2159ca0cd24..e4a419f00df1 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh @@ -21,40 +21,40 @@ source "$TEST_DIR"/testlib.sh with_old_version() { execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot - generate old1 - validate old1 + generate old1 "$SCM" + validate old1 "$SCM" } with_this_version_pre_finalized() { # No check for pre-finalized status here, because the release may not have # added layout features to OM or HDDS. - validate old1 + validate old1 "$SCM" # HDDS-6261: overwrite the same keys intentionally - generate old1 --exclude create-volume-and-bucket + generate old1 "$SCM" --exclude create-volume-and-bucket - generate new1 - validate new1 + generate new1 "$SCM" + validate new1 "$SCM" } with_old_version_downgraded() { execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot - validate old1 - validate new1 + validate old1 "$SCM" + validate new1 "$SCM" - generate old2 - validate old2 + generate old2 "$SCM" + validate old2 "$SCM" # HDDS-6261: overwrite the same keys again to trigger the precondition check # that exists <= 1.1.0 OM - generate old1 --exclude create-volume-and-bucket + generate old1 "$SCM" --exclude create-volume-and-bucket } with_this_version_finalized() { execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot - validate old1 - validate new1 - validate old2 + validate old1 "$SCM" + validate new1 "$SCM" + validate old2 "$SCM" - generate new2 - validate new2 + generate new2 "$SCM" + validate new2 "$SCM" } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh index 5160b4ef384b..e5e693e41b51 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh @@ -20,9 +20,9 @@ source "$TEST_DIR"/testlib.sh ### CALLBACKS ### before_service_restart() { - generate "generate-${SERVICE}" + generate "generate-${SERVICE}" "$CLIENT" } after_service_restart() { - validate "generate-${SERVICE}" + validate "generate-${SERVICE}" "$CLIENT" } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh index 73663dc53614..1621e2b42d90 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh @@ -27,6 +27,7 @@ set -u : "${OZONE_UPGRADE_TO}" : "${TEST_DIR}" : "${SCM}" +: "${CLIENT}" : "${OZONE_CURRENT_VERSION}" set +u @@ -43,17 +44,6 @@ rolling_restart_service() { # Stop service stop_containers "${SERVICE}" - # Check if this SCM container is running, as during a rolling upgrade it does stop-start one-by-one and - # we want to run write/read tests while one service is unavailable. Choose SCM (the container where the generate and - # validate robot tests are running) considering availability. - if [[ "$(docker inspect -f '{{.State.Running}}' "ha-${SCM}-1" 2>/dev/null)" != "true" ]]; then - local fallback_scm - fallback_scm="$(docker-compose --project-directory="$TEST_DIR/compose/ha" config --services | grep scm | grep -v "^${SCM}$" | head -n1)" - if [[ -n "$fallback_scm" ]]; then - export SCM="$fallback_scm" - fi - fi - # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated # TODO find a better solution if [[ ${SERVICE} != "s3g" ]]; then @@ -129,4 +119,4 @@ OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-finalized" # TODO Add validation for pre-finalized state # Sends commands to finalize OM and SCM. -execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot +execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot