HDDS-14896. Use separate container for clients in rolling-upgrade suite

hadoop-ozone/dist/src/main/compose/upgrade/README.md

@@ -100,6 +100,7 @@ Docker compose cluster definitions to be used in upgrade testing are defined in
   - `SCM`: The name of the SCM container to run robot tests from.
     - This can be passed as the first argument to `execute_robot_test`.
     - This allows the same tests to work with and without SCM HA.
+  - `CLIENT`: The dedicated client container used to run Robot tests in rolling-upgrade HA.
 
 ### Testing New Versions
 

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml

@@ -281,6 +281,29 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+  upgrade_client:
+    command: ["tail", "-f", "/dev/null"]
+    dns_search: .
+    env_file:
+      - docker-config
+      - ../../../common/security.conf
+    extra_hosts:
+      - "om1:10.9.0.11"
+      - "om2:10.9.0.12"
+      - "om3:10.9.0.13"
+      - "scm1.org:10.9.0.14"
+      - "scm2.org:10.9.0.15"
+      - "scm3.org:10.9.0.16"
+    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+    hostname: scm
+    networks:
+      net:
+        ipv4_address: 10.9.0.24
+    volumes:
+      - *keytabs
+      - *krb5conf
+      - ../../../..:/opt/hadoop
+      - ../../../..:/opt/ozone
 
 networks:
   net:

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh

@@ -24,6 +24,7 @@ set +u
 source "$TEST_DIR/testlib.sh"
 
 export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml"
+export CLIENT=upgrade_client
 export OM_SERVICE_ID=omservice
 export SECURITY_ENABLED="true"
 

hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh

@@ -117,16 +117,18 @@ run_test() {
 
 ## @description Generates data on the cluster.
 ## @param The prefix to use for data generated.
-## @param All parameters after the first one are passed directly to the robot command,
+## @param The container to run the robot test in.
+## @param All remaining parameters are passed directly to the robot command,
 ##        see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options
 generate() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:2} upgrade/generate.robot
+  execute_robot_test "${2}" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:3} upgrade/generate.robot
 }
 
 ## @description Validates that data exists on the cluster.
 ## @param The prefix of the data to be validated.
-## @param All parameters after the first one are passed directly to the robot command,
+## @param The container to run the robot test in.
+## @param All remaining parameters are passed directly to the robot command,
 ##        see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options
 validate() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:2} upgrade/validate.robot
+  execute_robot_test "${2}" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:3} upgrade/validate.robot
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh

@@ -21,40 +21,40 @@ source "$TEST_DIR"/testlib.sh
 
 with_old_version() {
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
-  generate old1
-  validate old1
+  generate old1 "$SCM"
+  validate old1 "$SCM"
 }
 
 with_this_version_pre_finalized() {
   # No check for pre-finalized status here, because the release may not have
   # added layout features to OM or HDDS.
-  validate old1
+  validate old1 "$SCM"
   # HDDS-6261: overwrite the same keys intentionally
-  generate old1 --exclude create-volume-and-bucket
+  generate old1 "$SCM" --exclude create-volume-and-bucket
 
-  generate new1
-  validate new1
+  generate new1 "$SCM"
+  validate new1 "$SCM"
 }
 
 with_old_version_downgraded() {
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
-  validate old1
-  validate new1
+  validate old1 "$SCM"
+  validate new1 "$SCM"
 
-  generate old2
-  validate old2
+  generate old2 "$SCM"
+  validate old2 "$SCM"
 
   # HDDS-6261: overwrite the same keys again to trigger the precondition check
   # that exists <= 1.1.0 OM
-  generate old1 --exclude create-volume-and-bucket
+  generate old1 "$SCM" --exclude create-volume-and-bucket
 }
 
 with_this_version_finalized() {
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
-  validate old1
-  validate new1
-  validate old2
+  validate old1 "$SCM"
+  validate new1 "$SCM"
+  validate old2 "$SCM"
 
-  generate new2
-  validate new2
+  generate new2 "$SCM"
+  validate new2 "$SCM"
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh

@@ -20,9 +20,9 @@ source "$TEST_DIR"/testlib.sh
 ### CALLBACKS ###
 
 before_service_restart() {
-  generate "generate-${SERVICE}"
+  generate "generate-${SERVICE}" "$CLIENT"
 }
 
 after_service_restart() {
-  validate "generate-${SERVICE}"
+  validate "generate-${SERVICE}" "$CLIENT"
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh

@@ -27,6 +27,7 @@ set -u
 : "${OZONE_UPGRADE_TO}"
 : "${TEST_DIR}"
 : "${SCM}"
+: "${CLIENT}"
 : "${OZONE_CURRENT_VERSION}"
 set +u
 
@@ -43,17 +44,6 @@ rolling_restart_service() {
   # Stop service
   stop_containers "${SERVICE}"
 
-  # Check if this SCM container is running, as during a rolling upgrade it does stop-start one-by-one and
-  # we want to run write/read tests while one service is unavailable. Choose SCM (the container where the generate and
-  # validate robot tests are running) considering availability.
-  if [[ "$(docker inspect -f '{{.State.Running}}' "ha-${SCM}-1" 2>/dev/null)" != "true" ]]; then
-    local fallback_scm
-    fallback_scm="$(docker-compose --project-directory="$TEST_DIR/compose/ha" config --services | grep scm | grep -v "^${SCM}$" | head -n1)"
-    if [[ -n "$fallback_scm" ]]; then
-      export SCM="$fallback_scm"
-    fi
-  fi
-
   # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated
   # TODO find a better solution
   if [[ ${SERVICE} != "s3g" ]]; then
@@ -129,4 +119,4 @@ OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-finalized"
 # TODO Add validation for pre-finalized state
 
 # Sends commands to finalize OM and SCM.
-execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot
+execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot