HDDS-16467. Run S3 smoketests with AWS CLI v1 and v2 (parallel CI)

.github/workflows/ci.yml

@@ -56,6 +56,7 @@ jobs:
       GITHUB_CONTEXT: ${{ toJson(github) }}
     outputs:
       acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
+      acceptance-matrix: ${{ steps.acceptance-suites.outputs.matrix }}
       acceptance-versions: ${{ steps.acceptance-suites.outputs.versions }}
       basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
       integration-suites: ${{ steps.integration-suites.outputs.suites }}
@@ -321,16 +322,14 @@ jobs:
       needs-ozone-binary-tarball: true
       ratis-args: ${{ inputs.ratis_args }}
       script: acceptance
-      script-args: ${{ matrix.suite }}
+      script-args: ${{ matrix.aws_cli != '' && format('{0} {1}', matrix.suite, matrix.aws_cli) || matrix.suite }}
       sha: ${{ needs.build-info.outputs.sha }}
-      split: ${{ matrix.suite }}
+      split: ${{ matrix.split }}
       timeout-minutes: 150
       with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
     strategy:
       matrix:
-        suite: ${{ fromJson(needs.build-info.outputs.acceptance-suites) }}
-        exclude:
-          - suite: compat-old
+        include: ${{ fromJson(needs.build-info.outputs.acceptance-matrix) }}
       fail-fast: false
 
   # acceptance (compat-old), split by old Ozone version

dev-support/ci/acceptance_suites.sh

@@ -24,7 +24,28 @@ SUITES=$(grep --no-filename -r '^#suite:' hadoop-ozone/dist/src/main/compose \
 VERSIONS=$(grep '^old_versions=' hadoop-ozone/dist/src/main/compose/xcompat/lib.sh \
   | cut -f2 -d'"')
 
+MATRIX=$(
+  export SUITES
+  python3 <<'PY'
+import json
+import os
+
+parallel_s3 = {"HA-unsecure", "HA-secure", "secure", "EC"}
+matrix = []
+for suite in os.environ["SUITES"].split():
+    if suite in ("compat-old", "failing"):
+        continue
+    if suite in parallel_s3:
+        matrix.append({"suite": suite, "aws_cli": "aws", "split": f"{suite}-awscli-v1"})
+        matrix.append({"suite": suite, "aws_cli": "aws2", "split": f"{suite}-awscli-v2"})
+    else:
+        matrix.append({"suite": suite, "aws_cli": "", "split": suite})
+print(json.dumps(matrix))
+PY
+)
+
 initialization::ga_output suites \
     "$(initialization::parameters_to_json ${SUITES})"
 initialization::ga_output versions \
     "$(initialization::parameters_to_json ${VERSIONS})"
+initialization::ga_output matrix "${MATRIX}"

hadoop-ozone/dev-support/checks/acceptance.sh

@@ -30,6 +30,8 @@ OZONE_ROOT=$(pwd -P)
 if [[ "${OZONE_ACCEPTANCE_SUITE}" == "compat-old" ]]; then
   : ${OZONE_OLD_VERSIONS:="${2:-}"}
   export OZONE_OLD_VERSIONS
+else
+  export OZONE_AWS_CLI="${2:-}"
 fi
 
 source "${DIR}/_lib.sh"
@@ -89,7 +91,7 @@ if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
   fi
 fi
 
-export OZONE_ACCEPTANCE_SUITE OZONE_ACCEPTANCE_TEST_TYPE
+export OZONE_ACCEPTANCE_SUITE OZONE_ACCEPTANCE_TEST_TYPE OZONE_AWS_CLI
 
 cd "$DIST_DIR/compose" || exit 1
 ./test-all.sh 2>&1 | tee "${REPORT_DIR}/output.log"

hadoop-ozone/dist/src/main/compose/common/ec-test.sh

@@ -17,7 +17,7 @@
 
 start_docker_env 5
 
-execute_robot_test scm -v BUCKET:erasure s3
+execute_s3_smoketests scm -N s3 -v BUCKET:erasure
 
 execute_robot_test scm ec/rewrite.robot
 

hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh

@@ -32,7 +32,7 @@ start_docker_env
 
 exclude=""
 for bucket in generated; do
-  execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  execute_s3_smoketests ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
   # some tests are independent of the bucket type, only need to be run once
   exclude="--exclude no-bucket-type"
 done

hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh

@@ -40,7 +40,7 @@ execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link
 exclude=""
 for bucket in generated; do
   for layout in OBJECT_STORE LEGACY FILE_SYSTEM_OPTIMIZED; do
-    execute_robot_test ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude} s3
+    execute_s3_smoketests ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude}
     # some tests are independent of the bucket type, only need to be run once
     exclude="--exclude no-bucket-type"
   done

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh

@@ -38,7 +38,7 @@ execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
 
 exclude=""
 for bucket in encrypted; do
-  execute_robot_test recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  execute_s3_smoketests recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
   # some tests are independent of the bucket type, only need to be run once
   exclude="--exclude no-bucket-type"
 done

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh

@@ -48,7 +48,7 @@ execute_robot_test s3g basic/links.robot
 
 exclude=""
 for bucket in link; do
-  execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  execute_s3_smoketests s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
   # some tests are independent of the bucket type, only need to be run once
   exclude="--exclude no-bucket-type"
 done

hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh

@@ -30,4 +30,4 @@ export COMPOSE_FILE=docker-compose.yaml:vault.yaml
 
 start_docker_env
 
-execute_robot_test scm s3
+execute_s3_smoketests scm

hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh

@@ -45,7 +45,7 @@ execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket
 
 exclude=""
 for bucket in encrypted; do
-  execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  execute_s3_smoketests s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
   # some tests are independent of the bucket type, only need to be run once
   exclude="--exclude no-bucket-type"
 done

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -304,6 +304,47 @@ execute_robot_test(){
   return ${rc}
 }
 
+## @description  Execute the S3 smoketest directory with AWS CLI v1 and v2.
+## @param        Name of the container in the docker-compose file
+## @param        robot arguments (same as execute_robot_test, except the test path is always s3)
+##               If -N PREFIX is given, result names use PREFIX-awscli-v1 and PREFIX-awscli-v2.
+##               When OZONE_AWS_CLI is aws or aws2, only that CLI runs (CI parallel jobs).
+execute_s3_smoketests(){
+  local container="$1"
+  shift
+  local v1_args=()
+  local v2_args=()
+  local has_name=false
+  while [[ $# -gt 0 ]]; do
+    if [[ "$1" == "-N" && -n "${2:-}" ]]; then
+      has_name=true
+      v1_args+=("-N" "$2-awscli-v1")
+      v2_args+=("-N" "$2-awscli-v2")
+      shift 2
+    else
+      v1_args+=("$1")
+      v2_args+=("$1")
+      shift
+    fi
+  done
+  if [[ "$has_name" == "false" ]]; then
+    v1_args=("-N" "s3-awscli-v1" "${v1_args[@]}")
+    v2_args=("-N" "s3-awscli-v2" "${v2_args[@]}")
+  fi
+  local -i rc=0
+  if [[ -n "${OZONE_AWS_CLI:-}" ]]; then
+    if [[ "${OZONE_AWS_CLI}" == "aws2" ]]; then
+      execute_robot_test "$container" -v AWS_CLI:aws2 "${v2_args[@]}" s3 || rc=$?
+    else
+      execute_robot_test "$container" -v AWS_CLI:aws "${v1_args[@]}" s3 || rc=$?
+    fi
+    return ${rc}
+  fi
+  execute_robot_test "$container" -v AWS_CLI:aws "${v1_args[@]}" s3 || rc=$?
+  execute_robot_test "$container" -v AWS_CLI:aws2 "${v2_args[@]}" s3 || rc=$?
+  return ${rc}
+}
+
 ## @description Replace OM node order in config
 reorder_om_nodes() {
   local c new_order

hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot

@@ -67,8 +67,8 @@ Test Multipart Upload Complete With Chunked Transfer Encoding
     ...                is sent with chunked transfer encoding (no Content-Length, as
     ...                e.g. the AWS C++ SDK does with Expect: 100-continue), it must
     ...                not be rejected as an empty part list (MalformedXML).
-    ${access_key} =     Execute    aws configure get aws_access_key_id
-    ${secret_key} =     Execute    aws configure get aws_secret_access_key
+    ${access_key} =     Execute    ${AWS_CLI} configure get aws_access_key_id
+    ${secret_key} =     Execute    ${AWS_CLI} configure get aws_secret_access_key
     ${key} =            Set Variable    ${PREFIX}/chunkedCompleteKey
     ${uploadID} =       Set Variable    ${EMPTY}
     ${uploadID} =       Initiate MPU    ${BUCKET}    ${key}
@@ -315,7 +315,7 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince
     ${result} =         Execute AWSS3APICli      head-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source
     ${lastModified} =   Execute and checkrc      echo '${result}' | jq -r '.LastModified'    0
                         Should contain           ${result}    ${LastModified}
-    ${lmDate} =         Convert Date 	 	 ${lastModified}  date_format=%a, %d %b %Y %H:%M:%S %Z
+    ${lmDate} =         Parse S3 LastModified    ${lastModified}
     ${afterCreate} =    Add Time To Date         ${lmDate}  3 seconds
     Wait Til Date Past  ${afterCreate}
 

hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot

@@ -115,8 +115,9 @@ Test buckets named like web endpoints
     ${path} =    Create Random File KB    64
 
     FOR  ${name}   IN    conf    jmx    logs    logstream    prof    prom    secret    stacks    static
-        Create bucket with name    ${name}
-        Put object to bucket    bucket=${name}    key=testkey    path=${path}
+        ${bucket} =    Set Variable    ${PREFIX}-${name}
+        Create bucket with name    ${bucket}
+        Put object to bucket    bucket=${bucket}    key=testkey    path=${path}
     END
 
 Check bucket ownership verification

hadoop-ozone/dist/src/main/smoketest/s3/bucketlist.robot

@@ -49,7 +49,7 @@ Get bucket info with Ozone Shell to check the owner field
 
 List buckets with empty access id
     [setup]             Save AWS access key
-                        Execute     aws configure set aws_access_key_id ''
+                        Execute     ${AWS_CLI} configure set aws_access_key_id ''
     ${result} =         Execute AWSS3APICli and checkrc         list-buckets    255
                         Should contain            ${result}         The authorization header you provided is invalid
     [teardown]          Restore AWS access key

hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot

@@ -15,6 +15,7 @@
 
 *** Settings ***
 Library             Collections
+Library             DateTime
 Resource            ../commonlib.robot
 Resource            ../ozone-lib/shell.robot
 
@@ -27,37 +28,58 @@ ${BUCKET_LAYOUT}               OBJECT_STORE
 ${ENCRYPTION_KEY}              key1
 ${OZONE_S3_TESTS_SET_UP}       ${FALSE}
 ${OZONE_AWS_ACCESS_KEY_ID}     ${EMPTY}
+${S3_AWS_ACCESS_KEY_ID}        ${EMPTY}
+${S3_AWS_SECRET_ACCESS_KEY}    ${EMPTY}
 ${OZONE_S3_ADDRESS_STYLE}      path
+${AWS_CLI}                     aws
 
 *** Keywords ***
+Normalize AWS CLI service error rc
+    [Arguments]    ${expected_error_code}
+    IF    '${AWS_CLI}' == 'aws2' and '${expected_error_code}' == '255'
+        RETURN    254
+    END
+    RETURN    ${expected_error_code}
+
 Execute AWSS3APICli
     [Arguments]       ${command}
-    ${output} =       Execute                    aws s3api --endpoint-url ${ENDPOINT_URL} ${command}
-    [return]          ${output}
+    ${output} =       Execute                    ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command}
+    RETURN            ${output}
 
 # For possible AWS CLI return codes see: https://docs.aws.amazon.com/cli/latest/topic/return-codes.html
 Execute AWSS3APICli and checkrc
     [Arguments]       ${command}                 ${expected_error_code}
-    ${output} =       Execute and checkrc        aws s3api --endpoint-url ${ENDPOINT_URL} ${command}  ${expected_error_code}
-    [return]          ${output}
+    ${expected_rc} =  Normalize AWS CLI service error rc    ${expected_error_code}
+    ${output} =       Execute and checkrc        ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command}  ${expected_rc}
+    RETURN            ${output}
 
 Execute AWSS3APICli and ignore error
     [Arguments]       ${command}
-    ${output} =       Execute And Ignore Error   aws s3api --endpoint-url ${ENDPOINT_URL} ${command}
-    [return]          ${output}
+    ${output} =       Execute And Ignore Error   ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command}
+    RETURN            ${output}
 
 Execute AWSS3Cli
     [Arguments]       ${command}
-    ${output} =       Execute                     aws s3 --endpoint-url ${ENDPOINT_URL} ${command}
-    [return]          ${output}
+    ${output} =       Execute                     ${AWS_CLI} s3 --endpoint-url ${ENDPOINT_URL} ${command}
+    RETURN            ${output}
 
 Execute AWSS3CliDebug
     [Arguments]       ${command}
-    ${output} =       Execute                     aws --debug s3 --endpoint ${ENDPOINT_URL} ${command}
-    [return]          ${output}
+    ${output} =       Execute                     ${AWS_CLI} --debug s3 --endpoint ${ENDPOINT_URL} ${command}
+    RETURN            ${output}
+
+Parse S3 LastModified
+    [Arguments]    ${lastModified}
+    ${is_iso}=    Evaluate    bool(__import__('re').match(r'^\\d{4}-\\d{2}-\\d{2}T', '''${lastModified}'''))
+    IF    ${is_iso}
+        ${lmDate}=    Evaluate    __import__('datetime').datetime.fromisoformat('''${lastModified}'''.replace('Z', '+00:00'))
+    ELSE
+        ${lmDate}=    Convert Date    ${lastModified}    date_format=%a, %d %b %Y %H:%M:%S %Z
+    END
+    RETURN    ${lmDate}
 
 Install aws cli
-    ${rc}              ${output} =                 Run And Return Rc And Output           which aws
+    ${rc}              ${output} =                 Run And Return Rc And Output           which ${AWS_CLI}
     Return From Keyword If    '${rc}' == '0'
     ${rc}              ${output} =                 Run And Return Rc And Output           which apt-get
     Run Keyword if     '${rc}' == '0'              Install aws cli s3 debian
@@ -80,6 +102,11 @@ Setup v4 headers
     Run Keyword if      '${SECURITY_ENABLED}' == 'true'     Setup secure v4 headers
     Run Keyword if      '${SECURITY_ENABLED}' == 'false'    Setup dummy credentials for S3
 
+Store S3 aws credentials
+    [Arguments]    ${accessKey}    ${secret}
+    Set Suite Variable    ${S3_AWS_ACCESS_KEY_ID}    ${accessKey}
+    Set Suite Variable    ${S3_AWS_SECRET_ACCESS_KEY}    ${secret}
+
 Setup secure v4 headers
     ${result} =         Execute and Ignore error             ozone s3 getsecret ${OM_HA_PARAM}
     ${exists} =         Run Keyword And Return Status    Should Contain    ${result}    S3_SECRET_ALREADY_EXISTS
@@ -94,25 +121,27 @@ Setup secure v4 headers
     ${secret} =         Get Regexp Matches         ${result}     (?<=awsSecret=).*
     ${accessKey} =      Set Variable               ${accessKey[0]}
     ${secret} =         Set Variable               ${secret[0]}
-                        Execute                    aws configure set default.s3.signature_version s3v4
-                        Execute                    aws configure set aws_access_key_id ${accessKey}
-                        Execute                    aws configure set aws_secret_access_key ${secret}
-                        Execute                    aws configure set region us-west-1
-                        Execute                    aws configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE}
-
+                        Execute                    ${AWS_CLI} configure set default.s3.signature_version s3v4
+                        Execute                    ${AWS_CLI} configure set aws_access_key_id ${accessKey}
+                        Execute                    ${AWS_CLI} configure set aws_secret_access_key ${secret}
+                        Execute                    ${AWS_CLI} configure set region us-west-1
+                        Execute                    ${AWS_CLI} configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE}
+    Run Keyword    Store S3 aws credentials    ${accessKey}    ${secret}
 
 Setup dummy credentials for S3
-                        Execute                    aws configure set default.s3.signature_version s3v4
-                        Execute                    aws configure set aws_access_key_id dlfknslnfslf
-                        Execute                    aws configure set aws_secret_access_key dlfknslnfslf
-                        Execute                    aws configure set region us-west-1
+                        Execute                    ${AWS_CLI} configure set default.s3.signature_version s3v4
+                        Execute                    ${AWS_CLI} configure set aws_access_key_id dlfknslnfslf
+                        Execute                    ${AWS_CLI} configure set aws_secret_access_key dlfknslnfslf
+                        Execute                    ${AWS_CLI} configure set region us-west-1
+                        Execute                    ${AWS_CLI} configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE}
+    Run Keyword    Store S3 aws credentials    dlfknslnfslf    dlfknslnfslf
 
 Save AWS access key
-    ${OZONE_AWS_ACCESS_KEY_ID} =      Execute     aws configure get aws_access_key_id
+    ${OZONE_AWS_ACCESS_KEY_ID} =      Execute     ${AWS_CLI} configure get aws_access_key_id
     Set Test Variable     ${OZONE_AWS_ACCESS_KEY_ID}
 
 Restore AWS access key
-    Execute    aws configure set aws_access_key_id ${OZONE_AWS_ACCESS_KEY_ID}
+    Execute    ${AWS_CLI} configure set aws_access_key_id ${OZONE_AWS_ACCESS_KEY_ID}
 
 Generate Ozone String
     ${randStr} =         Generate Random String     10  [NUMBERS]
@@ -208,7 +237,7 @@ Revoke S3 secrets
 
 Get bucket owner
     [arguments]    ${bucket}
-    ${owner} =     Execute    aws s3api --endpoint-url ${ENDPOINT_URL} get-bucket-acl --bucket ${bucket} | jq -r .Owner.DisplayName
+    ${owner} =     Execute    ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} get-bucket-acl --bucket ${bucket} | jq -r .Owner.DisplayName
     [return]       ${owner}
 
 Execute AWSS3APICli using bucket ownership verification