From 1a1b21685164f3f18500071a3a6e1b5c38f51fed Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati <115860222+sreejasahithi@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:38:12 +0530 Subject: [PATCH] HDDS-16467. Run S3 smoketests with AWS CLI v1 and v2 --- .../dist/src/main/compose/common/ec-test.sh | 2 +- .../main/compose/ozone-ha/test-haproxy-s3g.sh | 2 +- .../dist/src/main/compose/ozone-ha/test.sh | 2 +- .../ozonesecure-ha/test-haproxy-s3g.sh | 2 +- .../src/main/compose/ozonesecure-ha/test.sh | 2 +- .../main/compose/ozonesecure/test-vault.sh | 2 +- .../dist/src/main/compose/ozonesecure/test.sh | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 32 ++++++++ .../main/smoketest/s3/MultipartUpload.robot | 6 +- .../src/main/smoketest/s3/bucketcreate.robot | 5 +- .../src/main/smoketest/s3/bucketlist.robot | 2 +- .../src/main/smoketest/s3/commonawslib.robot | 77 +++++++++++++------ .../main/smoketest/s3/conditionalput.robot | 14 ++-- .../dist/src/main/smoketest/s3/freon.robot | 23 ++++-- .../src/main/smoketest/s3/objectlist.robot | 2 +- .../src/main/smoketest/s3/presignedurl.robot | 8 +- .../dist/src/main/smoketest/s3/rclone.robot | 4 +- 17 files changed, 128 insertions(+), 59 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh index 2e3552fc969b..105a9f3575dd 100755 --- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh @@ -17,7 +17,7 @@ start_docker_env 5 -execute_robot_test scm -v BUCKET:erasure s3 +execute_s3_smoketests scm -N s3 -v BUCKET:erasure execute_robot_test scm ec/rewrite.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh index 83c1c364a23a..169c00ad9220 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh @@ -32,7 +32,7 @@ start_docker_env exclude="" for bucket in generated; do - execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + execute_s3_smoketests ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" done diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh index c27f14e579ae..0ac466aeee25 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh @@ -40,7 +40,7 @@ execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link exclude="" for bucket in generated; do for layout in OBJECT_STORE LEGACY FILE_SYSTEM_OPTIMIZED; do - execute_robot_test ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude} s3 + execute_s3_smoketests ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude} # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" done diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh index 23d56b22e931..941a6b387f50 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh @@ -38,7 +38,7 @@ execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} exclude="" for bucket in encrypted; do - execute_robot_test recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + execute_s3_smoketests recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude} # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" done diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh index 250c66860f88..be4325e92f92 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh @@ -48,7 +48,7 @@ execute_robot_test s3g basic/links.robot exclude="" for bucket in link; do - execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + execute_s3_smoketests s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" done diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh index 1c6cc3740537..d2adc78eab80 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh @@ -30,4 +30,4 @@ export COMPOSE_FILE=docker-compose.yaml:vault.yaml start_docker_env -execute_robot_test scm s3 +execute_s3_smoketests scm diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index 26983aebea68..03b67bae332f 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -45,7 +45,7 @@ execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket exclude="" for bucket in encrypted; do - execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + execute_s3_smoketests s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" done diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index e3e178a947bd..65e8cd1c7d43 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -304,6 +304,38 @@ execute_robot_test(){ return ${rc} } +## @description Execute the S3 smoketest directory with AWS CLI v1 and v2. +## @param Name of the container in the docker-compose file +## @param robot arguments (same as execute_robot_test, except the test path is always s3) +## If -N PREFIX is given, result names use PREFIX-awscli-v1 and PREFIX-awscli-v2. +execute_s3_smoketests(){ + local container="$1" + shift + local v1_args=() + local v2_args=() + local has_name=false + while [[ $# -gt 0 ]]; do + if [[ "$1" == "-N" && -n "${2:-}" ]]; then + has_name=true + v1_args+=("-N" "$2-awscli-v1") + v2_args+=("-N" "$2-awscli-v2") + shift 2 + else + v1_args+=("$1") + v2_args+=("$1") + shift + fi + done + if [[ "$has_name" == "false" ]]; then + v1_args=("-N" "s3-awscli-v1" "${v1_args[@]}") + v2_args=("-N" "s3-awscli-v2" "${v2_args[@]}") + fi + local -i rc=0 + execute_robot_test "$container" -v AWS_CLI:aws "${v1_args[@]}" s3 || rc=$? + execute_robot_test "$container" -v AWS_CLI:aws2 "${v2_args[@]}" s3 || rc=$? + return ${rc} +} + ## @description Replace OM node order in config reorder_om_nodes() { local c new_order diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index 54e878740733..35d9c2c2dd07 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -67,8 +67,8 @@ Test Multipart Upload Complete With Chunked Transfer Encoding ... is sent with chunked transfer encoding (no Content-Length, as ... e.g. the AWS C++ SDK does with Expect: 100-continue), it must ... not be rejected as an empty part list (MalformedXML). - ${access_key} = Execute aws configure get aws_access_key_id - ${secret_key} = Execute aws configure get aws_secret_access_key + ${access_key} = Execute ${AWS_CLI} configure get aws_access_key_id + ${secret_key} = Execute ${AWS_CLI} configure get aws_secret_access_key ${key} = Set Variable ${PREFIX}/chunkedCompleteKey ${uploadID} = Set Variable ${EMPTY} ${uploadID} = Initiate MPU ${BUCKET} ${key} @@ -315,7 +315,7 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source ${lastModified} = Execute and checkrc echo '${result}' | jq -r '.LastModified' 0 Should contain ${result} ${LastModified} - ${lmDate} = Convert Date ${lastModified} date_format=%a, %d %b %Y %H:%M:%S %Z + ${lmDate} = Parse S3 LastModified ${lastModified} ${afterCreate} = Add Time To Date ${lmDate} 3 seconds Wait Til Date Past ${afterCreate} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot index a121df502b69..c986dde45cf0 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot @@ -115,8 +115,9 @@ Test buckets named like web endpoints ${path} = Create Random File KB 64 FOR ${name} IN conf jmx logs logstream prof prom secret stacks static - Create bucket with name ${name} - Put object to bucket bucket=${name} key=testkey path=${path} + ${bucket} = Set Variable ${PREFIX}-${name} + Create bucket with name ${bucket} + Put object to bucket bucket=${bucket} key=testkey path=${path} END Check bucket ownership verification diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketlist.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketlist.robot index 5e8ed10e9823..f1712e34cf53 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketlist.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketlist.robot @@ -49,7 +49,7 @@ Get bucket info with Ozone Shell to check the owner field List buckets with empty access id [setup] Save AWS access key - Execute aws configure set aws_access_key_id '' + Execute ${AWS_CLI} configure set aws_access_key_id '' ${result} = Execute AWSS3APICli and checkrc list-buckets 255 Should contain ${result} The authorization header you provided is invalid [teardown] Restore AWS access key diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 16bf579f4dfd..a01567908057 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -15,6 +15,7 @@ *** Settings *** Library Collections +Library DateTime Resource ../commonlib.robot Resource ../ozone-lib/shell.robot @@ -27,37 +28,58 @@ ${BUCKET_LAYOUT} OBJECT_STORE ${ENCRYPTION_KEY} key1 ${OZONE_S3_TESTS_SET_UP} ${FALSE} ${OZONE_AWS_ACCESS_KEY_ID} ${EMPTY} +${S3_AWS_ACCESS_KEY_ID} ${EMPTY} +${S3_AWS_SECRET_ACCESS_KEY} ${EMPTY} ${OZONE_S3_ADDRESS_STYLE} path +${AWS_CLI} aws *** Keywords *** +Normalize AWS CLI service error rc + [Arguments] ${expected_error_code} + IF '${AWS_CLI}' == 'aws2' and '${expected_error_code}' == '255' + RETURN 254 + END + RETURN ${expected_error_code} + Execute AWSS3APICli [Arguments] ${command} - ${output} = Execute aws s3api --endpoint-url ${ENDPOINT_URL} ${command} - [return] ${output} + ${output} = Execute ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command} + RETURN ${output} # For possible AWS CLI return codes see: https://docs.aws.amazon.com/cli/latest/topic/return-codes.html Execute AWSS3APICli and checkrc [Arguments] ${command} ${expected_error_code} - ${output} = Execute and checkrc aws s3api --endpoint-url ${ENDPOINT_URL} ${command} ${expected_error_code} - [return] ${output} + ${expected_rc} = Normalize AWS CLI service error rc ${expected_error_code} + ${output} = Execute and checkrc ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command} ${expected_rc} + RETURN ${output} Execute AWSS3APICli and ignore error [Arguments] ${command} - ${output} = Execute And Ignore Error aws s3api --endpoint-url ${ENDPOINT_URL} ${command} - [return] ${output} + ${output} = Execute And Ignore Error ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command} + RETURN ${output} Execute AWSS3Cli [Arguments] ${command} - ${output} = Execute aws s3 --endpoint-url ${ENDPOINT_URL} ${command} - [return] ${output} + ${output} = Execute ${AWS_CLI} s3 --endpoint-url ${ENDPOINT_URL} ${command} + RETURN ${output} Execute AWSS3CliDebug [Arguments] ${command} - ${output} = Execute aws --debug s3 --endpoint ${ENDPOINT_URL} ${command} - [return] ${output} + ${output} = Execute ${AWS_CLI} --debug s3 --endpoint ${ENDPOINT_URL} ${command} + RETURN ${output} + +Parse S3 LastModified + [Arguments] ${lastModified} + ${is_iso}= Evaluate bool(__import__('re').match(r'^\\d{4}-\\d{2}-\\d{2}T', '''${lastModified}''')) + IF ${is_iso} + ${lmDate}= Evaluate __import__('datetime').datetime.fromisoformat('''${lastModified}'''.replace('Z', '+00:00')) + ELSE + ${lmDate}= Convert Date ${lastModified} date_format=%a, %d %b %Y %H:%M:%S %Z + END + RETURN ${lmDate} Install aws cli - ${rc} ${output} = Run And Return Rc And Output which aws + ${rc} ${output} = Run And Return Rc And Output which ${AWS_CLI} Return From Keyword If '${rc}' == '0' ${rc} ${output} = Run And Return Rc And Output which apt-get Run Keyword if '${rc}' == '0' Install aws cli s3 debian @@ -80,6 +102,11 @@ Setup v4 headers Run Keyword if '${SECURITY_ENABLED}' == 'true' Setup secure v4 headers Run Keyword if '${SECURITY_ENABLED}' == 'false' Setup dummy credentials for S3 +Store S3 aws credentials + [Arguments] ${accessKey} ${secret} + Set Suite Variable ${S3_AWS_ACCESS_KEY_ID} ${accessKey} + Set Suite Variable ${S3_AWS_SECRET_ACCESS_KEY} ${secret} + Setup secure v4 headers ${result} = Execute and Ignore error ozone s3 getsecret ${OM_HA_PARAM} ${exists} = Run Keyword And Return Status Should Contain ${result} S3_SECRET_ALREADY_EXISTS @@ -94,25 +121,27 @@ Setup secure v4 headers ${secret} = Get Regexp Matches ${result} (?<=awsSecret=).* ${accessKey} = Set Variable ${accessKey[0]} ${secret} = Set Variable ${secret[0]} - Execute aws configure set default.s3.signature_version s3v4 - Execute aws configure set aws_access_key_id ${accessKey} - Execute aws configure set aws_secret_access_key ${secret} - Execute aws configure set region us-west-1 - Execute aws configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE} - + Execute ${AWS_CLI} configure set default.s3.signature_version s3v4 + Execute ${AWS_CLI} configure set aws_access_key_id ${accessKey} + Execute ${AWS_CLI} configure set aws_secret_access_key ${secret} + Execute ${AWS_CLI} configure set region us-west-1 + Execute ${AWS_CLI} configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE} + Run Keyword Store S3 aws credentials ${accessKey} ${secret} Setup dummy credentials for S3 - Execute aws configure set default.s3.signature_version s3v4 - Execute aws configure set aws_access_key_id dlfknslnfslf - Execute aws configure set aws_secret_access_key dlfknslnfslf - Execute aws configure set region us-west-1 + Execute ${AWS_CLI} configure set default.s3.signature_version s3v4 + Execute ${AWS_CLI} configure set aws_access_key_id dlfknslnfslf + Execute ${AWS_CLI} configure set aws_secret_access_key dlfknslnfslf + Execute ${AWS_CLI} configure set region us-west-1 + Execute ${AWS_CLI} configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE} + Run Keyword Store S3 aws credentials dlfknslnfslf dlfknslnfslf Save AWS access key - ${OZONE_AWS_ACCESS_KEY_ID} = Execute aws configure get aws_access_key_id + ${OZONE_AWS_ACCESS_KEY_ID} = Execute ${AWS_CLI} configure get aws_access_key_id Set Test Variable ${OZONE_AWS_ACCESS_KEY_ID} Restore AWS access key - Execute aws configure set aws_access_key_id ${OZONE_AWS_ACCESS_KEY_ID} + Execute ${AWS_CLI} configure set aws_access_key_id ${OZONE_AWS_ACCESS_KEY_ID} Generate Ozone String ${randStr} = Generate Random String 10 [NUMBERS] @@ -208,7 +237,7 @@ Revoke S3 secrets Get bucket owner [arguments] ${bucket} - ${owner} = Execute aws s3api --endpoint-url ${ENDPOINT_URL} get-bucket-acl --bucket ${bucket} | jq -r .Owner.DisplayName + ${owner} = Execute ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} get-bucket-acl --bucket ${bucket} | jq -r .Owner.DisplayName [return] ${owner} Execute AWSS3APICli using bucket ownership verification diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot index 93bd25a43e2b..a1455c774723 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot @@ -31,24 +31,24 @@ ${BUCKET} generated Conditional Put If-None-Match Star Creates New Key [Documentation] If-None-Match: * should succeed when key does not exist - ${key} = Set Variable condput-ifnonematch-new + ${key} = Set Variable ${PREFIX}-condput-ifnonematch-new Execute echo "test-content" > /tmp/${key} - ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*" + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match '*' Should contain ${result} ETag Conditional Put If-None-Match Star Fails For Existing Key [Documentation] If-None-Match: * should fail with 412 when key already exists - ${key} = Set Variable condput-ifnonematch-existing + ${key} = Set Variable ${PREFIX}-condput-ifnonematch-existing Execute echo "initial-content" > /tmp/${key} ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} Should contain ${result} ETag # Now try again with If-None-Match: * - ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*" + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match '*' Should contain ${result} PreconditionFailed Conditional Put If-Match With Correct ETag Succeeds [Documentation] If-Match with correct ETag should succeed - ${key} = Set Variable condput-ifmatch-success + ${key} = Set Variable ${PREFIX}-condput-ifmatch-success Execute echo "initial-content" > /tmp/${key} ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} Should contain ${result} ETag @@ -66,7 +66,7 @@ Conditional Put If-Match With Correct ETag Succeeds Conditional Put If-Match With Wrong ETag Fails [Documentation] If-Match with wrong ETag should fail with 412 - ${key} = Set Variable condput-ifmatch-fail + ${key} = Set Variable ${PREFIX}-condput-ifmatch-fail Execute echo "initial-content" > /tmp/${key} ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} Should contain ${result} ETag @@ -80,7 +80,7 @@ Conditional Put If-Match With Wrong ETag Fails Conditional Put If-Match On Non-Existent Key Fails [Documentation] If-Match on a key that does not exist should fail with 412 - ${key} = Set Variable condput-ifmatch-nonexistent + ${key} = Set Variable ${PREFIX}-condput-ifmatch-nonexistent Execute echo "test-content" > /tmp/${key} ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match some-etag Should contain ${result} PreconditionFailed diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot index 24ba88bba62f..c10f200b009d 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot @@ -27,12 +27,19 @@ ${ENDPOINT_URL} http://s3g:9878 ${BUCKET} generated *** Keywords *** -# Export access key and secret to the environment +# Export access key and secret to the environment for Freon (AWS SDK v1) Setup aws credentials - ${accessKey} = Execute aws configure get aws_access_key_id - ${secret} = Execute aws configure get aws_secret_access_key - Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret} - Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey} + IF '${AWS_CLI}' == 'aws2' + ${accessKey} = Execute ${AWS_CLI} configure get aws_access_key_id --output text + ${secret} = Execute ${AWS_CLI} configure get aws_secret_access_key --output text + ELSE + ${accessKey} = Execute ${AWS_CLI} configure get aws_access_key_id + ${secret} = Execute ${AWS_CLI} configure get aws_secret_access_key + END + ${accessKey} = Strip String ${accessKey} + ${secret} = Strip String ${secret} + Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey} + Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret} Freon S3BG [arguments] ${prefix}=s3bg ${n}=100 ${threads}=10 ${args}=${EMPTY} @@ -47,12 +54,12 @@ Freon S3KG *** Test Cases *** Run Freon S3BG [Setup] Setup aws credentials - Freon S3BG s3bg-${BUCKET} + Freon S3BG s3bg-${PREFIX}-${BUCKET} Run Freon S3KG [Setup] Setup aws credentials - Freon S3KG s3kg-${BUCKET} + Freon S3KG s3kg-${PREFIX}-${BUCKET} Run Freon S3KG MPU [Setup] Setup aws credentials - Freon S3KG s3kg-mpu-${BUCKET} 10 1 --multi-part-upload --parts=2 --size=5242880 + Freon S3KG s3kg-mpu-${PREFIX}-${BUCKET} 10 1 --multi-part-upload --parts=2 --size=5242880 diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot index 1793a4b6081d..c9d3d6049ccf 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot @@ -33,7 +33,7 @@ Prepare Many Objects In Bucket FOR ${i} IN RANGE ${count} Execute echo "test-${i}" > /tmp/manyfiles/obj-${i} END - Execute aws s3 cp /tmp/manyfiles s3://${BUCKET}/ --recursive --endpoint-url=${ENDPOINT_URL} + Execute ${AWS_CLI} s3 cp /tmp/manyfiles s3://${BUCKET}/ --recursive --endpoint-url=${ENDPOINT_URL} *** Test Cases *** diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/presignedurl.robot b/hadoop-ozone/dist/src/main/smoketest/s3/presignedurl.robot index 8cc4ff6cbf77..21a140e81404 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/presignedurl.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/presignedurl.robot @@ -32,8 +32,8 @@ ${BUCKET} generated Presigned URL PUT Object [Documentation] Test presigned URL PUT object Execute echo "Randomtext" > /tmp/testfile - ${ACCESS_KEY} = Execute aws configure get aws_access_key_id - ${SECRET_ACCESS_KEY} = Execute aws configure get aws_secret_access_key + ${ACCESS_KEY} = Execute ${AWS_CLI} configure get aws_access_key_id + ${SECRET_ACCESS_KEY} = Execute ${AWS_CLI} configure get aws_secret_access_key ${presigned_url}= Generate Presigned Put Object Url ${ACCESS_KEY} ${SECRET_ACCESS_KEY} ${BUCKET} test-presigned-put us-east-1 3600 ${EMPTY} ${ENDPOINT_URL} ${SHA256} = Compute Sha256 File /tmp/testfile ${result} = Execute curl -X PUT -T "/tmp/testfile" -H "x-amz-content-sha256: ${SHA256}" "${presigned_url}" @@ -44,8 +44,8 @@ Presigned URL PUT Object Presigned URL PUT Object using wrong x-amz-content-sha256 [Documentation] Test presigned URL PUT object with wrong x-amz-content-sha256 Execute echo "Randomtext" > /tmp/testfile - ${ACCESS_KEY} = Execute aws configure get aws_access_key_id - ${SECRET_ACCESS_KEY} = Execute aws configure get aws_secret_access_key + ${ACCESS_KEY} = Execute ${AWS_CLI} configure get aws_access_key_id + ${SECRET_ACCESS_KEY} = Execute ${AWS_CLI} configure get aws_secret_access_key ${presigned_url}= Generate Presigned Put Object Url ${ACCESS_KEY} ${SECRET_ACCESS_KEY} ${BUCKET} test-presigned-put-wrong-sha us-east-1 3600 ${EMPTY} ${ENDPOINT_URL} ${result} = Execute curl -X PUT -T "/tmp/testfile" -H "x-amz-content-sha256: wronghash" "${presigned_url}" Should Contain ${result} The provided 'x-amz-content-sha256' header does not match the computed hash. diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/rclone.robot b/hadoop-ozone/dist/src/main/smoketest/s3/rclone.robot index b3405d3acf8a..247c344f7bbe 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/rclone.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/rclone.robot @@ -32,8 +32,8 @@ ${RCLONE_VERBOSE_LEVEL} 2 *** Keywords *** # Export access key and secret to the environment Setup aws credentials - ${accessKey} = Execute aws configure get aws_access_key_id - ${secret} = Execute aws configure get aws_secret_access_key + ${accessKey} = Execute ${AWS_CLI} configure get aws_access_key_id + ${secret} = Execute ${AWS_CLI} configure get aws_secret_access_key Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret} Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey}