Skip to content

HDDS-16467. Run S3 smoketests with AWS CLI v1 and v2 #11509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sreejasahithi wants to merge 1 commit into from
+128 −59
Closed

HDDS-16467. Run S3 smoketests with AWS CLI v1 and v2 #11509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/common/ec-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

start_docker_env 5

execute_robot_test scm -v BUCKET:erasure s3
execute_s3_smoketests scm -N s3 -v BUCKET:erasure

execute_robot_test scm ec/rewrite.robot

Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ start_docker_env

exclude=""
for bucket in generated; do
execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
execute_s3_smoketests ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude no-bucket-type"
done
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link
exclude=""
for bucket in generated; do
for layout in OBJECT_STORE LEGACY FILE_SYSTEM_OPTIMIZED; do
execute_robot_test ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude} s3
execute_s3_smoketests ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude}
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude no-bucket-type"
done
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}

exclude=""
for bucket in encrypted; do
execute_robot_test recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
execute_s3_smoketests recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude no-bucket-type"
done
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ execute_robot_test s3g basic/links.robot

exclude=""
for bucket in link; do
execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
execute_s3_smoketests s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude no-bucket-type"
done
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@ export COMPOSE_FILE=docker-compose.yaml:vault.yaml

start_docker_env

execute_robot_test scm s3
execute_s3_smoketests scm
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket

exclude=""
for bucket in encrypted; do
execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
execute_s3_smoketests s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude}
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude no-bucket-type"
done
Expand Down
32 changes: 32 additions & 0 deletions hadoop-ozone/dist/src/main/compose/testlib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,38 @@ execute_robot_test(){
return ${rc}
}

## @description Execute the S3 smoketest directory with AWS CLI v1 and v2.
## @param Name of the container in the docker-compose file
## @param robot arguments (same as execute_robot_test, except the test path is always s3)
## If -N PREFIX is given, result names use PREFIX-awscli-v1 and PREFIX-awscli-v2.
execute_s3_smoketests(){
local container="$1"
shift
local v1_args=()
local v2_args=()
local has_name=false
while [[ $# -gt 0 ]]; do
if [[ "$1" == "-N" && -n "${2:-}" ]]; then
has_name=true
v1_args+=("-N" "$2-awscli-v1")
v2_args+=("-N" "$2-awscli-v2")
shift 2
else
v1_args+=("$1")
v2_args+=("$1")
shift
fi
done
if [[ "$has_name" == "false" ]]; then
v1_args=("-N" "s3-awscli-v1" "${v1_args[@]}")
v2_args=("-N" "s3-awscli-v2" "${v2_args[@]}")
fi
local -i rc=0
execute_robot_test "$container" -v AWS_CLI:aws "${v1_args[@]}" s3 || rc=$?
execute_robot_test "$container" -v AWS_CLI:aws2 "${v2_args[@]}" s3 || rc=$?
return ${rc}
}

## @description Replace OM node order in config
reorder_om_nodes() {
local c new_order
Expand Down
6 changes: 3 additions & 3 deletions hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ Test Multipart Upload Complete With Chunked Transfer Encoding
... is sent with chunked transfer encoding (no Content-Length, as
... e.g. the AWS C++ SDK does with Expect: 100-continue), it must
... not be rejected as an empty part list (MalformedXML).
${access_key} = Execute aws configure get aws_access_key_id
${secret_key} = Execute aws configure get aws_secret_access_key
${access_key} = Execute ${AWS_CLI} configure get aws_access_key_id
${secret_key} = Execute ${AWS_CLI} configure get aws_secret_access_key
${key} = Set Variable ${PREFIX}/chunkedCompleteKey
${uploadID} = Set Variable ${EMPTY}
${uploadID} = Initiate MPU ${BUCKET} ${key}
Expand Down Expand Up @@ -315,7 +315,7 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince
${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source
${lastModified} = Execute and checkrc echo '${result}' | jq -r '.LastModified' 0
Should contain ${result} ${LastModified}
${lmDate} = Convert Date ${lastModified} date_format=%a, %d %b %Y %H:%M:%S %Z
${lmDate} = Parse S3 LastModified ${lastModified}
${afterCreate} = Add Time To Date ${lmDate} 3 seconds
Wait Til Date Past ${afterCreate}

Expand Down
5 changes: 3 additions & 2 deletions hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,9 @@ Test buckets named like web endpoints
${path} = Create Random File KB 64

FOR ${name} IN conf jmx logs logstream prof prom secret stacks static
Create bucket with name ${name}
Put object to bucket bucket=${name} key=testkey path=${path}
${bucket} = Set Variable ${PREFIX}-${name}
Create bucket with name ${bucket}
Put object to bucket bucket=${bucket} key=testkey path=${path}
END

Check bucket ownership verification
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/smoketest/s3/bucketlist.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ Get bucket info with Ozone Shell to check the owner field

List buckets with empty access id
[setup] Save AWS access key
Execute aws configure set aws_access_key_id ''
Execute ${AWS_CLI} configure set aws_access_key_id ''
${result} = Execute AWSS3APICli and checkrc list-buckets 255
Should contain ${result} The authorization header you provided is invalid
[teardown] Restore AWS access key
77 changes: 53 additions & 24 deletions hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

*** Settings ***
Library Collections
Library DateTime
Resource ../commonlib.robot
Resource ../ozone-lib/shell.robot

Expand All @@ -27,37 +28,58 @@ ${BUCKET_LAYOUT} OBJECT_STORE
${ENCRYPTION_KEY} key1
${OZONE_S3_TESTS_SET_UP} ${FALSE}
${OZONE_AWS_ACCESS_KEY_ID} ${EMPTY}
${S3_AWS_ACCESS_KEY_ID} ${EMPTY}
${S3_AWS_SECRET_ACCESS_KEY} ${EMPTY}
${OZONE_S3_ADDRESS_STYLE} path
${AWS_CLI} aws

*** Keywords ***
Normalize AWS CLI service error rc
[Arguments] ${expected_error_code}
IF '${AWS_CLI}' == 'aws2' and '${expected_error_code}' == '255'
RETURN 254
END
RETURN ${expected_error_code}

Execute AWSS3APICli
[Arguments] ${command}
${output} = Execute aws s3api --endpoint-url ${ENDPOINT_URL} ${command}
[return] ${output}
${output} = Execute ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command}
RETURN ${output}

# For possible AWS CLI return codes see: https://docs.aws.amazon.com/cli/latest/topic/return-codes.html
Execute AWSS3APICli and checkrc
[Arguments] ${command} ${expected_error_code}
${output} = Execute and checkrc aws s3api --endpoint-url ${ENDPOINT_URL} ${command} ${expected_error_code}
[return] ${output}
${expected_rc} = Normalize AWS CLI service error rc ${expected_error_code}
${output} = Execute and checkrc ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command} ${expected_rc}
RETURN ${output}

Execute AWSS3APICli and ignore error
[Arguments] ${command}
${output} = Execute And Ignore Error aws s3api --endpoint-url ${ENDPOINT_URL} ${command}
[return] ${output}
${output} = Execute And Ignore Error ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} ${command}
RETURN ${output}

Execute AWSS3Cli
[Arguments] ${command}
${output} = Execute aws s3 --endpoint-url ${ENDPOINT_URL} ${command}
[return] ${output}
${output} = Execute ${AWS_CLI} s3 --endpoint-url ${ENDPOINT_URL} ${command}
RETURN ${output}

Execute AWSS3CliDebug
[Arguments] ${command}
${output} = Execute aws --debug s3 --endpoint ${ENDPOINT_URL} ${command}
[return] ${output}
${output} = Execute ${AWS_CLI} --debug s3 --endpoint ${ENDPOINT_URL} ${command}
RETURN ${output}

Parse S3 LastModified
[Arguments] ${lastModified}
${is_iso}= Evaluate bool(__import__('re').match(r'^\\d{4}-\\d{2}-\\d{2}T', '''${lastModified}'''))
IF ${is_iso}
${lmDate}= Evaluate __import__('datetime').datetime.fromisoformat('''${lastModified}'''.replace('Z', '+00:00'))
ELSE
${lmDate}= Convert Date ${lastModified} date_format=%a, %d %b %Y %H:%M:%S %Z
END
RETURN ${lmDate}

Install aws cli
${rc} ${output} = Run And Return Rc And Output which aws
${rc} ${output} = Run And Return Rc And Output which ${AWS_CLI}
Return From Keyword If '${rc}' == '0'
${rc} ${output} = Run And Return Rc And Output which apt-get
Run Keyword if '${rc}' == '0' Install aws cli s3 debian
Expand All @@ -80,6 +102,11 @@ Setup v4 headers
Run Keyword if '${SECURITY_ENABLED}' == 'true' Setup secure v4 headers
Run Keyword if '${SECURITY_ENABLED}' == 'false' Setup dummy credentials for S3

Store S3 aws credentials
[Arguments] ${accessKey} ${secret}
Set Suite Variable ${S3_AWS_ACCESS_KEY_ID} ${accessKey}
Set Suite Variable ${S3_AWS_SECRET_ACCESS_KEY} ${secret}

Setup secure v4 headers
${result} = Execute and Ignore error ozone s3 getsecret ${OM_HA_PARAM}
${exists} = Run Keyword And Return Status Should Contain ${result} S3_SECRET_ALREADY_EXISTS
Expand All @@ -94,25 +121,27 @@ Setup secure v4 headers
${secret} = Get Regexp Matches ${result} (?<=awsSecret=).*
${accessKey} = Set Variable ${accessKey[0]}
${secret} = Set Variable ${secret[0]}
Execute aws configure set default.s3.signature_version s3v4
Execute aws configure set aws_access_key_id ${accessKey}
Execute aws configure set aws_secret_access_key ${secret}
Execute aws configure set region us-west-1
Execute aws configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE}

Execute ${AWS_CLI} configure set default.s3.signature_version s3v4
Execute ${AWS_CLI} configure set aws_access_key_id ${accessKey}
Execute ${AWS_CLI} configure set aws_secret_access_key ${secret}
Execute ${AWS_CLI} configure set region us-west-1
Execute ${AWS_CLI} configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE}
Run Keyword Store S3 aws credentials ${accessKey} ${secret}

Setup dummy credentials for S3
Execute aws configure set default.s3.signature_version s3v4
Execute aws configure set aws_access_key_id dlfknslnfslf
Execute aws configure set aws_secret_access_key dlfknslnfslf
Execute aws configure set region us-west-1
Execute ${AWS_CLI} configure set default.s3.signature_version s3v4
Execute ${AWS_CLI} configure set aws_access_key_id dlfknslnfslf
Execute ${AWS_CLI} configure set aws_secret_access_key dlfknslnfslf
Execute ${AWS_CLI} configure set region us-west-1
Execute ${AWS_CLI} configure set default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE}
Run Keyword Store S3 aws credentials dlfknslnfslf dlfknslnfslf

Save AWS access key
${OZONE_AWS_ACCESS_KEY_ID} = Execute aws configure get aws_access_key_id
${OZONE_AWS_ACCESS_KEY_ID} = Execute ${AWS_CLI} configure get aws_access_key_id
Set Test Variable ${OZONE_AWS_ACCESS_KEY_ID}

Restore AWS access key
Execute aws configure set aws_access_key_id ${OZONE_AWS_ACCESS_KEY_ID}
Execute ${AWS_CLI} configure set aws_access_key_id ${OZONE_AWS_ACCESS_KEY_ID}

Generate Ozone String
${randStr} = Generate Random String 10 [NUMBERS]
Expand Down Expand Up @@ -208,7 +237,7 @@ Revoke S3 secrets

Get bucket owner
[arguments] ${bucket}
${owner} = Execute aws s3api --endpoint-url ${ENDPOINT_URL} get-bucket-acl --bucket ${bucket} | jq -r .Owner.DisplayName
${owner} = Execute ${AWS_CLI} s3api --endpoint-url ${ENDPOINT_URL} get-bucket-acl --bucket ${bucket} | jq -r .Owner.DisplayName
[return] ${owner}

Execute AWSS3APICli using bucket ownership verification
Expand Down
14 changes: 7 additions & 7 deletions hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,24 @@ ${BUCKET} generated

Conditional Put If-None-Match Star Creates New Key
[Documentation] If-None-Match: * should succeed when key does not exist
${key} = Set Variable condput-ifnonematch-new
${key} = Set Variable ${PREFIX}-condput-ifnonematch-new
Execute echo "test-content" > /tmp/${key}
${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*"
${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match '*'
Should contain ${result} ETag

Conditional Put If-None-Match Star Fails For Existing Key
[Documentation] If-None-Match: * should fail with 412 when key already exists
${key} = Set Variable condput-ifnonematch-existing
${key} = Set Variable ${PREFIX}-condput-ifnonematch-existing
Execute echo "initial-content" > /tmp/${key}
${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}
Should contain ${result} ETag
# Now try again with If-None-Match: *
${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*"
${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match '*'
Should contain ${result} PreconditionFailed

Conditional Put If-Match With Correct ETag Succeeds
[Documentation] If-Match with correct ETag should succeed
${key} = Set Variable condput-ifmatch-success
${key} = Set Variable ${PREFIX}-condput-ifmatch-success
Execute echo "initial-content" > /tmp/${key}
${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}
Should contain ${result} ETag
Expand All @@ -66,7 +66,7 @@ Conditional Put If-Match With Correct ETag Succeeds

Conditional Put If-Match With Wrong ETag Fails
[Documentation] If-Match with wrong ETag should fail with 412
${key} = Set Variable condput-ifmatch-fail
${key} = Set Variable ${PREFIX}-condput-ifmatch-fail
Execute echo "initial-content" > /tmp/${key}
${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}
Should contain ${result} ETag
Expand All @@ -80,7 +80,7 @@ Conditional Put If-Match With Wrong ETag Fails

Conditional Put If-Match On Non-Existent Key Fails
[Documentation] If-Match on a key that does not exist should fail with 412
${key} = Set Variable condput-ifmatch-nonexistent
${key} = Set Variable ${PREFIX}-condput-ifmatch-nonexistent
Execute echo "test-content" > /tmp/${key}
${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match some-etag
Should contain ${result} PreconditionFailed
Expand Down
23 changes: 15 additions & 8 deletions hadoop-ozone/dist/src/main/smoketest/s3/freon.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,19 @@ ${ENDPOINT_URL} http://s3g:9878
${BUCKET} generated

*** Keywords ***
# Export access key and secret to the environment
# Export access key and secret to the environment for Freon (AWS SDK v1)
Setup aws credentials
${accessKey} = Execute aws configure get aws_access_key_id
${secret} = Execute aws configure get aws_secret_access_key
Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret}
Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey}
IF '${AWS_CLI}' == 'aws2'
${accessKey} = Execute ${AWS_CLI} configure get aws_access_key_id --output text
${secret} = Execute ${AWS_CLI} configure get aws_secret_access_key --output text
ELSE
${accessKey} = Execute ${AWS_CLI} configure get aws_access_key_id
${secret} = Execute ${AWS_CLI} configure get aws_secret_access_key
END
${accessKey} = Strip String ${accessKey}
${secret} = Strip String ${secret}
Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey}
Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret}

Freon S3BG
[arguments] ${prefix}=s3bg ${n}=100 ${threads}=10 ${args}=${EMPTY}
Expand All @@ -47,12 +54,12 @@ Freon S3KG
*** Test Cases ***
Run Freon S3BG
[Setup] Setup aws credentials
Freon S3BG s3bg-${BUCKET}
Freon S3BG s3bg-${PREFIX}-${BUCKET}

Run Freon S3KG
[Setup] Setup aws credentials
Freon S3KG s3kg-${BUCKET}
Freon S3KG s3kg-${PREFIX}-${BUCKET}

Run Freon S3KG MPU
[Setup] Setup aws credentials
Freon S3KG s3kg-mpu-${BUCKET} 10 1 --multi-part-upload --parts=2 --size=5242880
Freon S3KG s3kg-mpu-${PREFIX}-${BUCKET} 10 1 --multi-part-upload --parts=2 --size=5242880
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ Prepare Many Objects In Bucket
FOR ${i} IN RANGE ${count}
Execute echo "test-${i}" > /tmp/manyfiles/obj-${i}
END
Execute aws s3 cp /tmp/manyfiles s3://${BUCKET}/ --recursive --endpoint-url=${ENDPOINT_URL}
Execute ${AWS_CLI} s3 cp /tmp/manyfiles s3://${BUCKET}/ --recursive --endpoint-url=${ENDPOINT_URL}

*** Test Cases ***

Expand Down
Loading