diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index 5dab10019565..3f27f2610673 100644 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@ -266,15 +266,10 @@ Apache License 2.0 ===================== ch.qos.reload4j:reload4j - com.amazonaws:aws-java-sdk-core - com.amazonaws:aws-java-sdk-kms - com.amazonaws:aws-java-sdk-s3 - com.amazonaws:jmespath-java com.carrotsearch:hppc com.fasterxml.jackson.core:jackson-annotations com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.dataformat:jackson-dataformat-cbor com.fasterxml.jackson.dataformat:jackson-dataformat-xml com.fasterxml.jackson.datatype:jackson-datatype-jsr310 com.fasterxml.jackson.jaxrs:jackson-jaxrs-base diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 886623bde38f..661b9c065cfc 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -12,9 +12,6 @@ share/ozone/lib/asm-util.jar share/ozone/lib/asm.jar share/ozone/lib/aspectjrt.jar share/ozone/lib/avro.jar -share/ozone/lib/aws-java-sdk-core.jar -share/ozone/lib/aws-java-sdk-kms.jar -share/ozone/lib/aws-java-sdk-s3.jar share/ozone/lib/bcpkix-jdk18on.jar share/ozone/lib/bcprov-jdk18on.jar share/ozone/lib/bcutil-jdk18on.jar @@ -97,7 +94,6 @@ share/ozone/lib/j2objc-annotations.jar share/ozone/lib/jackson-annotations.jar share/ozone/lib/jackson-core.jar share/ozone/lib/jackson-databind.jar -share/ozone/lib/jackson-dataformat-cbor.jar share/ozone/lib/jackson-dataformat-xml.jar share/ozone/lib/jackson-datatype-jsr310.jar share/ozone/lib/jackson-jaxrs-base.jar @@ -161,7 +157,6 @@ share/ozone/lib/jgrapht-ext.jar share/ozone/lib/jgraphx.jar share/ozone/lib/jheaps.jar share/ozone/lib/jline.jar -share/ozone/lib/jmespath-java.jar share/ozone/lib/jnr-a64asm.jar share/ozone/lib/jnr-constants.jar share/ozone/lib/jnr-ffi.jar diff --git a/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot b/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot index 256a9c3e8051..530aa2f55a4a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot +++ b/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot @@ -37,12 +37,6 @@ Setup headers Kinit test user testuser testuser.keytab Setup secure v4 headers -Setup aws credentials - ${accessKey} = Execute aws configure get aws_access_key_id - ${secret} = Execute aws configure get aws_secret_access_key - Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret} - Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey} - Verify endpoint is up [arguments] ${url} Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit HTTP user @@ -52,10 +46,16 @@ Verify endpoint is up Setup bucket1 Execute AWSS3APICli create-bucket --bucket bucket1 -Freon s3kg - [arguments] ${prefix}=s3bg ${n}=1000 ${threads}=10 ${args}=${EMPTY} - ${result} = Execute ozone freon s3kg -e ${S3_ENDPOINT_URL} -t ${threads} -n ${n} -p ${prefix} ${args} - Should contain ${result} Successful executions: ${n} +Generate S3 load + [arguments] ${n}=100 + # one aws process uploads all the files, several at a time + ${dir} = Execute mktemp -d + Execute for i in $(seq 1 ${n}); do echo "$i" > ${dir}/key-$i; done + ${result} = Execute AWSS3Cli cp --recursive ${dir} s3://bucket1/fcq/ + Should contain ${result} upload: + # count the objects: the CLI merges the pages of the listing, which drops KeyCount + ${count} = Execute AWSS3APICli list-objects-v2 --bucket bucket1 --prefix fcq/ --query 'length(Contents)' + Should Be Equal As Integers ${count} ${n} Check metrics caller ${result} = Execute curl --negotiate -u : -LSs ${OM_JMX_ENDPOINT} | sed -n '/${DECAY_RPC_SCHEDULER_METRICS_NAME}/,/}/p' | grep 'Caller(' @@ -65,9 +65,6 @@ Check metrics caller Test setup headers Setup headers -Test setup credentials - Setup aws credentials - Test create bucket 1 Setup bucket1 @@ -78,8 +75,8 @@ Test metrics are registered ${result} = Execute curl --negotiate -u : -LSs ${OM_JMX_ENDPOINT} | grep ${DECAY_RPC_SCHEDULER_METRICS_NAME} Should contain ${result} ${DECAY_RPC_SCHEDULER_METRICS_NAME} -Run freon s3kg - Freon s3kg +Run S3 load + Generate S3 load Test caller from metrics Check metrics caller diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot deleted file mode 100644 index 24ba88bba62f..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation S3 gateway test with aws cli -Library OperatingSystem -Library String -Library BuiltIn -Resource ./commonawslib.robot -Suite Setup Setup s3 tests -Default Tags no-bucket-type - -*** Variables *** -${ENDPOINT_URL} http://s3g:9878 -${BUCKET} generated - -*** Keywords *** -# Export access key and secret to the environment -Setup aws credentials - ${accessKey} = Execute aws configure get aws_access_key_id - ${secret} = Execute aws configure get aws_secret_access_key - Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret} - Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey} - -Freon S3BG - [arguments] ${prefix}=s3bg ${n}=100 ${threads}=10 ${args}=${EMPTY} - ${result} = Execute ozone freon s3bg -e ${ENDPOINT_URL} -t ${threads} -n ${n} -p ${prefix} ${args} - Should contain ${result} Successful executions: ${n} - -Freon S3KG - [arguments] ${prefix}=s3kg ${n}=100 ${threads}=10 ${args}=${EMPTY} - ${result} = Execute ozone freon s3kg -e ${ENDPOINT_URL} -t ${threads} -n ${n} -p ${prefix} --bucket ${BUCKET} ${args} - Should contain ${result} Successful executions: ${n} - -*** Test Cases *** -Run Freon S3BG - [Setup] Setup aws credentials - Freon S3BG s3bg-${BUCKET} - -Run Freon S3KG - [Setup] Setup aws credentials - Freon S3KG s3kg-${BUCKET} - -Run Freon S3KG MPU - [Setup] Setup aws credentials - Freon S3KG s3kg-mpu-${BUCKET} 10 1 --multi-part-upload --parts=2 --size=5242880 diff --git a/hadoop-ozone/dist/src/shell/conf/log4j.properties b/hadoop-ozone/dist/src/shell/conf/log4j.properties index 5ed63ffd1ee5..eaa4f1343150 100644 --- a/hadoop-ozone/dist/src/shell/conf/log4j.properties +++ b/hadoop-ozone/dist/src/shell/conf/log4j.properties @@ -167,9 +167,7 @@ log4j.logger.http.requests=${ozone.http.request.logger} #log4j.logger.http.requests.webhdfs=INFO,HttpAccess # Custom Logging levels -# AWS SDK & S3A FileSystem -#log4j.logger.com.amazonaws=ERROR -log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR +# S3A FileSystem #log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN # Log levels of third-party libraries diff --git a/hadoop-ozone/freon/pom.xml b/hadoop-ozone/freon/pom.xml index aed70f18d5f4..7289cf2a202b 100644 --- a/hadoop-ozone/freon/pom.xml +++ b/hadoop-ozone/freon/pom.xml @@ -31,14 +31,6 @@ - - com.amazonaws - aws-java-sdk-core - - - com.amazonaws - aws-java-sdk-s3 - com.fasterxml.jackson.core jackson-annotations diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextRequestHandler.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextRequestHandler.java deleted file mode 100644 index a5f40c6412b0..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextRequestHandler.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import com.amazonaws.Request; -import com.amazonaws.handlers.RequestHandler2; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.Context; - -/** - * Adds W3C trace context headers to each outgoing S3 request so the S3 Gateway - * can attach its spans to the Freon task span, using {@link W3CTraceContextPropagator}. - */ -public final class FreonS3TraceContextRequestHandler extends RequestHandler2 { - - @Override - public void beforeRequest(Request request) { - if (!Span.current().getSpanContext().isValid()) { - return; - } - W3CTraceContextPropagator.getInstance().inject( - Context.current(), - request, - (carrier, key, value) -> carrier.addHeader(key, value)); - } -} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java deleted file mode 100644 index f3ebccce24f8..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import com.codahale.metrics.Timer; -import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.kohsuke.MetaInfServices; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine.Command; - -/** - * Generate buckets via the s3 interface. - * - * For a secure cluster, - * $> init user keytab - * $> kinit -kt /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM - * $> eval $(ozone s3 getsecret -e) - * for getting and exporting access_key_id and secret_access_key - * to freon shell test environment - * secret access key. - */ -@Command(name = "s3bg", - aliases = "s3-bucket-generator", - description = "Create buckets via the s3 interface.%n" + - "Command requires AWS authentication environment " + - "variables set, AWS_ACCESS_KEY_ID and " + - "AWS_SECRET_ACCESS_KEY", - versionProvider = HddsVersionProvider.class, - mixinStandardHelpOptions = true, - showDefaultValues = true) -@MetaInfServices(FreonSubcommand.class) -public class S3BucketGenerator extends S3EntityGenerator - implements Callable { - - private static final Logger LOG = - LoggerFactory.getLogger(S3BucketGenerator.class); - - private Timer timer; - - @Override - public Void call() throws Exception { - s3ClientInit(); - - timer = getMetrics().timer("bucket-create"); - - runTests(this::createBucket); - - return null; - } - - private void createBucket(long bucketNum) throws Exception { - String bName = getPrefix() + bucketNum; - timer.time(() -> { - getS3().createBucket(bName); - return null; - }); - } -} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java deleted file mode 100644 index bc8660489876..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import picocli.CommandLine; - -/** - * Initiliazing common aspects of S3BucketGenerator and S3KeyGenerator. - */ -public class S3EntityGenerator extends BaseFreonGenerator { - @CommandLine.Option(names = {"-e", "--endpoint"}, - description = "S3 HTTP endpoint", - defaultValue = "http://localhost:9878") - private String endpoint; - - private AmazonS3 s3; - - protected void s3ClientInit() { - init(); - - AmazonS3ClientBuilder amazonS3ClientBuilder = - AmazonS3ClientBuilder.standard() - .withCredentials(new EnvironmentVariableCredentialsProvider()); - - if (!endpoint.isEmpty()) { - amazonS3ClientBuilder - .withPathStyleAccessEnabled(true) - .withEndpointConfiguration( - new EndpointConfiguration(endpoint, "us-east-1")); - } else { - amazonS3ClientBuilder.withRegion(Regions.DEFAULT_REGION); - } - - amazonS3ClientBuilder.withRequestHandlers(new FreonS3TraceContextRequestHandler()); - s3 = amazonS3ClientBuilder.build(); - } - - public String getEndpoint() { - return endpoint; - } - - public AmazonS3 getS3() { - return s3; - } -} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java deleted file mode 100644 index 05bb75100192..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_MULTIPART_MIN_SIZE; - -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; -import com.codahale.metrics.Timer; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.kohsuke.MetaInfServices; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - -/** - * Generate random keys via the s3 interface. - */ -@Command(name = "s3kg", - aliases = "s3-key-generator", - description = "Create random keys via the s3 interface." + - "Command requires AWS authentication environment " + - "variables set, AWS_ACCESS_KEY_ID and " + - "AWS_SECRET_ACCESS_KEY", - versionProvider = HddsVersionProvider.class, - mixinStandardHelpOptions = true, - showDefaultValues = true) -@MetaInfServices(FreonSubcommand.class) -@SuppressWarnings("java:S2245") // no need for secure random -public class S3KeyGenerator extends S3EntityGenerator - implements Callable { - - private static final Logger LOG = - LoggerFactory.getLogger(S3KeyGenerator.class); - - @Option(names = {"-b", "--bucket"}, - description = - "Name of the (S3!) bucket which contains the test data.", - defaultValue = "bucket1") - private String bucketName; - - @Option(names = {"-s", "--size"}, - description = "Size of the generated key (in bytes) or size of one " - + "multipart upload part (in case of multipart upload)", - defaultValue = "10240") - private int fileSize; - - @Option(names = {"--multi-part-upload"}, - description = "Use multi-part upload", - defaultValue = "false") - private boolean multiPart; - - @Option(names = {"--parts"}, - description = "Number of parts for multipart upload (final size = " - + "--size * --parts)", - defaultValue = "10") - private int numberOfParts; - - private Timer timer; - - private String content; - - @Override - public Void call() throws Exception { - - if (multiPart && fileSize < OM_MULTIPART_MIN_SIZE) { - throw new IllegalArgumentException( - "Size of multipart upload parts should be at least 5MB (5242880)"); - } - - s3ClientInit(); - - content = RandomStringUtils.secure().nextAscii(fileSize); - - timer = getMetrics().timer("key-create"); - - System.setProperty("com.amazonaws.services.s3.disablePutObjectMD5Validation", "true"); - runTests(this::createKey); - - return null; - } - - private void createKey(long counter) throws Exception { - timer.time(() -> { - if (multiPart) { - - final String keyName = generateObjectName(counter); - final InitiateMultipartUploadRequest initiateRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - - final InitiateMultipartUploadResult initiateMultipartUploadResult = - getS3().initiateMultipartUpload(initiateRequest); - final String uploadId = initiateMultipartUploadResult.getUploadId(); - - List parts = new ArrayList<>(); - for (int i = 1; i <= numberOfParts; i++) { - - final UploadPartRequest uploadPartRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withPartNumber(i) - .withLastPart(i == numberOfParts) - .withUploadId(uploadId) - .withPartSize(fileSize) - .withInputStream(new ByteArrayInputStream(content.getBytes( - StandardCharsets.UTF_8))); - - final UploadPartResult uploadPartResult = - getS3().uploadPart(uploadPartRequest); - parts.add(uploadPartResult.getPartETag()); - } - - getS3().completeMultipartUpload( - new CompleteMultipartUploadRequest(bucketName, keyName, uploadId, - parts)); - - } else { - getS3().putObject(bucketName, generateObjectName(counter), - content); - } - - return null; - }); - } -} diff --git a/hadoop-ozone/integration-test-s3/pom.xml b/hadoop-ozone/integration-test-s3/pom.xml index 1962dbd00dab..9ed561e90fde 100644 --- a/hadoop-ozone/integration-test-s3/pom.xml +++ b/hadoop-ozone/integration-test-s3/pom.xml @@ -26,16 +26,6 @@ Apache Ozone Integration Tests with S3 Gateway - - com.amazonaws - aws-java-sdk-core - test - - - com.amazonaws - aws-java-sdk-s3 - test - com.google.guava guava diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java index 9ad26f180f6f..3186b97b97cf 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java @@ -23,14 +23,6 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import java.net.URI; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.http.HttpConfig; @@ -61,58 +53,6 @@ public S3ClientFactory(OzoneConfiguration conf) { this.conf = conf; } - /** - * Creates an AmazonS3 client (AWS SDK V1) with path style access enabled. - * - * @return AmazonS3 client - */ - public AmazonS3 createS3Client() { - return createS3Client(true); - } - - /** - * Creates an AmazonS3 client (AWS SDK V1). - * - * @param enablePathStyle whether to enable path style access - * @return AmazonS3 client - */ - public AmazonS3 createS3Client(boolean enablePathStyle) { - final String accessKey = "user"; - final String secretKey = "password"; - final Regions region = Regions.DEFAULT_REGION; - - final String protocol; - final HttpConfig.Policy webPolicy = getHttpPolicy(conf); - String host; - - if (webPolicy.isHttpsEnabled()) { - // TODO: Currently HTTPS is disabled in the test, we can add HTTPS - // integration in the future - protocol = HTTPS_SCHEME; - host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY); - } else { - protocol = HTTP_SCHEME; - host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY); - } - - String endpoint = protocol + "://" + host; - - AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider( - new BasicAWSCredentials(accessKey, secretKey)); - - ClientConfiguration clientConfiguration = new ClientConfiguration(); - LOG.info("S3 Endpoint is {}", endpoint); - - return AmazonS3ClientBuilder.standard() - .withPathStyleAccessEnabled(enablePathStyle) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration( - endpoint, region.getName())) - .withClientConfiguration(clientConfiguration) - .withCredentials(credentials) - .build(); - } - /** * Creates an S3Client (AWS SDK V2) with path style access enabled. * diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java index 7074e469b7c0..908e0b462fe6 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java @@ -23,7 +23,6 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.s3.MultiS3GatewayService; import org.apache.hadoop.ozone.s3.S3GatewayService; -import org.apache.hadoop.ozone.s3.awssdk.v1.AbstractS3SDKV1Tests; import org.apache.hadoop.ozone.s3.awssdk.v2.AbstractS3SDKV2Tests; import org.apache.ozone.test.ClusterForTests; import org.junit.jupiter.api.AfterAll; @@ -62,14 +61,6 @@ protected MiniOzoneCluster createCluster() throws Exception { .build(); } - @Nested - class V1 extends AbstractS3SDKV1Tests { - @Override - public MiniOzoneCluster cluster() { - return getCluster(); - } - } - @Nested class V2 extends AbstractS3SDKV2Tests { @Override diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java deleted file mode 100644 index 74d9be18e9d0..000000000000 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ /dev/null @@ -1,3167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.s3.awssdk.v1; - -import static org.apache.hadoop.ozone.OzoneConsts.MB; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.getOmKeyModificationTime; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.getOmPartModificationTime; -import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; -import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.AmazonServiceException.ErrorType; -import com.amazonaws.HttpMethod; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.Headers; -import com.amazonaws.services.s3.model.AbortIncompleteMultipartUpload; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.AccessControlList; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; -import com.amazonaws.services.s3.model.BucketTaggingConfiguration; -import com.amazonaws.services.s3.model.CanonicalGrantee; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.CopyPartRequest; -import com.amazonaws.services.s3.model.CopyPartResult; -import com.amazonaws.services.s3.model.CreateBucketRequest; -import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest; -import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; -import com.amazonaws.services.s3.model.GetBucketAccelerateConfigurationRequest; -import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest; -import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest; -import com.amazonaws.services.s3.model.GetObjectAclRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.GetObjectTaggingRequest; -import com.amazonaws.services.s3.model.GetObjectTaggingResult; -import com.amazonaws.services.s3.model.Grantee; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ListBucketsPaginatedRequest; -import com.amazonaws.services.s3.model.ListBucketsPaginatedResult; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ListPartsRequest; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.MultipartUploadListing; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.ObjectTagging; -import com.amazonaws.services.s3.model.Owner; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PartListing; -import com.amazonaws.services.s3.model.PartSummary; -import com.amazonaws.services.s3.model.Permission; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.model.SetBucketLifecycleConfigurationRequest; -import com.amazonaws.services.s3.model.SetBucketTaggingConfigurationRequest; -import com.amazonaws.services.s3.model.SetObjectAclRequest; -import com.amazonaws.services.s3.model.SetObjectTaggingRequest; -import com.amazonaws.services.s3.model.Tag; -import com.amazonaws.services.s3.model.TagSet; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; -import com.amazonaws.services.s3.transfer.model.UploadResult; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.MessageDigest; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import javax.xml.bind.DatatypeConverter; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.hdds.client.OzoneQuota; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.client.BucketArgs; -import org.apache.hadoop.ozone.client.ObjectStore; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; -import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; -import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.service.KeyLifecycleService; -import org.apache.hadoop.ozone.s3.S3ClientFactory; -import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils; -import org.apache.hadoop.ozone.s3.endpoint.S3Owner; -import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; -import org.apache.hadoop.ozone.s3.util.S3Consts; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.ozone.test.ClusterForTests; -import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.OzoneTestBase; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.function.Executable; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; - -/** - * This is an abstract class to test the AWS Java S3 SDK operations. - * This class should be extended for OM standalone and OM HA (Ratis) cluster setup. - * - * The test scenarios are adapted from - * - https://github.com/awsdocs/aws-doc-sdk-examples/tree/main/java/example_code/s3/ - * - https://github.com/ceph/s3-tests - * - */ -@TestMethodOrder(MethodOrderer.MethodName.class) -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public abstract class AbstractS3SDKV1Tests extends OzoneTestBase implements ClusterForTests.TestCase { - - // server-side limitation - private static final int MAX_UPLOADS_LIMIT = 1000; - private static final String READ_CONSISTENCY_HEADER = - "x-ozone-read-consistency"; - private static final String LOCAL_LEASE_LOG_LIMIT_HEADER = - "x-ozone-local-lease-log-limit"; - - /** - * There are still some unsupported S3 operations. - * Current unsupported S3 operations (non-exhaustive): - * - Cross Region Replication (CrossRegionReplication.java) - * - Versioned enabled buckets - * - DeleteObjectVersionEnabledBucket.java - * - DeleteMultipleObjectsVersionEnabledBucket.java - * - ListKeysVersioningEnabledBucket.java - * - Website configurations - * - WebsiteConfiguration.java - * - SetWebsiteConfiguration.java - * - GetWebsiteConfiguration.java - * - DeleteWebsiteConfiguration.java - * - S3 Event Notifications - * - EnableNotificationOnABucket.java - * - Object tags - * - GetObjectTags.java - * - GetObjectTags2.java - * - Bucket policy - * - SetBucketPolicy.java - * - GetBucketPolicy.java - * - DeleteBucketPolicy.java - * - Bucket lifecycle configuration - * - LifecycleConfiguration.java - * - Canned Bucket ACL - * - CreateBucketWithACL.java - * - Object ACL - * - SetAcl.java - * - ModifyACLExistingObject.java - * - GetAcl.java - * - S3 Encryption - * - S3Encrypt.java - * - S3EncryptV2.java - * - Client-side encryption - * - S3ClientSideEncryptionAsymmetricMasterKey.java - * - S3ClientSideEncryptionSymMasterKey.java - * - Server-side encryption - * - SpecifyServerSideEncryption.ajva - * - ServerSideEncryptionCopyObjectUsingHLWithSSEC.java - * - ServerSideEncryptionUsingClientSideEncryptionKey.java - * - Dual stack endpoints - * - DualStackEndpoints.java - * - Transfer acceleration - * - TransferAcceleration.java - * - Temp credentials - * - MakingRequestsWithFederatedTempCredentials.java - * - MakingRequestsWithIAMTempCredentials.java - * - Object archival - * - RestoreArchivedObject - * - KMS key - * - UploadObjectKMSKey.java - */ - - private MiniOzoneCluster cluster; - private AmazonS3 s3Client; - - @BeforeAll - void createClient() { - cluster = cluster(); - s3Client = new S3ClientFactory(cluster.getConf()) - .createS3Client(); - } - - @Test - public void testCreateBucket() { - final String bucketName = getBucketName(); - - Bucket b = s3Client.createBucket(bucketName); - - assertEquals(bucketName, b.getName()); - assertTrue(s3Client.doesBucketExist(bucketName)); - assertTrue(s3Client.doesBucketExistV2(bucketName)); - assertTrue(isBucketEmpty(b)); - } - - /** - * s3-tests: test_bucket_create_exists. - */ - @Test - public void testCreateBucketAlreadyOwnedByYou() { - final String bucketName = getBucketName("owned-by-you"); - s3Client.createBucket(bucketName); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.createBucket(bucketName)); - assertEquals(409, ase.getStatusCode()); - assertEquals(S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU.getCode(), ase.getErrorCode()); - assertEquals("application/xml", ase.getHttpHeaders().get("Content-Type")); - } - - @Test - public void testCreateBucketAlreadyExistsDifferentOwner() throws IOException { - final String bucketName = getBucketName("other-owner"); - final String otherOwner = "other-s3-owner"; - try (OzoneClient ozoneClient = cluster.newClient()) { - ozoneClient.getObjectStore().getS3Volume().createBucket(bucketName, - BucketArgs.newBuilder().setOwner(otherOwner).build()); - } - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.createBucket(bucketName)); - assertEquals(409, ase.getStatusCode()); - assertEquals(S3ErrorTable.BUCKET_ALREADY_EXISTS.getCode(), ase.getErrorCode()); - } - - @Test - public void testBucketACLOperations() { - // TODO HDDS-11738: Uncomment assertions when bucket S3 ACL logic has been fixed - final String bucketName = getBucketName(); - - AccessControlList aclList = new AccessControlList(); - Owner owner = new Owner("owner", "owner"); - aclList.withOwner(owner); - Grantee grantee = new CanonicalGrantee("testGrantee"); - aclList.grantPermission(grantee, Permission.Read); - - - CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName) - .withAccessControlList(aclList); - - s3Client.createBucket(createBucketRequest); - - //assertEquals(aclList, s3Client.getBucketAcl(bucketName)); - - aclList.grantPermission(grantee, Permission.Write); - s3Client.setBucketAcl(bucketName, aclList); - - //assertEquals(aclList, s3Client.getBucketAcl(bucketName)); - } - - /** - * Integration tests for ListBuckets (GET / ListAllMyBuckets). - */ - @Nested - class ListBucketsTests { - - @Test - public void testListBuckets() throws IOException { - List bucketNames = new ArrayList<>(); - for (int i = 0; i <= 5; i++) { - String bucketName = getBucketName(String.valueOf(i)); - s3Client.createBucket(bucketName); - bucketNames.add(bucketName); - } - - List bucketList = s3Client.listBuckets(); - List listBucketNames = bucketList.stream() - .map(Bucket::getName).collect(Collectors.toList()); - - assertThat(listBucketNames).containsAll(bucketNames); - - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - String expectOwner = ugi.getShortUserName(); - - Owner s3AccountOwner = s3Client.getS3AccountOwner(); - - assertThat(s3AccountOwner.getDisplayName()).isEqualTo(expectOwner); - assertThat(s3AccountOwner.getId()).isEqualTo(S3Owner.DEFAULT_S3OWNER_ID); - } - - /** - * Verifies {@code maxBuckets=1} returns one bucket per page and a continuation token - * when more buckets exist. - */ - @Test - public void testListBucketsPaginatedMaxBucketsOne() { - final String bucketA = uniqueObjectName("bucket-a"); - final String bucketB = uniqueObjectName("bucket-b"); - s3Client.createBucket(bucketA); - s3Client.createBucket(bucketB); - try { - List found = S3SDKTestUtils.collectBucketsOnePerPage((token, max) -> { - ListBucketsPaginatedRequest request = new ListBucketsPaginatedRequest() - .withMaxBuckets(max); - if (token != null) { - request.withContinuationToken(token); - } - - ListBucketsPaginatedResult page = s3Client.listBuckets(request); - return new S3SDKTestUtils.BucketListPage( - page.getBuckets().stream().map(Bucket::getName).collect(Collectors.toList()), - page.getContinuationToken()); - }); - List foundTestBuckets = S3SDKTestUtils.filterToExpectedBuckets( - found, bucketA, bucketB); - assertThat(foundTestBuckets).containsExactlyInAnyOrder(bucketA, bucketB); - } finally { - s3Client.deleteBucket(bucketA); - s3Client.deleteBucket(bucketB); - } - } - - /** - * Verifies pagination: listing buckets page-by-page using {@code maxBuckets} - * and the returned continuation token, until all buckets are retrieved. - */ - @Test - public void testListBucketsPaginationReturnsAllBuckets() { - final int totalBuckets = 5; - final int pageSize = 2; - List created = new ArrayList<>(); - - for (int i = 0; i < totalBuckets; i++) { - String name = uniqueObjectName("paginated-" + i); - s3Client.createBucket(name); - created.add(name); - } - - try { - List retrieved = new ArrayList<>(); - String continuationToken = null; - - do { - ListBucketsPaginatedRequest request = new ListBucketsPaginatedRequest() - .withMaxBuckets(pageSize); - if (continuationToken != null) { - request.withContinuationToken(continuationToken); - } - - ListBucketsPaginatedResult response = s3Client.listBuckets(request); - - response.getBuckets().stream() - .map(Bucket::getName) - .filter(created::contains) - .forEach(retrieved::add); - - continuationToken = response.getContinuationToken(); - } while (continuationToken != null); - - assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created); - } finally { - for (String name : created) { - s3Client.deleteBucket(name); - } - } - } - - /** - * verifies that Page 1 uses maxBuckets=1; - * later pages send only continuationToken (no maxBuckets). - */ - @Test - public void testListBucketsContinuationTokenWithoutMaxBuckets() { - List created = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - String name = uniqueObjectName("token-only-" + i); - s3Client.createBucket(name); - created.add(name); - } - - try { - List retrieved = new ArrayList<>(); - String continuationToken = null; - boolean firstPage = true; - - do { - ListBucketsPaginatedRequest request = new ListBucketsPaginatedRequest(); - if (firstPage) { - request.withMaxBuckets(1); - firstPage = false; - } else { - request.withContinuationToken(continuationToken); - } - - ListBucketsPaginatedResult response = s3Client.listBuckets(request); - if (continuationToken == null) { - assertEquals(1, response.getBuckets().size()); - assertNotNull(response.getContinuationToken()); - } else { - assertFalse(response.getBuckets().isEmpty()); - } - - response.getBuckets().stream() - .map(Bucket::getName) - .filter(created::contains) - .forEach(retrieved::add); - - continuationToken = response.getContinuationToken(); - } while (continuationToken != null); - - assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created); - } finally { - for (String name : created) { - s3Client.deleteBucket(name); - } - } - } - } - - @Test - public void testDeleteBucket() { - final String bucketName = getBucketName(); - - s3Client.createBucket(bucketName); - - s3Client.deleteBucket(bucketName); - - assertFalse(s3Client.doesBucketExist(bucketName)); - assertFalse(s3Client.doesBucketExistV2(bucketName)); - } - - @Test - public void testDeleteBucketNotExist() { - final String bucketName = getBucketName(); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucket(bucketName)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchBucket", ase.getErrorCode()); - } - - @Test - public void testDeleteBucketNonEmptyWithKeys() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Upload some objects to the bucket - for (int i = 1; i <= 10; i++) { - s3Client.putObject(bucketName, "key-" + i, RandomStringUtils.secure().nextAlphanumeric(1024)); - } - - // Bucket deletion should fail if there are still keys in the bucket - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucket(bucketName) - ); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(409, ase.getStatusCode()); - assertEquals("BucketNotEmpty", ase.getErrorCode()); - - // Delete all the keys - ObjectListing objectListing = s3Client.listObjects(bucketName); - while (true) { - for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { - s3Client.deleteObject(bucketName, summary.getKey()); - } - - // more object_listing to retrieve? - if (objectListing.isTruncated()) { - objectListing = s3Client.listNextBatchOfObjects(objectListing); - } else { - break; - } - } - } - - @Test - public void testDeleteBucketNonEmptyWithIncompleteMultipartUpload(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (5 * MB)); - - // Create an incomplete multipart upload by initiating multipart upload, - // uploading some parts, but not actually completing it. - String uploadId = initiateMultipartUpload(bucketName, keyName, null, null, null); - - uploadParts(bucketName, keyName, uploadId, multipartUploadFile, 1 * MB); - - // Bucket deletion should fail if there are still keys in the bucket - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucket(bucketName) - ); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(409, ase.getStatusCode()); - assertEquals("BucketNotEmpty", ase.getErrorCode()); - - // After the multipart upload is aborted, the bucket deletion should succeed - abortMultipartUpload(bucketName, keyName, uploadId); - - s3Client.deleteBucket(bucketName); - - assertFalse(s3Client.doesBucketExistV2(bucketName)); - } - - @Test - public void testPutObject() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - } - - @ParameterizedTest - @ValueSource(strings = {"follower-stale", "follower-linearizable", - "leader-only"}) - public void testGetObjectWithReadConsistencyHeader(String readConsistency) - throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, content); - - GetObjectRequest request = new GetObjectRequest(bucketName, keyName); - request.putCustomRequestHeader(READ_CONSISTENCY_HEADER, - readConsistency); - if ("follower-stale".equals(readConsistency)) { - request.putCustomRequestHeader(LOCAL_LEASE_LOG_LIMIT_HEADER, "10"); - } - - try (S3Object object = s3Client.getObject(request); - S3ObjectInputStream objectContent = object.getObjectContent()) { - assertEquals(content, IOUtils.toString(objectContent, - StandardCharsets.UTF_8)); - } - } - - @Test - public void testGetObjectWithInvalidReadConsistencyHeader() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, "bar"); - - GetObjectRequest request = new GetObjectRequest(bucketName, keyName); - request.putCustomRequestHeader(READ_CONSISTENCY_HEADER, - "invalid"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals("InvalidArgument", ase.getErrorCode()); - } - - @Test - public void testPutObjectWithEmptyContentType() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentType(""); - InputStream inputStream = new ByteArrayInputStream( - content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject( - bucketName, keyName, inputStream, metadata); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", - putObjectResult.getETag()); - } - - @Test - public void testGetObjectWithMalformedReadConsistencyHeader() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, "bar"); - - GetObjectRequest request = new GetObjectRequest(bucketName, keyName); - request.putCustomRequestHeader(READ_CONSISTENCY_HEADER, - "follower-stale;logLimit=10"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals("InvalidArgument", ase.getErrorCode()); - } - - @Test - public void testPutObjectIfNoneMatch() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is, new ObjectMetadata()).ifNoneMatch("*"); - - PutObjectResult putObjectResult = s3Client.putObject(request); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - } - - @Test - public void testPutObjectIfNoneMatchFail() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - InputStream is2 = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is2, new ObjectMetadata()).ifNoneMatch("*"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - } - - @Test - public void testPutObjectIfMatch() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - String etag = putObjectResult.getETag(); - - InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is2, new ObjectMetadata()).ifMatch(etag); - - PutObjectResult putObjectResult2 = s3Client.putObject(request); - assertNotNull(putObjectResult2.getETag()); - assertNotEquals(etag, putObjectResult2.getETag()); - - ObjectMetadata updatedObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(putObjectResult2.getETag(), updatedObjectMetadata.getETag()); - } - - @Test - public void testPutObjectIfMatchFail() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult initialResult = - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is2, new ObjectMetadata()).ifMatch("wrong-etag"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - - ObjectMetadata existingObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(initialResult.getETag(), existingObjectMetadata.getETag()); - } - - @ParameterizedTest - @ValueSource(strings = {"*", "\"some-etag\""}) - public void testPutObjectIfMatchMissingKeyFail(String ifMatch) { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream("bar2".getBytes( - StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is, new ObjectMetadata()).ifMatch(ifMatch); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchKey", ase.getErrorCode()); - - AmazonServiceException missingKey = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(bucketName, keyName)); - assertEquals(ErrorType.Client, missingKey.getErrorType()); - assertEquals(404, missingKey.getStatusCode()); - assertEquals("NoSuchKey", missingKey.getErrorCode()); - } - - @Test - public void testDeleteObjectIfMatch() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, putObjectResult.getETag()); - - assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testDeleteObjectIfMatchFail() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, "wrong-etag"); - - assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, responseCode); - ObjectMetadata existingObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(putObjectResult.getETag(), existingObjectMetadata.getETag()); - } - - @Test - public void testDeleteObjectIfMatchWildcard() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, "*"); - - assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testDeleteObjectIfMatchWildcardMissingKeyFail() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, "*"); - - assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, responseCode); - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - private int deleteObjectWithIfMatch(String bucketName, String keyName, String ifMatch) throws IOException { - GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.DELETE) - .withExpiration(Date.from(Instant.now().plusMillis(1000 * 60 * 60))); - request.putCustomRequestHeader(Headers.GET_OBJECT_IF_MATCH, ifMatch); - URL presignedUrl = s3Client.generatePresignedUrl(request); - Map> headers = Collections.singletonMap(Headers.GET_OBJECT_IF_MATCH, - Collections.singletonList(ifMatch)); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "DELETE", headers, null); - return connection.getResponseCode(); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - } - - @Test - public void testCopyObject() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putResult.getETag()); - - CopyObjectResult copyResult = s3Client.copyObject(sourceBucketName, sourceKey, destBucketName, destKey); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", copyResult.getETag()); - } - - @Test - public void testCopyObjectLastModifiedMatchesOmDb() throws Exception { - final String sourceBucketName = getBucketName(); - final String destBucketName = getBucketName(); - final String sourceKey = getKeyName(); - final String destKey = getKeyName(); - final String content = "copy-last-modified-test-content"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - final InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(sourceBucketName, sourceKey, inputStream, new ObjectMetadata()); - - final long sourceModificationTime = getOmKeyModificationTime(cluster, sourceBucketName, sourceKey); - - Thread.sleep(100); - - final CopyObjectResult copyResult = s3Client.copyObject(sourceBucketName, sourceKey, destBucketName, destKey); - - assertNotNull(copyResult.getLastModifiedDate()); - final long responseModificationTime = copyResult.getLastModifiedDate().getTime(); - final long destModificationTime = getOmKeyModificationTime(cluster, destBucketName, destKey); - - assertEquals(destModificationTime, responseModificationTime); - assertNotEquals(sourceModificationTime, responseModificationTime); - } - - @Test - public void testUploadPartCopyLastModifiedMatchesOmDb() throws Exception { - final String bucketName = getBucketName(); - final String sourceKey = getKeyName(); - final String destKey = getKeyName(); - final String content = "copy-last-modified-test-content"; - s3Client.createBucket(bucketName); - - final InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, sourceKey, inputStream, new ObjectMetadata()); - - final long sourceModificationTime = getOmKeyModificationTime(cluster, bucketName, sourceKey); - - Thread.sleep(100); - - final InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucketName, destKey)); - final String uploadId = initResponse.getUploadId(); - - final CopyPartResult copyPartResult = s3Client.copyPart( - new CopyPartRequest() - .withSourceBucketName(bucketName) - .withSourceKey(sourceKey) - .withDestinationBucketName(bucketName) - .withDestinationKey(destKey) - .withUploadId(uploadId) - .withPartNumber(1)); - - assertNotNull(copyPartResult.getLastModifiedDate()); - final long responseModificationTime = copyPartResult.getLastModifiedDate().getTime(); - final long destPartModificationTime = getOmPartModificationTime(cluster, bucketName, destKey, uploadId, 1); - - assertEquals(destPartModificationTime, responseModificationTime); - assertNotEquals(sourceModificationTime, responseModificationTime); - } - - @Test - public void testCopyObjectWithSourceIfMatch() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - String sourceETag = putResult.getETag(); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withMatchingETagConstraint(sourceETag); - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertEquals(sourceETag, copyResult.getETag()); - } - - @Test - public void testCopyObjectWithSourceIfMatchFail() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withMatchingETagConstraint("wrong-etag"); - - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertNull(copyResult); - } - - @Test - public void testCopyObjectWithSourceIfNoneMatch() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - String sourceETag = putResult.getETag(); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withNonmatchingETagConstraint("different-etag"); - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertEquals(sourceETag, copyResult.getETag()); - } - - @Test - public void testCopyObjectWithSourceIfNoneMatchFail() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - String sourceETag = putResult.getETag(); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withNonmatchingETagConstraint(sourceETag); - - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertNull(copyResult); - } - - @Test - public void testPutObjectWithMD5Header() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - byte[] md5Bytes = calculateDigest(new ByteArrayInputStream(contentBytes), 0, contentBytes.length); - String md5Base64 = Base64.getEncoder().encodeToString(md5Bytes); - - InputStream is = new ByteArrayInputStream(contentBytes); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentMD5(md5Base64); - objectMetadata.setContentLength(contentBytes.length); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, objectMetadata); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - - S3Object object = s3Client.getObject(bucketName, keyName); - assertEquals(content.length(), object.getObjectMetadata().getContentLength()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", object.getObjectMetadata().getETag()); - } - - static Stream wrongContentMD5Provider() throws Exception { - byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); - byte[] wrongMd5Bytes = MessageDigest.getInstance("MD5").digest(wrongContentBytes); - String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); - - return Stream.of( - Arguments.of(wrongMd5Base64, S3ErrorTable.BAD_DIGEST.getCode()), - Arguments.of("invalid-base64", S3ErrorTable.INVALID_DIGEST.getCode()) - ); - } - - @ParameterizedTest - @MethodSource("wrongContentMD5Provider") - public void testPutObjectWithWrongMD5Header(String wrongMd5Base64, String expectedErrorCode) { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - InputStream is = new ByteArrayInputStream(contentBytes); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentMD5(wrongMd5Base64); - objectMetadata.setContentLength(contentBytes.length); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(bucketName, keyName, is, objectMetadata)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals(expectedErrorCode, ase.getErrorCode()); - - // Verify the object was not uploaded - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testMultipartUploadWithMD5Header() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate multipart upload - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Prepare part data - String part1Content = "part1data"; - byte[] part1Bytes = part1Content.getBytes(StandardCharsets.UTF_8); - byte[] part1Md5Bytes = calculateDigest(new ByteArrayInputStream(part1Bytes), 0, part1Bytes.length); - String part1Md5Base64 = Base64.getEncoder().encodeToString(part1Md5Bytes); - - // Upload part 1 with MD5 - InputStream part1InputStream = new ByteArrayInputStream(part1Bytes); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentMD5(part1Md5Base64); - metadata.setContentLength(part1Bytes.length); - - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(part1InputStream) - .withPartSize(part1Bytes.length) - .withObjectMetadata(metadata); - - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Verify ETag - String expectedETag = DatatypeConverter.printHexBinary(part1Md5Bytes).toLowerCase(); - assertEquals(expectedETag, uploadResult.getPartETag().getETag()); - - // Complete multipart upload - List partETags = new ArrayList<>(); - partETags.add(uploadResult.getPartETag()); - - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - s3Client.completeMultipartUpload(completeRequest); - - // Verify object was uploaded - S3Object object = s3Client.getObject(bucketName, keyName); - try (S3ObjectInputStream s3is = object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream()) { - IOUtils.copy(s3is, bos); - assertEquals(part1Content, bos.toString("UTF-8")); - } - } - - @ParameterizedTest - @MethodSource("wrongContentMD5Provider") - public void testMultipartUploadPartWithWrongMD5Header(String wrongMd5Base64, String expectedErrorCode) { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate multipart upload - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Prepare part data with wrong MD5 - String partContent = "partdata"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - - // Upload part with wrong/invalid MD5 should fail - InputStream partInputStream = new ByteArrayInputStream(partBytes); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentMD5(wrongMd5Base64); - metadata.setContentLength(partBytes.length); - - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(partInputStream) - .withPartSize(partBytes.length) - .withObjectMetadata(metadata); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.uploadPart(uploadRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals(expectedErrorCode, ase.getErrorCode()); - - // Abort the multipart upload - AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(bucketName, keyName, uploadId); - s3Client.abortMultipartUpload(abortRequest); - - // Verify object was not created - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testCompleteMultipartUploadIfNoneMatch() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate and complete multipart upload with If-None-Match: * - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "part1data"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-None-Match: * (key doesn't exist, should succeed) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-None-Match", "*"); - - CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(completeRequest); - assertNotNull(result.getETag()); - - // Verify object was created - assertTrue(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testCompleteMultipartUploadIfNoneMatchFail() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // First, create an existing key - s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream("existing".getBytes(StandardCharsets.UTF_8)), - new ObjectMetadata()); - - // Initiate multipart upload for same key - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "part1data"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-None-Match: * (key exists, should fail) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-None-Match", "*"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.completeMultipartUpload(completeRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - } - - @Test - public void testCompleteMultipartUploadIfMatch() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // First, create an existing key and get its ETag - PutObjectResult existingResult = s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream("existing".getBytes(StandardCharsets.UTF_8)), - new ObjectMetadata()); - String existingETag = existingResult.getETag(); - - // Initiate multipart upload for same key - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "newcontent"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-Match: (should succeed) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-Match", "\"" + existingETag + "\""); - - CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(completeRequest); - assertNotNull(result.getETag()); - assertNotEquals(existingETag, stripQuotes(result.getETag())); - } - - @Test - public void testCompleteMultipartUploadIfMatchFail() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // First, create an existing key - s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream("existing".getBytes(StandardCharsets.UTF_8)), - new ObjectMetadata()); - - // Initiate multipart upload for same key - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "newcontent"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-Match: wrong-etag (should fail) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-Match", "\"wrong-etag\""); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.completeMultipartUpload(completeRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - } - - @ParameterizedTest - @ValueSource(strings = {"*", "\"some-etag\""}) - public void testCompleteMultipartUploadIfMatchMissingKeyFail(String ifMatch) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate multipart upload (key doesn't exist) - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "newcontent"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-Match on non-existent key (should fail) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-Match", ifMatch); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.completeMultipartUpload(completeRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchKey", ase.getErrorCode()); - } - - @Test - public void testPutDoubleSlashPrefixObject() throws IOException { - final String bucketName = getBucketName(); - final String keyName = "//dir1"; - final String content = "bar"; - // Create a FSO bucket for test - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - OzoneVolume volume = store.getS3Volume(); - OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder() - .setVolumeName(volume.getName()) - .setBucketName(bucketName) - .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); - OzoneManagerProtocol ozoneManagerProtocol = store.getClientProxy().getOzoneManagerClient(); - ozoneManagerProtocol.createBucket(bucketInfo.build()); - } - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - - S3Object object = s3Client.getObject(bucketName, keyName); - assertEquals(content.length(), object.getObjectMetadata().getContentLength()); - } - - @Test - public void testPutObjectEmpty() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = ""; - s3Client.createBucket(bucketName); - - long initialAllocatedBlocks = - cluster.getStorageContainerManager().getPipelineManager() - .getMetrics().getTotalNumBlocksAllocated(); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - assertEquals("d41d8cd98f00b204e9800998ecf8427e", putObjectResult.getETag()); - - // Verify via Ozone client that the key has no block locations - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - OzoneVolume volume = store.getS3Volume(); - OzoneBucket bucket = volume.getBucket(bucketName); - - OzoneKeyDetails keyDetails = bucket.getKey(keyName); - assertNotNull(keyDetails); - assertEquals(0, keyDetails.getDataSize(), - "Empty S3 object should have dataSize of 0"); - assertTrue(keyDetails.getOzoneKeyLocations().isEmpty(), - "Empty S3 object should have no block locations"); - } - - // createKey should skip block allocation if data size is 0 - long currentAllocatedBlocks = - cluster.getStorageContainerManager().getPipelineManager().getMetrics().getTotalNumBlocksAllocated(); - assertEquals(initialAllocatedBlocks, currentAllocatedBlocks); - } - - @Test - public void testPutObjectACL() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - final byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - String originalObjectETag = putObjectResult.getETag(); - assertTrue(s3Client.doesObjectExist(bucketName, keyName)); - - AccessControlList aclList = new AccessControlList(); - Owner owner = new Owner("owner", "owner"); - aclList.withOwner(owner); - Grantee grantee = new CanonicalGrantee("testGrantee"); - aclList.grantPermission(grantee, Permission.Read); - - SetObjectAclRequest setObjectAclRequest = new SetObjectAclRequest(bucketName, keyName, aclList); - - assertNotImplemented(() -> s3Client.setObjectAcl(setObjectAclRequest)); - - // Ensure that the object content remains unchanged - ObjectMetadata updatedObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(originalObjectETag, updatedObjectMetadata.getETag()); - S3Object updatedObject = s3Client.getObject(bucketName, keyName); - - try (S3ObjectInputStream s3is = updatedObject.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(contentBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(content, bos.toString("UTF-8")); - } - } - - @Test - public void testGetObjectAclNotImplemented() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - s3Client.putObject( - bucketName, keyName, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); - - assertNotImplemented(() -> s3Client.getObjectAcl(new GetObjectAclRequest(bucketName, keyName))); - } - - @Test - public void testGetBucketAccelerateConfigurationNotImplemented() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - assertNotImplemented( - () -> s3Client.getBucketAccelerateConfiguration(new GetBucketAccelerateConfigurationRequest(bucketName))); - } - - @Test - public void testGetObject() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - final byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(contentBytes); - ObjectMetadata objectMetadata = new ObjectMetadata(); - Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); - objectMetadata.setUserMetadata(userMetadata); - - List tags = Arrays.asList(new Tag("tag1", "value1"), new Tag("tag2", "value2")); - ObjectTagging objectTagging = new ObjectTagging(tags); - - - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, is, objectMetadata) - .withTagging(objectTagging); - - s3Client.putObject(putObjectRequest); - - S3Object s3Object = s3Client.getObject(bucketName, keyName); - assertEquals(tags.size(), s3Object.getTaggingCount()); - - try (S3ObjectInputStream s3is = s3Object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(contentBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(content, bos.toString("UTF-8")); - } - } - - /** - * Adapted from ceph s3-tests test_object_read_unreadable. - */ - @Test - public void testGetObjectUnreadableKey() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, - StandardCharsets.ISO_8859_1); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(bucketName, unreadableKey)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals(S3ErrorTable.INVALID_URI.getCode(), ase.getErrorCode()); - assertEquals(S3ErrorTable.INVALID_URI.getErrorMessage(), ase.getErrorMessage()); - } - - static Stream onlyTagKeyCasesV1() { - Map fooBarEmptyBar = new HashMap<>(); - fooBarEmptyBar.put("foo", "bar"); - fooBarEmptyBar.put("bar", ""); - return Stream.of( - Arguments.of( - new ObjectTagging(Collections.singletonList(new Tag("tag1", null))), - Collections.singletonMap("tag1", "")), - Arguments.of( - new ObjectTagging(Arrays.asList(new Tag("foo", "bar"), new Tag("bar", null))), - fooBarEmptyBar) - ); - } - - @ParameterizedTest - @MethodSource("onlyTagKeyCasesV1") - public void testPutObjectWithOnlyTagKey(ObjectTagging objectTagging, - Map expectedTags) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "0123456789"; - s3Client.createBucket(bucketName); - - try (InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) { - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, is, new ObjectMetadata()) - .withTagging(objectTagging); - s3Client.putObject(putObjectRequest); - } - - GetObjectTaggingResult taggingResult = s3Client.getObjectTagging( - new GetObjectTaggingRequest(bucketName, keyName)); - Map actualTags = taggingResult.getTagSet().stream() - .collect(Collectors.toMap(Tag::getKey, Tag::getValue)); - assertEquals(expectedTags, actualTags); - } - - @Test - public void testHeadObjectReturnsTaggingCount() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "head-object-tag-count"; - s3Client.createBucket(bucketName); - - s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); - - List tags = Arrays.asList(new Tag("tag1", "v1"), new Tag("tag2", "v2")); - s3Client.setObjectTagging( - new SetObjectTaggingRequest(bucketName, keyName, new ObjectTagging(tags))); - - ObjectMetadata head = s3Client.getObjectMetadata(bucketName, keyName); - // AWS SDK v1: getTaggingCount() exists on S3Object (GET), not on ObjectMetadata (HEAD). - // x-amz-tagging-count is exposed via raw metadata. - Object tagCountHeader = head.getRawMetadataValue(Headers.S3_TAGGING_COUNT); - assertNotNull(tagCountHeader); - assertEquals(tags.size(), Integer.parseInt(tagCountHeader.toString())); - } - - @Test - public void testGetObjectTaggingReturnsTagsSortedByKey() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, ""); - - List tagsPutOrder = Arrays.asList(new Tag("key2", "val2"), new Tag("key", "val")); - s3Client.setObjectTagging( - new SetObjectTaggingRequest(bucketName, keyName, new ObjectTagging(tagsPutOrder))); - - GetObjectTaggingResult taggingResult = - s3Client.getObjectTagging(new GetObjectTaggingRequest(bucketName, keyName)); - List tagSet = taggingResult.getTagSet(); - assertEquals(2, tagSet.size()); - assertEquals("key", tagSet.get(0).getKey()); - assertEquals("val", tagSet.get(0).getValue()); - assertEquals("key2", tagSet.get(1).getKey()); - assertEquals("val2", tagSet.get(1).getValue()); - } - - @Test - public void testBucketTaggingPutGetDelete() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // AWS SDK v1 returns null when no bucket tagging is configured. - assertNull(s3Client.getBucketTaggingConfiguration( - new GetBucketTaggingConfigurationRequest(bucketName))); - - TagSet tagSet = new TagSet(); - tagSet.setTag("tag-key1", "tag-value1"); - tagSet.setTag("tag-key2", "tag-value2"); - s3Client.setBucketTaggingConfiguration(new SetBucketTaggingConfigurationRequest(bucketName, - new BucketTaggingConfiguration(Collections.singletonList(tagSet)))); - - BucketTaggingConfiguration taggingConfiguration = - s3Client.getBucketTaggingConfiguration(new GetBucketTaggingConfigurationRequest(bucketName)); - Map actualTags = taggingConfiguration.getTagSet().getAllTags(); - assertEquals(2, actualTags.size()); - assertEquals("tag-value1", actualTags.get("tag-key1")); - assertEquals("tag-value2", actualTags.get("tag-key2")); - - s3Client.deleteBucketTaggingConfiguration(new DeleteBucketTaggingConfigurationRequest(bucketName)); - - assertNull(s3Client.getBucketTaggingConfiguration( - new GetBucketTaggingConfigurationRequest(bucketName))); - } - - @Test - public void testGetObjectWithoutETag() throws Exception { - // Object uploaded using other protocols (e.g. ofs / ozone cli) will not - // have ETag. Ensure that ETag will not do ETag validation on GetObject if there - // is no ETag present. - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - String value = "sample value"; - byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); - - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - - OzoneVolume volume = store.getS3Volume(); - OzoneBucket bucket = volume.getBucket(bucketName); - - try (OzoneOutputStream out = bucket.createKey(keyName, - valueBytes.length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), - Collections.emptyMap())) { - out.write(valueBytes); - } - } - - S3Object s3Object = s3Client.getObject(bucketName, keyName); - assertNull(s3Object.getObjectMetadata().getETag()); - - try (S3ObjectInputStream s3is = s3Object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(valueBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(value, bos.toString("UTF-8")); - } - } - - @Test - public void testListObjectsMany() throws Exception { - testListObjectsMany(false); - } - - @Test - public void testListObjectsManyV2() throws Exception { - testListObjectsMany(true); - } - - @Test - public void testListObjectsSpecialKeyNames() throws Exception { - final String bucketName = getBucketName("special-keys"); - final String content = "x"; - s3Client.createBucket(bucketName); - - for (String keyName : S3SDKTestUtils.S3_SPECIAL_KEY_NAMES) { - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - try (S3Object object = s3Client.getObject(bucketName, keyName)) { - assertEquals(content, IOUtils.toString(object.getObjectContent(), StandardCharsets.UTF_8)); - } - } - - ObjectListing listObjectsResponse = s3Client.listObjects( - new ListObjectsRequest().withBucketName(bucketName)); - List listedKeys = listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey) - .collect(Collectors.toList()); - assertEquals(S3SDKTestUtils.S3_SPECIAL_KEY_NAMES, listedKeys); - } - - private void testListObjectsMany(boolean isListV2) throws Exception { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - final List keyNames = Arrays.asList( - getKeyName("1"), - getKeyName("2"), - getKeyName("3") - ); - final List keyNamesWithoutETag = Arrays.asList( - getKeyName("4"), - getKeyName("5") - ); - - final Map keyToEtag = new HashMap<>(); - for (String keyName: keyNames) { - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, - RandomStringUtils.secure().nextAlphanumeric(5)); - keyToEtag.put(keyName, putObjectResult.getETag()); - } - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - - OzoneVolume volume = store.getS3Volume(); - OzoneBucket bucket = volume.getBucket(bucketName); - - for (String keyNameWithoutETag : keyNamesWithoutETag) { - byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); - try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, - valueBytes.length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), - Collections.emptyMap())) { - out.write(valueBytes); - } - } - } - - List objectSummaries; - String continuationToken; - if (isListV2) { - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(2); - ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertTrue(listObjectsResponse.isTruncated()); - continuationToken = listObjectsResponse.getNextContinuationToken(); - } else { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(2); - ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertTrue(listObjectsResponse.isTruncated()); - continuationToken = listObjectsResponse.getNextMarker(); - } - assertThat(objectSummaries).hasSize(2); - assertEquals(objectSummaries.stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(0, 2)); - for (S3ObjectSummary objectSummary : objectSummaries) { - assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); - } - - // Include both keys with and without ETag - if (isListV2) { - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(5) - .withContinuationToken(continuationToken); - ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertFalse(listObjectsResponse.isTruncated()); - } else { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(5) - .withMarker(continuationToken); - ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertFalse(listObjectsResponse.isTruncated()); - } - - assertThat(objectSummaries).hasSize(3); - assertEquals(keyNames.get(2), objectSummaries.get(0).getKey()); - assertEquals(keyNamesWithoutETag.get(0), objectSummaries.get(1).getKey()); - assertEquals(keyNamesWithoutETag.get(1), objectSummaries.get(2).getKey()); - for (S3ObjectSummary objectSummary : objectSummaries) { - assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); - } - } - - @Test - public void testListObjectsBucketNotExist() { - final String bucketName = getBucketName(); - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName); - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.listObjects(listObjectsRequest)); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchBucket", ase.getErrorCode()); - } - - @Test - public void testListObjectsV2BucketNotExist() { - final String bucketName = getBucketName(); - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName); - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.listObjectsV2(listObjectsRequest)); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchBucket", ase.getErrorCode()); - } - - @Test - public void testListObjectsV2FetchOwner() { - final String bucketName = getBucketName("fetch-owner"); - final String keyName = getKeyName("obj"); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, RandomStringUtils.secure().nextAlphanumeric(5)); - - ListObjectsV2Result defaultResponse = s3Client.listObjectsV2( - new ListObjectsV2Request().withBucketName(bucketName)); - assertThat(defaultResponse.getObjectSummaries()).isNotEmpty(); - assertNull(defaultResponse.getObjectSummaries().get(0).getOwner()); - - ListObjectsV2Result falseResponse = s3Client.listObjectsV2( - new ListObjectsV2Request().withBucketName(bucketName).withFetchOwner(false)); - assertNull(falseResponse.getObjectSummaries().get(0).getOwner()); - - ListObjectsV2Result trueResponse = s3Client.listObjectsV2( - new ListObjectsV2Request().withBucketName(bucketName).withFetchOwner(true)); - Owner owner = trueResponse.getObjectSummaries().get(0).getOwner(); - assertNotNull(owner); - assertNotNull(owner.getDisplayName()); - assertEquals(S3Owner.DEFAULT_S3OWNER_ID, owner.getId()); - } - - @Test - public void testHighLevelMultipartUpload(@TempDir Path tempDir) throws Exception { - TransferManager tm = TransferManagerBuilder.standard() - .withS3Client(s3Client) - .build(); - - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - // The minimum file size to for TransferManager to initiate multipart upload is 16MB, so create a file - // larger than the threshold. - // See TransferManagerConfiguration#getMultipartUploadThreshold - int fileSize = (int) (20 * MB); - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, fileSize); - - // TransferManager processes all transfers asynchronously, - // so this call returns immediately. - Upload upload = tm.upload(bucketName, keyName, multipartUploadFile); - - upload.waitForCompletion(); - UploadResult uploadResult = upload.waitForUploadResult(); - assertEquals(bucketName, uploadResult.getBucketName()); - assertEquals(keyName, uploadResult.getKey()); - } - - @Test - public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); - - List tags = Arrays.asList(new Tag("tag1", "value1"), new Tag("tag2", "value2")); - - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (25 * MB)); - - multipartUpload(bucketName, keyName, multipartUploadFile, 5 * MB, null, userMetadata, tags); - - S3Object s3Object = s3Client.getObject(bucketName, keyName); - assertEquals(keyName, s3Object.getKey()); - assertEquals(bucketName, s3Object.getBucketName()); - assertEquals(tags.size(), s3Object.getTaggingCount()); - - ObjectMetadata objectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(userMetadata, objectMetadata.getUserMetadata()); - } - - @Test - public void testListMultipartUploads() { - final String bucketName = getBucketName(); - final String multipartKey1 = getKeyName("multipart1"); - final String multipartKey2 = getKeyName("multipart2"); - - s3Client.createBucket(bucketName); - - List uploadIds = new ArrayList<>(); - - String uploadId1 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); - uploadIds.add(uploadId1); - String uploadId2 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); - uploadIds.add(uploadId2); - - String uploadId3 = initiateMultipartUpload(bucketName, multipartKey2, null, null, null); - uploadIds.add(uploadId3); - - ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName); - listMultipartUploadsRequest.setMaxUploads(5000); - - MultipartUploadListing result = s3Client.listMultipartUploads(listMultipartUploadsRequest); - - List listUploadIds = result.getMultipartUploads().stream() - .map(MultipartUpload::getUploadId) - .collect(Collectors.toList()); - - assertEquals(uploadIds, listUploadIds); - } - - @ParameterizedTest - @ValueSource(ints = {10, 5000}) - public void testListMultipartUploadsPagination(int requestedMaxUploads) { - final String bucketName = getBucketName() + "-" + requestedMaxUploads; - final String multipartKeyPrefix = getKeyName("multipart"); - - s3Client.createBucket(bucketName); - - // Create multipart uploads to test pagination - List allKeys = new ArrayList<>(); - Map keyToUploadId = new HashMap<>(); - - final int effectiveMaxUploads = Math.min(requestedMaxUploads, MAX_UPLOADS_LIMIT); - final int uploadsCreated = 2 * effectiveMaxUploads + 5; - final int expectedPages = uploadsCreated / effectiveMaxUploads + 1; - - for (int i = 0; i < uploadsCreated; i++) { - String key = String.format("%s-%04d", multipartKeyPrefix, i); - allKeys.add(key); - String uploadId = initiateMultipartUpload(bucketName, key, null, null, null); - keyToUploadId.put(key, uploadId); - } - Collections.sort(allKeys); - - // Test pagination - Set retrievedKeys = new HashSet<>(); - String keyMarker = null; - String uploadIdMarker = null; - boolean truncated = true; - int pageCount = 0; - - do { - ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(bucketName) - .withMaxUploads(requestedMaxUploads) - .withKeyMarker(keyMarker) - .withUploadIdMarker(uploadIdMarker); - - MultipartUploadListing result = s3Client.listMultipartUploads(request); - pageCount++; - - // Verify page size - if (pageCount < expectedPages) { - assertEquals(effectiveMaxUploads, result.getMultipartUploads().size()); - assertTrue(result.isTruncated()); - } else { - assertEquals(uploadsCreated % effectiveMaxUploads, result.getMultipartUploads().size()); - assertFalse(result.isTruncated()); - } - - // Collect keys and verify uploadIds - for (MultipartUpload upload : result.getMultipartUploads()) { - String key = upload.getKey(); - retrievedKeys.add(key); - assertEquals(keyToUploadId.get(key), upload.getUploadId()); - } - - // Verify response - assertNull(result.getPrefix()); - assertEquals(result.getUploadIdMarker(), uploadIdMarker); - assertEquals(result.getKeyMarker(), keyMarker); - assertEquals(effectiveMaxUploads, result.getMaxUploads()); - - // Verify next markers content - if (result.isTruncated()) { - MultipartUpload lastUploadOnPage = result.getMultipartUploads() - .get(result.getMultipartUploads().size() - 1); - assertEquals(lastUploadOnPage.getKey(), result.getNextKeyMarker()); - assertEquals(lastUploadOnPage.getUploadId(), result.getNextUploadIdMarker()); - } else { - assertNull(result.getNextKeyMarker()); - assertNull(result.getNextUploadIdMarker()); - } - - // Update markers for next page - keyMarker = result.getNextKeyMarker(); - uploadIdMarker = result.getNextUploadIdMarker(); - - truncated = result.isTruncated(); - - } while (truncated); - - // Verify pagination results - assertEquals(expectedPages, pageCount); - assertEquals(uploadsCreated, retrievedKeys.size(), "Should retrieve all uploads"); - assertEquals( - allKeys, - retrievedKeys.stream().sorted().collect(Collectors.toList()), - "Retrieved keys should match expected keys in order"); - - // Test with prefix - String prefix = multipartKeyPrefix + "-001"; - ListMultipartUploadsRequest prefixRequest = new ListMultipartUploadsRequest(bucketName) - .withPrefix(prefix); - - MultipartUploadListing prefixResult = s3Client.listMultipartUploads(prefixRequest); - - assertEquals(prefix, prefixResult.getPrefix()); - assertEquals( - IntStream.rangeClosed(0, 9) - .mapToObj(i -> prefix + i) - .collect(Collectors.toList()), - prefixResult.getMultipartUploads().stream() - .map(MultipartUpload::getKey) - .collect(Collectors.toList())); - } - - @Test - public void testListMultipartUploadsPaginationCornerCases() { - final String bucketName = getBucketName(); - final String keyA = getKeyName("samekey"); - final String keyB = getKeyName("after"); - - s3Client.createBucket(bucketName); - - // Create multiple MPUs for the same key to verify upload-id-marker semantics - List keyAUploadIds = new ArrayList<>(); - keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null, null)); - keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null, null)); - keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null, null)); - // Also create another key to ensure listing proceeds past keyA - initiateMultipartUpload(bucketName, keyB, null, null, null); - - // Sort upload IDs lexicographically to match listing order for the same key - Collections.sort(keyAUploadIds); - - // Case 1: key-marker=keyA and upload-id-marker set to the lowest uploadId - // Per spec, same-key uploads MAY be included if uploadId > marker - ListMultipartUploadsRequest request1 = new ListMultipartUploadsRequest(bucketName) - .withKeyMarker(keyA) - .withUploadIdMarker(keyAUploadIds.get(0)) - .withMaxUploads(100); - MultipartUploadListing result1 = s3Client.listMultipartUploads(request1); - - List uploads1 = result1.getMultipartUploads(); - // Collect same-key uploads and verify none are <= marker - List sameKeyIds1 = uploads1.stream() - .filter(u -> keyA.equals(u.getKey())) - .map(MultipartUpload::getUploadId) - .collect(Collectors.toList()); - assertThat(sameKeyIds1).allSatisfy(id -> assertTrue(id.compareTo(keyAUploadIds.get(0)) > 0)); - - // Case 2: key-marker=keyA and upload-id-marker set to the highest uploadId - // Expect no same-key (keyA) uploads to be returned - ListMultipartUploadsRequest request2 = new ListMultipartUploadsRequest(bucketName) - .withKeyMarker(keyA) - .withUploadIdMarker(keyAUploadIds.get(2)) - .withMaxUploads(100); - MultipartUploadListing result2 = s3Client.listMultipartUploads(request2); - assertTrue(result2.getMultipartUploads().stream().noneMatch(u -> keyA.equals(u.getKey()))); - } - - @Test - public void testListParts(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final long fileSize = 5 * MB; - final long partSize = 1 * MB; - final int maxParts = 2; - - s3Client.createBucket(bucketName); - - String uploadId = initiateMultipartUpload(bucketName, keyName, null, null, null); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) fileSize); - - List partETags = uploadParts(bucketName, keyName, uploadId, multipartUploadFile, partSize); - - List listPartETags = new ArrayList<>(); - int partNumberMarker = 0; - int expectedNumOfParts = 5; - PartListing listPartsResult; - do { - ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, keyName, uploadId) - .withMaxParts(maxParts) - .withPartNumberMarker(partNumberMarker); - listPartsResult = s3Client.listParts(listPartsRequest); - if (expectedNumOfParts > maxParts) { - assertTrue(listPartsResult.isTruncated()); - partNumberMarker = listPartsResult.getNextPartNumberMarker(); - expectedNumOfParts -= maxParts; - } else { - assertFalse(listPartsResult.isTruncated()); - } - for (PartSummary partSummary : listPartsResult.getParts()) { - listPartETags.add(new PartETag(partSummary.getPartNumber(), partSummary.getETag())); - } - } while (listPartsResult.isTruncated()); - - assertEquals(partETags.size(), listPartETags.size()); - for (int i = 0; i < partETags.size(); i++) { - assertEquals(partETags.get(i).getPartNumber(), listPartETags.get(i).getPartNumber()); - assertEquals(partETags.get(i).getETag(), listPartETags.get(i).getETag()); - } - } - - @Test - public void testGetParticularPart(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (15 * MB)); - - multipartUpload(bucketName, keyName, multipartUploadFile, 5 * MB, null, null, null); - - GetObjectRequest getObjectRequestAll = new GetObjectRequest(bucketName, keyName); - getObjectRequestAll.setPartNumber(0); - S3Object s3ObjectAll = s3Client.getObject(getObjectRequestAll); - long allPartContentLength = s3ObjectAll.getObjectMetadata().getContentLength(); - - GetObjectRequest getObjectRequestOne = new GetObjectRequest(bucketName, keyName); - getObjectRequestOne.setPartNumber(1); - S3Object s3ObjectOne = s3Client.getObject(getObjectRequestOne); - long partOneContentLength = s3ObjectOne.getObjectMetadata().getContentLength(); - assertEquals(allPartContentLength / 3, partOneContentLength); - - GetObjectRequest getObjectRequestTwo = new GetObjectRequest(bucketName, keyName); - getObjectRequestTwo.setPartNumber(2); - S3Object s3ObjectTwo = s3Client.getObject(getObjectRequestTwo); - long partTwoContentLength = s3ObjectTwo.getObjectMetadata().getContentLength(); - assertEquals(allPartContentLength / 3, partTwoContentLength); - - GetObjectRequest getObjectRequestThree = new GetObjectRequest(bucketName, keyName); - getObjectRequestThree.setPartNumber(1); - S3Object s3ObjectThree = s3Client.getObject(getObjectRequestTwo); - long partThreeContentLength = s3ObjectThree.getObjectMetadata().getContentLength(); - assertEquals(allPartContentLength / 3, partThreeContentLength); - - assertEquals(allPartContentLength, (partOneContentLength + partTwoContentLength + partThreeContentLength)); - } - - @Test - public void testGetNotExistedPart(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (15 * MB)); - - multipartUpload(bucketName, keyName, multipartUploadFile, 5 * MB, null, null, null); - - GetObjectRequest getObjectRequestOne = new GetObjectRequest(bucketName, keyName); - getObjectRequestOne.setPartNumber(4); - // Reading a part number beyond the object's part count must fail with - // InvalidPart, instead of returning an empty (0-byte) object. - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(getObjectRequestOne)); - assertEquals(400, ase.getStatusCode()); - assertEquals("InvalidPart", ase.getErrorCode()); - } - - @Test - public void testListPartsNotFound() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - ListPartsRequest listPartsRequest = - new ListPartsRequest(bucketName, keyName, "nonexist"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.listParts(listPartsRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchUpload", ase.getErrorCode()); - } - - @Test - public void testQuotaExceeded() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - cluster.newClient().getObjectStore() - .getVolume("s3v") - .getBucket(bucketName) - .setQuota(OzoneQuota.parseQuota("1", "10")); - - // Upload some objects to the bucket - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(bucketName, keyName, - RandomStringUtils.secure().nextAlphanumeric(1024))); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(403, ase.getStatusCode()); - assertEquals("QuotaExceeded", ase.getErrorCode()); - } - - @Test - public void testS3LifecycleConfigurationCreateSuccessfully() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule rule1 = new BucketLifecycleConfiguration.Rule() - .withId("expire-logs-after-365-days") - .withPrefix("logs/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(365); - rules.add(rule1); - - configuration.setRules(rules); - - // Set lifecycle configuration - SetBucketLifecycleConfigurationRequest request = - new SetBucketLifecycleConfigurationRequest(bucketName, configuration); - s3Client.setBucketLifecycleConfiguration(request); - - // Verify the configuration was set - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - assertEquals(1, retrievedConfig.getRules().size()); - - // Verify rule 1 - BucketLifecycleConfiguration.Rule retrievedRule1 = retrievedConfig.getRules().get(0); - assertEquals("expire-logs-after-365-days", retrievedRule1.getId()); - assertEquals("logs/", retrievedRule1.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule1.getStatus()); - assertEquals(365, retrievedRule1.getExpirationInDays()); - } - - @Test - public void testS3LifecycleConfigurationCreationFailed() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Test 1: Invalid configuration - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - - BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule() - .withId("invalid") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rules.add(rule); - configuration.setRules(rules); - SetBucketLifecycleConfigurationRequest request = - new SetBucketLifecycleConfigurationRequest(bucketName, configuration); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.setBucketLifecycleConfiguration(request)); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, ase.getStatusCode()); - - // Test 2: Non-existent bucket - final String nonExistentBucket = getBucketName("nonexistent"); - BucketLifecycleConfiguration validConfig = new BucketLifecycleConfiguration(); - List validRules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule validRule = new BucketLifecycleConfiguration.Rule() - .withId("test-rule") - .withPrefix("test/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(30); - validRules.add(validRule); - validConfig.setRules(validRules); - - SetBucketLifecycleConfigurationRequest nonExistentRequest = - new SetBucketLifecycleConfigurationRequest(nonExistentBucket, validConfig); - - AmazonServiceException ase2 = assertThrows(AmazonServiceException.class, - () -> s3Client.setBucketLifecycleConfiguration(nonExistentRequest)); - assertEquals(ErrorType.Client, ase2.getErrorType()); - assertEquals(HttpURLConnection.HTTP_NOT_FOUND, ase2.getStatusCode()); - assertEquals("NoSuchBucket", ase2.getErrorCode()); - - BucketLifecycleConfiguration pastDateConfig = new BucketLifecycleConfiguration(); - List pastDateRules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule pastDateRule = new BucketLifecycleConfiguration.Rule() - .withId("past-date") - .withPrefix("logs/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationDate(Date.from(Instant.parse("2020-01-01T00:00:00Z"))); - pastDateRules.add(pastDateRule); - pastDateConfig.setRules(pastDateRules); - SetBucketLifecycleConfigurationRequest pastDateRequest = - new SetBucketLifecycleConfigurationRequest(bucketName, pastDateConfig); - - AmazonServiceException ase3 = assertThrows(AmazonServiceException.class, - () -> s3Client.setBucketLifecycleConfiguration(pastDateRequest)); - assertEquals(ErrorType.Client, ase3.getErrorType()); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, ase3.getStatusCode()); - assertEquals(S3ErrorTable.INVALID_REQUEST.getCode(), ase3.getErrorCode()); - assertTrue(ase3.getErrorMessage().contains("must be in the future"), - "The client should receive OM's detailed message explaining the date is in the past, got: " - + ase3.getErrorMessage()); - } - - @Test - public void testS3LifecycleConfigurationDelete() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Test delete lifecycle for a bucket, while doesn't have lifecycle - assertNull(s3Client.getBucketLifecycleConfiguration(bucketName)); - // Idempotent delete: no exception expected even without an existing config - s3Client.deleteBucketLifecycleConfiguration(bucketName); - - // First create a lifecycle configuration - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule() - .withId("test-rule") - .withPrefix("test/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(30); - rules.add(rule); - configuration.setRules(rules); - - s3Client.setBucketLifecycleConfiguration(bucketName, configuration); - // Verify it exists - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - assertEquals(1, retrievedConfig.getRules().size()); - // Delete the lifecycle configuration - s3Client.deleteBucketLifecycleConfiguration(bucketName); - assertNull(s3Client.getBucketLifecycleConfiguration(bucketName)); - // Test delete on non-existent bucket - final String nonExistentBucket = getBucketName("nonexistent"); - assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucketLifecycleConfiguration(nonExistentBucket)); - } - - @Test - public void testS3LifecycleConfigurationGet() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Test get on bucket without lifecycle configuration - assertNull(s3Client.getBucketLifecycleConfiguration(bucketName)); - - // Create a comprehensive lifecycle configuration - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - - // Rule with expiration and prefix - BucketLifecycleConfiguration.Rule rule1 = new BucketLifecycleConfiguration.Rule() - .withId("expire-old-files") - .withPrefix("old/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(365); - - rules.add(rule1); - configuration.setRules(rules); - - // Set the configuration - s3Client.setBucketLifecycleConfiguration(bucketName, configuration); - - // Get and verify the configuration - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - - assertEquals(1, retrievedConfig.getRules().size()); - - // Verify first rule - BucketLifecycleConfiguration.Rule retrievedRule1 = retrievedConfig.getRules().get(0); - assertEquals("expire-old-files", retrievedRule1.getId()); - assertEquals("old/", retrievedRule1.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule1.getStatus()); - assertEquals(365, retrievedRule1.getExpirationInDays()); - - // Test getting configuration using GetBucketLifecycleConfigurationRequest - GetBucketLifecycleConfigurationRequest getRequest = - new GetBucketLifecycleConfigurationRequest(bucketName); - BucketLifecycleConfiguration configFromRequest = - s3Client.getBucketLifecycleConfiguration(getRequest); - assertEquals(retrievedConfig.getRules().size(), configFromRequest.getRules().size()); - } - - @Test - public void testGetLifecycleWithAbortIncompleteMultipartUpload() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - - BucketLifecycleConfiguration.Rule rule1 = new BucketLifecycleConfiguration.Rule() - .withId("abort-incomplete-mpu-with-prefix") - .withPrefix("uploads/") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule1.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(7)); - - BucketLifecycleConfiguration.Rule rule2 = new BucketLifecycleConfiguration.Rule() - .withId("abort-incomplete-mpu-temp") - .withPrefix("temp/") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule2.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(3)); - - BucketLifecycleConfiguration.Rule rule3 = new BucketLifecycleConfiguration.Rule() - .withId("abort-incomplete-mpu-no-prefix") - .withPrefix("") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule3.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(29)); - - rules.add(rule1); - rules.add(rule2); - rules.add(rule3); - configuration.setRules(rules); - - // Set lifecycle configuration - s3Client.setBucketLifecycleConfiguration(bucketName, configuration); - - // Get and verify the configuration - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - - assertEquals(3, retrievedConfig.getRules().size()); - - BucketLifecycleConfiguration.Rule retrievedRule1 = retrievedConfig.getRules().get(0); - assertEquals("abort-incomplete-mpu-with-prefix", retrievedRule1.getId()); - assertEquals("uploads/", retrievedRule1.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule1.getStatus()); - assertEquals(7, retrievedRule1.getAbortIncompleteMultipartUpload().getDaysAfterInitiation()); - - BucketLifecycleConfiguration.Rule retrievedRule2 = retrievedConfig.getRules().get(1); - assertEquals("abort-incomplete-mpu-temp", retrievedRule2.getId()); - assertEquals("temp/", retrievedRule2.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule2.getStatus()); - assertEquals(3, retrievedRule2.getAbortIncompleteMultipartUpload().getDaysAfterInitiation()); - - BucketLifecycleConfiguration.Rule retrievedRule3 = retrievedConfig.getRules().get(2); - assertEquals("abort-incomplete-mpu-no-prefix", retrievedRule3.getId()); - assertEquals("", retrievedRule3.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule3.getStatus()); - assertEquals(29, retrievedRule3.getAbortIncompleteMultipartUpload().getDaysAfterInitiation()); - } - - /** - * End-to-end test verifying that KeyLifecycleService correctly aborts - * incomplete multipart uploads based on lifecycle configuration. - */ - @Test - void testAbortIncompleteMultipartUploadE2E() throws Exception { - OzoneManager ozoneManager = cluster().getOzoneManager(); - KeyLifecycleService lifecycleService = ozoneManager.getKeyManager().getKeyLifecycleService(); - if (lifecycleService == null) { - return; - } - - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - String s3VolumeName; - try (OzoneClient ozoneClient = cluster().newClient()) { - s3VolumeName = ozoneClient.getObjectStore().getS3Volume().getName(); - } - - OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); - - // Create 3 MPUs - String matchingOldKey = "temp/old-file.txt"; - InitiateMultipartUploadResult mpu1 = s3Client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucketName, matchingOldKey)); - - String nonMatchingOldKey = "permanent/old-file.txt"; - InitiateMultipartUploadResult mpu2 = s3Client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucketName, nonMatchingOldKey)); - - String matchingRecentKey = "temp/recent-file.txt"; - s3Client.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, matchingRecentKey)); - - MultipartUploadListing listBefore = s3Client.listMultipartUploads( - new ListMultipartUploadsRequest(bucketName)); - assertEquals(3, listBefore.getMultipartUploads().size()); - - // Backdate 2 MPUs to 2 days ago - long oldCreationTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2); - updateMpuCreationTime(metadataManager, s3VolumeName, bucketName, - matchingOldKey, mpu1.getUploadId(), oldCreationTime); - updateMpuCreationTime(metadataManager, s3VolumeName, bucketName, - nonMatchingOldKey, mpu2.getUploadId(), oldCreationTime); - - // Set lifecycle rule: abort MPUs with prefix "temp/" after 1 day - BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule() - .withId("abort-temp-uploads") - .withPrefix("temp/") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(1)); - - BucketLifecycleConfiguration config = new BucketLifecycleConfiguration(); - config.setRules(Collections.singletonList(rule)); - s3Client.setBucketLifecycleConfiguration(bucketName, config); - - // Trigger lifecycle service - lifecycleService.runPeriodicalTaskNow(); - - // Wait for abort - GenericTestUtils.waitFor(() -> { - MultipartUploadListing listing = s3Client.listMultipartUploads( - new ListMultipartUploadsRequest(bucketName)); - return listing.getMultipartUploads().size() == 2; - }, 500, 30000); - - // Verify results - MultipartUploadListing listAfter = s3Client.listMultipartUploads( - new ListMultipartUploadsRequest(bucketName)); - List remainingKeys = listAfter.getMultipartUploads().stream() - .map(MultipartUpload::getKey) - .collect(Collectors.toList()); - - assertFalse(remainingKeys.contains(matchingOldKey), "Old MPU with matching prefix should be aborted"); - assertTrue(remainingKeys.contains(nonMatchingOldKey), "Old MPU with non-matching prefix should remain"); - assertTrue(remainingKeys.contains(matchingRecentKey), "Recent MPU should remain"); - } - - private void updateMpuCreationTime(OMMetadataManager metadataManager, - String volumeName, String bucketName, String keyName, - String uploadId, long newCreationTime) throws Exception { - String multipartKey = metadataManager.getMultipartKey(volumeName, bucketName, keyName, uploadId); - OmMultipartKeyInfo existingInfo = metadataManager.getMultipartInfoTable().get(multipartKey); - if (existingInfo == null) { - throw new RuntimeException("Multipart key info not found: " + multipartKey); - } - - OmMultipartKeyInfo updatedInfo = new OmMultipartKeyInfo.Builder() - .setUploadID(existingInfo.getUploadID()) - .setCreationTime(newCreationTime) - .setReplicationConfig(existingInfo.getReplicationConfig()) - .setObjectID(existingInfo.getObjectID()) - .setUpdateID(existingInfo.getUpdateID()) - .setParentID(existingInfo.getParentID()) - .build(); - - metadataManager.getMultipartInfoTable().put(multipartKey, updatedInfo); - } - - @Nested - @TestInstance(TestInstance.Lifecycle.PER_CLASS) - class PresignedUrlTests { - private static final String BUCKET_NAME = "v1-presigned-url-bucket"; - private static final String CONTENT = "bar"; - // Set the presigned URL to expire after one hour. - private final Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); - - @BeforeAll - public void setup() { - s3Client.createBucket(BUCKET_NAME); - } - - @Test - public void testPresignedUrlGet() throws IOException { - final String keyName = getKeyName(); - - InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); - - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - - // Generate the presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.GET).withExpiration(expiration); - generatePresignedUrlRequest.addRequestParameter("x-custom-parameter", "custom-value"); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - // Download the object using HttpUrlConnection (since v1.1) - // Capture the response body to a byte array. - HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); - // Download the result of executing the request. - try (InputStream s3is = connection.getInputStream(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) { - IOUtils.copy(s3is, bos); - assertEquals(CONTENT, bos.toString("UTF-8")); - } - } - - @ParameterizedTest - @ValueSource(strings = {"annotation", "torrent"}) - public void testPresignedUrlGetObjectSubresourceNotImplemented(String subresource) throws Exception { - final String keyName = getKeyName(); - - InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - - // AmazonS3 (SDK v1) has no getObjectTorrent or listObjectAnnotations API, so exercise the same HTTP behavior - // via a presigned URL with the subresource query parameter, as with other request shapes - // the typed v1 API doesn't expose. - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.GET).withExpiration(expiration); - generatePresignedUrlRequest.addRequestParameter(subresource, ""); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); - assertEquals(HttpURLConnection.HTTP_NOT_IMPLEMENTED, connection.getResponseCode()); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // object must be untouched - ObjectMetadata metadata = s3Client.getObjectMetadata(BUCKET_NAME, keyName); - assertEquals(CONTENT.length(), metadata.getContentLength()); - } - - @Test - public void testPresignedUrlHead() throws IOException { - final String keyName = getKeyName(); - - InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - - // Test HeadObject presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.HEAD).withExpiration(expiration); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "HEAD", null, null); - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, "HeadObject presigned URL should return 200 OK"); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // Test HeadBucket presigned URL - GeneratePresignedUrlRequest generateBucketPresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, null).withMethod(HttpMethod.HEAD).withExpiration(expiration); - URL presignedBucketUrl = s3Client.generatePresignedUrl(generateBucketPresignedUrlRequest); - - HttpURLConnection bucketConnection = null; - try { - bucketConnection = S3SDKTestUtils.openHttpURLConnection(presignedBucketUrl, "HEAD", null, null); - int bucketResponseCode = bucketConnection.getResponseCode(); - assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should return 200 OK"); - } finally { - if (bucketConnection != null) { - bucketConnection.disconnect(); - } - } - } - - @Test - public void testPresignedUrlPutObject() throws Exception { - final String keyName = getKeyName(); - - // Test PutObjectRequest presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT).withExpiration(expiration); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", - null, CONTENT.getBytes(StandardCharsets.UTF_8)); - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, "PutObject presigned URL should return 200 OK"); - String actualContent; - S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName); - try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { - actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } - assertEquals(CONTENT, actualContent, "Downloaded content should match uploaded content"); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - } - - @Test - public void testPresignedUrlPutSingleChunkWithWrongSha256() throws Exception { - final String keyName = getKeyName(); - - // Test PutObjectRequest presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT).withExpiration(expiration); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - Map> headers = new HashMap<>(); - List sha256Value = new ArrayList<>(); - sha256Value.add("wrong-sha256-value"); - headers.put("x-amz-content-sha256", sha256Value); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", - headers, CONTENT.getBytes(StandardCharsets.UTF_8)); - int responseCode = connection.getResponseCode(); - assertEquals(400, responseCode, "PutObject presigned URL should return 400 because of wrong SHA256"); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // Verify the object was not uploaded - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(BUCKET_NAME, keyName)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchKey", ase.getErrorCode()); - } - - @Test - public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception { - final String keyName = getKeyName(); - final Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); - final Map tags = new HashMap<>(); - tags.put("tag1", "value1"); - tags.put("tag2", "value2"); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - createFile(multipartUploadFile, (int) (10 * MB)); - - // create MPU using presigned URL - GeneratePresignedUrlRequest initMPUPresignedUrlRequest = - createInitMPUPresignedUrlRequest(keyName, userMetadata, tags); - - - String uploadId = initMultipartUpload(initMPUPresignedUrlRequest); - - // upload parts using presigned URL - List completedParts = uploadParts(multipartUploadFile, keyName, uploadId); - - // Complete multipart upload using presigned URL - completeMPU(keyName, uploadId, completedParts); - - // Verify upload result - ObjectMetadata objectMeta = s3Client.getObjectMetadata(BUCKET_NAME, keyName); - assertEquals(userMetadata, objectMeta.getUserMetadata()); - - // Verify content - S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName); - assertEquals(tags.size(), s3Object.getTaggingCount()); - String actualContent; - try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { - actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } - String expectedContent = new String(Files.readAllBytes(multipartUploadFile.toPath()), StandardCharsets.UTF_8); - assertEquals(expectedContent, actualContent, "Downloaded content should match uploaded content"); - } - - private void completeMPU(String keyName, String uploadId, List completedParts) throws IOException { - GeneratePresignedUrlRequest request = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.POST).withExpiration(expiration); - request.addRequestParameter("uploadId", uploadId); - - URL presignedUrl = s3Client.generatePresignedUrl(request); - - // Generate completion XML payload - StringBuilder completionXml = new StringBuilder(); - completionXml.append("\n"); - for (PartETag part : completedParts) { - completionXml.append(" \n") - .append(" ").append(part.getPartNumber()).append("\n") - .append(" ").append(stripQuotes(part.getETag())).append("\n") - .append(" \n"); - } - completionXml.append(""); - - byte[] completionPayloadBytes = completionXml.toString().getBytes(StandardCharsets.UTF_8); - - HttpURLConnection httpConnection = null; - try { - httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "POST", null, completionPayloadBytes); - int responseCode = httpConnection.getResponseCode(); - assertEquals(200, responseCode, "Complete multipart upload should return 200 OK"); - } finally { - if (httpConnection != null) { - httpConnection.disconnect(); - } - } - } - - private List uploadParts(File multipartUploadFile, String keyName, String uploadId) throws IOException { - List completedParts = new ArrayList<>(); - ByteBuffer byteBuffer = ByteBuffer.allocate((int) (5 * MB)); - long filePosition = 0; - long fileLength = multipartUploadFile.length(); - int partNumber = 1; - - try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { - while (filePosition < fileLength) { - file.seek(filePosition); - long bytesRead = file.getChannel().read(byteBuffer); - byteBuffer.flip(); - - // generate presigned URL for each part - GeneratePresignedUrlRequest request = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT) - .withExpiration(expiration); - request.addRequestParameter("partNumber", String.valueOf(partNumber)); - request.addRequestParameter("uploadId", uploadId); - - URL presignedUrl = s3Client.generatePresignedUrl(request); - - // upload each part using presigned URL - HttpURLConnection connection = null; - try { - Map> headers = new HashMap<>(); - List header = Collections.singletonList(String.valueOf(byteBuffer.remaining())); - headers.put("Content-Length", header); - byte[] body = new byte[byteBuffer.remaining()]; - byteBuffer.get(body); - - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", headers, body); - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, String.format("Upload part %d should return 200 OK", partNumber)); - - String etag = connection.getHeaderField("ETag"); - PartETag partETag = new PartETag(partNumber, etag); - completedParts.add(partETag); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - byteBuffer.clear(); - filePosition += bytesRead; - partNumber++; - } - } - return completedParts; - } - - private String initMultipartUpload(GeneratePresignedUrlRequest request) throws IOException { - URL presignedUrl = s3Client.generatePresignedUrl(request); - String uploadId; - HttpURLConnection httpConnection = null; - try { - Map customRequestHeaders = request.getCustomRequestHeaders(); - Map> headers = new HashMap<>(); - if (customRequestHeaders != null) { - for (Map.Entry entry : customRequestHeaders.entrySet()) { - headers.put(entry.getKey(), Collections.singletonList(entry.getValue())); - } - } - httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "POST", headers, null); - int initMPUConnectionResponseCode = httpConnection.getResponseCode(); - assertEquals(200, initMPUConnectionResponseCode); - - try (InputStream is = httpConnection.getInputStream()) { - String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); - uploadId = S3SDKTestUtils.extractUploadId(responseXml); - } - } finally { - if (httpConnection != null) { - httpConnection.disconnect(); - } - } - return uploadId; - } - - private GeneratePresignedUrlRequest createInitMPUPresignedUrlRequest(String keyName, - Map userMetadata, - Map tags) throws Exception { - GeneratePresignedUrlRequest initMPUPresignUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.POST).withExpiration(expiration); - - userMetadata.forEach((k, v) -> { - initMPUPresignUrlRequest.putCustomRequestHeader(CUSTOM_METADATA_HEADER_PREFIX + k, v); - }); - - StringBuilder tagValueBuilder = new StringBuilder(); - for (Map.Entry entry : tags.entrySet()) { - if (tagValueBuilder.length() > 0) { - tagValueBuilder.append('&'); - } - tagValueBuilder.append(entry.getKey()).append('=').append(URLEncoder.encode(entry.getValue(), "UTF-8")); - } - initMPUPresignUrlRequest.putCustomRequestHeader(S3Consts.TAG_HEADER, tagValueBuilder.toString()); - return initMPUPresignUrlRequest; - } - - @Test - public void testPresignedUrlDelete() throws IOException { - final String keyName = getKeyName(); - - try (InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8))) { - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - } - - // Generate the presigned URL for DELETE - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.DELETE) - .withExpiration(expiration); - URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - // Execute the DELETE request using HttpUrlConnection - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(url, "DELETE", null, null); - int responseCode = connection.getResponseCode(); - // Verify the response code is 204 (No Content) - assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // Verify the object is deleted - assertFalse(s3Client.doesObjectExist(BUCKET_NAME, keyName)); - } - } - - /** - * Tests the functionality to create a snapshot of an Ozone bucket and then read files - * from the snapshot directory using the S3 SDK. - * - *

The test follows these steps: - *

    - *
  1. Create a bucket and upload a file via the S3 client.
    2. - *
  2. Create a snapshot on the bucket using the Ozone client.
    3. - *
  3. Construct the snapshot object key using the ".snapshot" directory format.
    4. - *
  4. Retrieve the object from the snapshot and verify that its content matches - * the originally uploaded content.
    5. - *
- *

- * - * @throws Exception if the test fails due to any errors during bucket creation, snapshot creation, - * file upload, or retrieval. - */ - @Test - public void testReadSnapshotDirectoryUsingS3SDK() throws Exception { - final String bucketName = getBucketName("snapshot"); - final String keyName = getKeyName("snapshotfile"); - final String content = "snapshot test content"; - final byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - - // Create the bucket and upload an object using S3 SDK. - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, - is, new ObjectMetadata()); - - String snapshotName = "snap1"; - // Create a snapshot using the Ozone client. - // Snapshots in Ozone are created on the bucket and are exposed via the ".snapshot" directory. - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - OzoneVolume volume = store.getS3Volume(); - - store.createSnapshot(volume.getName(), bucketName, snapshotName); - } - - // Use the S3 SDK to read the file from the snapshot directory. - // The key in the snapshot is constructed using the special ".snapshot" prefix. - String snapshotKey = ".snapshot/" + snapshotName + "/" + keyName; - - S3Object s3Object = s3Client.getObject(bucketName, snapshotKey); - try (S3ObjectInputStream s3is = s3Object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(contentBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(content, bos.toString("UTF-8")); - } - } - - private boolean isBucketEmpty(Bucket bucket) { - ObjectListing objectListing = s3Client.listObjects(bucket.getName()); - return objectListing.getObjectSummaries().isEmpty(); - } - - private String getBucketName() { - return getBucketName(""); - } - - private String getBucketName(String ignored) { - return uniqueObjectName(); - } - - private String getKeyName() { - return getKeyName(""); - } - - private String getKeyName(String ignored) { - return uniqueObjectName(); - } - - private static void assertNotImplemented(Executable request) { - final AmazonServiceException ase = assertThrows(AmazonServiceException.class, request); - assertEquals(S3ErrorTable.NOT_IMPLEMENTED.getCode(), ase.getErrorCode()); - assertEquals(501, ase.getStatusCode()); - assertEquals(ErrorType.Service, ase.getErrorType()); - } - - private String multipartUpload(String bucketName, String key, File file, long partSize, String contentType, - Map userMetadata, List tags) throws Exception { - String uploadId = initiateMultipartUpload(bucketName, key, contentType, userMetadata, tags); - - List partETags = uploadParts(bucketName, key, uploadId, file, partSize); - - completeMultipartUpload(bucketName, key, uploadId, partETags); - - return uploadId; - } - - private String initiateMultipartUpload(String bucketName, String key, String contentType, - Map metadata, List tags) { - InitiateMultipartUploadRequest initRequest; - if (metadata == null || metadata.isEmpty()) { - initRequest = new InitiateMultipartUploadRequest(bucketName, key); - } else { - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setUserMetadata(metadata); - if (contentType != null) { - objectMetadata.setContentType(contentType); - } - - initRequest = new InitiateMultipartUploadRequest(bucketName, key, objectMetadata) - .withTagging(new ObjectTagging(tags)); - } - - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - assertEquals(bucketName, initResponse.getBucketName()); - assertEquals(key, initResponse.getKey()); - // TODO: Once bucket lifecycle configuration is supported, should check for "abortDate" and "abortRuleId" - - return initResponse.getUploadId(); - } - - // TODO: Also support async upload parts (similar to v2 asyncClient) - private List uploadParts(String bucketName, String key, String uploadId, File file, long partSize) - throws Exception { - // Create a list of ETag objects. You retrieve ETags for each object part - // uploaded, - // then, after each individual part has been uploaded, pass the list of ETags to - // the request to complete the upload. - List partETags = new ArrayList<>(); - - // Upload the file parts. - long filePosition = 0; - long fileLength = file.length(); - try (InputStream fileInputStream = Files.newInputStream(file.toPath())) { - for (int i = 1; filePosition < fileLength; i++) { - // Because the last part could be less than 5 MB, adjust the part size as - // needed. - partSize = Math.min(partSize, (fileLength - filePosition)); - - // Create the request to upload a part. - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(key) - .withUploadId(uploadId) - .withPartNumber(i) - .withFileOffset(filePosition) - .withFile(file) - .withPartSize(partSize); - - // Upload the part and add the response's ETag to our list. - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - PartETag partETag = uploadResult.getPartETag(); - assertEquals(i, partETag.getPartNumber()); - assertEquals(DatatypeConverter.printHexBinary( - calculateDigest(fileInputStream, 0, (int) partSize)).toLowerCase(), partETag.getETag()); - partETags.add(partETag); - - filePosition += partSize; - } - } - - return partETags; - } - - private void completeMultipartUpload(String bucketName, String key, String uploadId, List partETags) { - CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, key, - uploadId, partETags); - CompleteMultipartUploadResult compResponse = s3Client.completeMultipartUpload(compRequest); - assertEquals(bucketName, compResponse.getBucketName()); - assertEquals(key, compResponse.getKey()); - } - - private void abortMultipartUpload(String bucketName, String key, String uploadId) { - AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId); - s3Client.abortMultipartUpload(abortRequest); - } -} diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 60a88af23777..5f2cf37c7df2 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -63,18 +63,22 @@ import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import javax.xml.bind.DatatypeConverter; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; @@ -169,6 +173,8 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; import software.amazon.awssdk.services.s3.model.GetObjectTaggingResponse; +import software.amazon.awssdk.services.s3.model.Grant; +import software.amazon.awssdk.services.s3.model.Grantee; import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -188,12 +194,19 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.ListPartsRequest; +import software.amazon.awssdk.services.s3.model.ListPartsResponse; import software.amazon.awssdk.services.s3.model.MetadataDirective; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.NoSuchUploadException; import software.amazon.awssdk.services.s3.model.ObjectAttributes; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; import software.amazon.awssdk.services.s3.model.ObjectVersion; +import software.amazon.awssdk.services.s3.model.Owner; +import software.amazon.awssdk.services.s3.model.Part; +import software.amazon.awssdk.services.s3.model.Permission; import software.amazon.awssdk.services.s3.model.PutBucketAclRequest; import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -203,6 +216,7 @@ import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.Tag; import software.amazon.awssdk.services.s3.model.Tagging; +import software.amazon.awssdk.services.s3.model.Type; import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; import software.amazon.awssdk.services.s3.model.UploadPartCopyResponse; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -225,6 +239,7 @@ import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest; import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedFileUpload; import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; @@ -254,6 +269,7 @@ public abstract class AbstractS3SDKV2Tests extends OzoneTestBase implements Clus */ public static final String EC_PARALLEL_READ_S3G_HTTP_ADDRESS_KEY = "ozone.test.s3g.ec.parallel.read.http.address"; + private static final int MAX_UPLOADS_LIMIT = 1000; private static final String READ_CONSISTENCY_HEADER = "x-ozone-read-consistency"; private static final String LOCAL_LEASE_LOG_LIMIT_HEADER = @@ -314,6 +330,189 @@ public void testCreateBucketAlreadyExistsDifferentOwner() throws IOException { exception.awsErrorDetails().errorCode()); } + @Test + public void testCreateBucket() { + final String bucketName = getBucketName(); + + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.headBucket(b -> b.bucket(bucketName)); + s3Client.getBucketAcl(b -> b.bucket(bucketName)); + assertThat(s3Client.listObjectsV2(b -> b.bucket(bucketName)).contents()).isEmpty(); + } + + @Test + public void testBucketAclOperations() { + // TODO HDDS-11738: Assert the bucket ACL when bucket S3 ACL logic has been fixed + final String bucketName = getBucketName(); + final Owner owner = Owner.builder().id("owner").displayName("owner").build(); + final Grantee grantee = Grantee.builder().type(Type.CANONICAL_USER).id("testGrantee").build(); + + s3Client.createBucket(b -> b.bucket(bucketName).grantRead("id=testGrantee")); + + s3Client.putBucketAcl(b -> b.bucket(bucketName).accessControlPolicy(p -> p.owner(owner).grants( + Grant.builder().grantee(grantee).permission(Permission.READ).build(), + Grant.builder().grantee(grantee).permission(Permission.WRITE).build()))); + } + + @Test + public void testDeleteBucket() { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.deleteBucket(b -> b.bucket(bucketName)); + + final S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.headBucket(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + } + + @Test + public void testDeleteBucketNotExist() { + final String bucketName = getBucketName(); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.deleteBucket(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testDeleteBucketNonEmptyWithKeys() { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + for (int i = 1; i <= 10; i++) { + final String keyName = "key-" + i; + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(RandomStringUtils.secure().nextAlphanumeric(1024))); + } + + // Bucket deletion should fail if there are still keys in the bucket + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.deleteBucket(b -> b.bucket(bucketName))); + assertEquals(409, exception.statusCode()); + assertEquals(S3ErrorTable.BUCKET_NOT_EMPTY.getCode(), exception.awsErrorDetails().errorCode()); + + for (S3Object object : s3Client.listObjectsV2Paginator(b -> b.bucket(bucketName)).contents()) { + s3Client.deleteObject(b -> b.bucket(bucketName).key(object.key())); + } + } + + @Test + public void testDeleteBucketNonEmptyWithIncompleteMultipartUpload(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (5 * MB)); + + // An incomplete multipart upload: parts uploaded, but not completed + final String uploadId = initiateMultipartUpload(bucketName, keyName, new HashMap<>(), Collections.emptyList()); + uploadParts(bucketName, keyName, uploadId, multipartUploadFile, (int) MB); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.deleteBucket(b -> b.bucket(bucketName))); + assertEquals(409, exception.statusCode()); + assertEquals(S3ErrorTable.BUCKET_NOT_EMPTY.getCode(), exception.awsErrorDetails().errorCode()); + + // After the multipart upload is aborted, the bucket deletion should succeed + s3Client.abortMultipartUpload(b -> b.bucket(bucketName).key(keyName).uploadId(uploadId)); + s3Client.deleteBucket(b -> b.bucket(bucketName)); + assertThrows(NoSuchBucketException.class, () -> s3Client.headBucket(b -> b.bucket(bucketName))); + } + + @Test + public void testQuotaExceeded() throws IOException { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + try (OzoneClient ozoneClient = cluster.newClient()) { + ozoneClient.getObjectStore().getS3Volume().getBucket(bucketName).setQuota(OzoneQuota.parseQuota("1", "10")); + } + + final String content = RandomStringUtils.secure().nextAlphanumeric(1024); + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content))); + assertEquals(403, exception.statusCode()); + assertEquals(S3ErrorTable.QUOTA_EXCEEDED.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testGetObject() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + final Map userMetadata = new HashMap<>(); + userMetadata.put("key1", "value1"); + userMetadata.put("key2", "value2"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName).metadata(userMetadata) + .tagging("tag1=value1&tag2=value2"), RequestBody.fromString(content)); + + final ResponseBytes object = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)); + assertEquals(2, object.response().tagCount()); + assertEquals(userMetadata, object.response().metadata()); + assertEquals(content, object.asUtf8String()); + } + + @Test + public void testGetObjectWithoutETag() throws Exception { + // Object uploaded using other protocols (e.g. ofs / ozone cli) will not have ETag. + // Ensure that GetObject does not validate the ETag if there is none. + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String value = "sample value"; + final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); + s3Client.createBucket(b -> b.bucket(bucketName)); + try (OzoneClient ozoneClient = cluster.newClient()) { + final OzoneBucket bucket = ozoneClient.getObjectStore().getS3Volume().getBucket(bucketName); + try (OzoneOutputStream out = bucket.createKey(keyName, valueBytes.length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), Collections.emptyMap())) { + out.write(valueBytes); + } + } + + final ResponseBytes object = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)); + assertNull(object.response().eTag()); + assertEquals(value, object.asUtf8String()); + } + + @Test + public void testPutDoubleSlashPrefixObject() throws Exception { + final String bucketName = getBucketName(); + final String keyName = "//dir1"; + final String content = "bar"; + createFsoBucket(bucketName); + + final PutObjectResponse putObjectResponse = s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", putObjectResponse.eTag()); + + final ResponseBytes object = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)); + assertEquals(content.length(), object.response().contentLength()); + assertEquals(content, object.asUtf8String()); + } + + @Test + public void testPutObjectAcl() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + final String originalETag = s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)).eTag(); + final Owner owner = Owner.builder().id("owner").displayName("owner").build(); + final Grantee grantee = Grantee.builder().type(Type.CANONICAL_USER).id("testGrantee").build(); + + assertNotImplemented(() -> s3Client.putObjectAcl(b -> b.bucket(bucketName).key(keyName) + .accessControlPolicy(p -> p.owner(owner) + .grants(Grant.builder().grantee(grantee).permission(Permission.READ).build())))); + + // Ensure that the object remains unchanged + assertEquals(originalETag, s3Client.headObject(b -> b.bucket(bucketName).key(keyName)).eTag()); + assertEquals(content, s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)).asUtf8String()); + } + @Test public void testPutObject() { final String bucketName = getBucketName(); @@ -2960,6 +3159,52 @@ public void tearDown() { sdkHttpClient.close(); } + @ParameterizedTest + @ValueSource(strings = {"annotation", "torrent"}) + public void testPresignedUrlGetObjectSubresourceNotImplemented(String subresource) throws Exception { + final String keyName = getKeyName(); + s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName), RequestBody.fromString(CONTENT)); + + // a presigned GetObject with the subresource query parameter, as clients without the typed API send it + final GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(BUCKET_NAME).key(keyName) + .overrideConfiguration(o -> o.putRawQueryParameter(subresource, "")) + .build(); + final URL presignedUrl = presigner.presignGetObject(b -> b.signatureDuration(duration) + .getObjectRequest(objectRequest)).url(); + assertThat(presignedUrl.getQuery()).contains(subresource); + + final HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); + try { + assertEquals(HttpURLConnection.HTTP_NOT_IMPLEMENTED, connection.getResponseCode()); + } finally { + connection.disconnect(); + } + + // the object must be untouched + assertEquals(CONTENT.length(), s3Client.headObject(b -> b.bucket(BUCKET_NAME).key(keyName)).contentLength()); + } + + @Test + public void testPresignedUrlGetWithCustomQueryParameter() throws Exception { + final String keyName = getKeyName(); + s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName), RequestBody.fromString(CONTENT)); + + // a query parameter unknown to S3, signed with the request + final GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(BUCKET_NAME).key(keyName) + .overrideConfiguration(o -> o.putRawQueryParameter("x-custom-parameter", "custom-value")) + .build(); + final URL presignedUrl = presigner.presignGetObject(b -> b.signatureDuration(duration) + .getObjectRequest(objectRequest)).url(); + assertThat(presignedUrl.getQuery()).contains("x-custom-parameter=custom-value"); + + final HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); + try (InputStream s3is = connection.getInputStream()) { + assertEquals(CONTENT, IoUtils.toUtf8String(s3is)); + } finally { + connection.disconnect(); + } + } + @Test public void testPresignedUrlGet() throws Exception { final String keyName = getKeyName(); @@ -3958,6 +4203,270 @@ private void updateMpuCreationTime(OMMetadataManager metadataManager, metadataManager.getMultipartInfoTable().put(multipartKey, updatedInfo); } + @Test + public void testListObjectsSpecialKeyNames() { + final String bucketName = getBucketName("special-keys"); + final String content = "x"; + s3Client.createBucket(b -> b.bucket(bucketName)); + for (String keyName : S3SDKTestUtils.S3_SPECIAL_KEY_NAMES) { + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + } + + // ListObjects (the original API, not ListObjectsV2) + final ListObjectsResponse response = s3Client.listObjects(b -> b.bucket(bucketName)); + assertEquals(S3SDKTestUtils.S3_SPECIAL_KEY_NAMES, + response.contents().stream().map(S3Object::key).collect(Collectors.toList())); + } + + @Test + public void testListObjectsBucketNotExist() { + final String bucketName = getBucketName(); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.listObjects(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testListObjectsV2BucketNotExist() { + final String bucketName = getBucketName(); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.listObjectsV2(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testHighLevelMultipartUpload(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final long fileSize = 20 * MB; + final long partSize = 5 * MB; + s3Client.createBucket(b -> b.bucket(bucketName)); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) fileSize); + + // the transfer manager uploads in parallel parts when its client has multipart upload enabled + try (S3AsyncClient multipartClient = S3AsyncClient.builder() + .multipartEnabled(true) + .multipartConfiguration(c -> c.thresholdInBytes(partSize).minimumPartSizeInBytes(partSize)) + .forcePathStyle(true) + .disableS3ExpressSessionAuth(true) + .endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get()) + .region(s3Client.serviceClientConfiguration().region()) + .credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()) + .build(); + S3TransferManager transferManager = S3TransferManager.builder().s3Client(multipartClient).build()) { + final CompletedFileUpload upload = transferManager.uploadFile(b -> b + .putObjectRequest(r -> r.bucket(bucketName).key(keyName)) + .source(multipartUploadFile)).completionFuture().get(); + + // a multipart upload ETag ends with the number of parts + assertThat(upload.response().eTag()).endsWith("-" + fileSize / partSize + "\""); + } + assertEquals(fileSize, s3Client.headObject(b -> b.bucket(bucketName).key(keyName)).contentLength()); + } + + @Test + public void testListMultipartUploads() { + final String bucketName = getBucketName(); + final String multipartKey1 = getKeyName("multipart1"); + final String multipartKey2 = getKeyName("multipart2"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + final List uploadIds = new ArrayList<>(); + uploadIds.add(initiateMultipartUpload(bucketName, multipartKey1, new HashMap<>(), Collections.emptyList())); + uploadIds.add(initiateMultipartUpload(bucketName, multipartKey1, new HashMap<>(), Collections.emptyList())); + uploadIds.add(initiateMultipartUpload(bucketName, multipartKey2, new HashMap<>(), Collections.emptyList())); + + final ListMultipartUploadsResponse response = s3Client.listMultipartUploads(b -> b + .bucket(bucketName).maxUploads(5000)); + assertEquals(uploadIds, response.uploads().stream().map(MultipartUpload::uploadId).collect(Collectors.toList())); + } + + @ParameterizedTest + @ValueSource(ints = {10, 5000}) + public void testListMultipartUploadsPagination(int requestedMaxUploads) { + final String bucketName = getBucketName() + "-" + requestedMaxUploads; + final String multipartKeyPrefix = getKeyName("multipart"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + final List allKeys = new ArrayList<>(); + final Map keyToUploadId = new HashMap<>(); + final int effectiveMaxUploads = Math.min(requestedMaxUploads, MAX_UPLOADS_LIMIT); + final int uploadsCreated = 2 * effectiveMaxUploads + 5; + final int expectedPages = uploadsCreated / effectiveMaxUploads + 1; + for (int i = 0; i < uploadsCreated; i++) { + final String key = String.format("%s-%04d", multipartKeyPrefix, i); + allKeys.add(key); + keyToUploadId.put(key, initiateMultipartUpload(bucketName, key, new HashMap<>(), Collections.emptyList())); + } + Collections.sort(allKeys); + + final Set retrievedKeys = new HashSet<>(); + String keyMarker = null; + String uploadIdMarker = null; + boolean truncated; + int pageCount = 0; + do { + final String requestKeyMarker = keyMarker; + final String requestUploadIdMarker = uploadIdMarker; + final ListMultipartUploadsResponse result = s3Client.listMultipartUploads(b -> b.bucket(bucketName) + .maxUploads(requestedMaxUploads).keyMarker(requestKeyMarker).uploadIdMarker(requestUploadIdMarker)); + pageCount++; + + if (pageCount < expectedPages) { + assertEquals(effectiveMaxUploads, result.uploads().size()); + assertTrue(result.isTruncated()); + } else { + assertEquals(uploadsCreated % effectiveMaxUploads, result.uploads().size()); + assertFalse(result.isTruncated()); + } + for (MultipartUpload upload : result.uploads()) { + retrievedKeys.add(upload.key()); + assertEquals(keyToUploadId.get(upload.key()), upload.uploadId()); + } + + assertThat(result.prefix()).isNullOrEmpty(); + assertEquals(requestUploadIdMarker, emptyToNull(result.uploadIdMarker())); + assertEquals(requestKeyMarker, emptyToNull(result.keyMarker())); + assertEquals(effectiveMaxUploads, result.maxUploads()); + if (result.isTruncated()) { + final MultipartUpload lastUploadOnPage = result.uploads().get(result.uploads().size() - 1); + assertEquals(lastUploadOnPage.key(), result.nextKeyMarker()); + assertEquals(lastUploadOnPage.uploadId(), result.nextUploadIdMarker()); + } else { + assertNull(emptyToNull(result.nextKeyMarker())); + assertNull(emptyToNull(result.nextUploadIdMarker())); + } + + keyMarker = result.nextKeyMarker(); + uploadIdMarker = result.nextUploadIdMarker(); + truncated = result.isTruncated(); + } while (truncated); + + assertEquals(expectedPages, pageCount); + assertEquals(uploadsCreated, retrievedKeys.size(), "Should retrieve all uploads"); + assertEquals(allKeys, retrievedKeys.stream().sorted().collect(Collectors.toList()), + "Retrieved keys should match expected keys in order"); + + final String prefix = multipartKeyPrefix + "-001"; + final ListMultipartUploadsResponse prefixResult = s3Client.listMultipartUploads(b -> b + .bucket(bucketName).prefix(prefix)); + assertEquals(prefix, prefixResult.prefix()); + assertEquals(IntStream.rangeClosed(0, 9).mapToObj(i -> prefix + i).collect(Collectors.toList()), + prefixResult.uploads().stream().map(MultipartUpload::key).collect(Collectors.toList())); + } + + @Test + public void testListMultipartUploadsPaginationCornerCases() { + final String bucketName = getBucketName(); + final String keyA = getKeyName("samekey"); + final String keyB = getKeyName("after"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + // Several uploads for the same key, to verify the upload-id-marker semantics + final List keyAUploadIds = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, new HashMap<>(), Collections.emptyList())); + } + // Another key, to ensure that the listing proceeds past keyA + initiateMultipartUpload(bucketName, keyB, new HashMap<>(), Collections.emptyList()); + // Uploads of the same key are listed in upload id order + Collections.sort(keyAUploadIds); + + // upload-id-marker at the lowest upload id: only the larger upload ids of keyA are returned + final List sameKeyIds = s3Client.listMultipartUploads(b -> b.bucket(bucketName) + .keyMarker(keyA).uploadIdMarker(keyAUploadIds.get(0)).maxUploads(100)) + .uploads().stream().filter(u -> keyA.equals(u.key())).map(MultipartUpload::uploadId) + .collect(Collectors.toList()); + assertThat(sameKeyIds).allSatisfy(id -> assertThat(id).isGreaterThan(keyAUploadIds.get(0))); + + // upload-id-marker at the highest upload id: no upload of keyA is returned + assertThat(s3Client.listMultipartUploads(b -> b.bucket(bucketName) + .keyMarker(keyA).uploadIdMarker(keyAUploadIds.get(2)).maxUploads(100)).uploads()) + .noneMatch(u -> keyA.equals(u.key())); + } + + @Test + public void testListParts(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final int maxParts = 2; + s3Client.createBucket(b -> b.bucket(bucketName)); + final String uploadId = initiateMultipartUpload(bucketName, keyName, new HashMap<>(), Collections.emptyList()); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (5 * MB)); + final List uploadedParts = uploadParts(bucketName, keyName, uploadId, multipartUploadFile, + (int) MB); + + final List listedParts = new ArrayList<>(); + int partNumberMarker = 0; + int remainingParts = uploadedParts.size(); + ListPartsResponse response; + do { + final int marker = partNumberMarker; + response = s3Client.listParts(b -> b.bucket(bucketName).key(keyName).uploadId(uploadId) + .maxParts(maxParts).partNumberMarker(marker)); + if (remainingParts > maxParts) { + assertTrue(response.isTruncated()); + partNumberMarker = response.nextPartNumberMarker(); + remainingParts -= maxParts; + } else { + assertFalse(response.isTruncated()); + } + listedParts.addAll(response.parts()); + } while (response.isTruncated()); + + assertEquals(uploadedParts.size(), listedParts.size()); + for (int i = 0; i < uploadedParts.size(); i++) { + assertEquals(uploadedParts.get(i).partNumber(), listedParts.get(i).partNumber()); + assertEquals(uploadedParts.get(i).eTag(), stripQuotes(listedParts.get(i).eTag())); + } + } + + @Test + public void testListPartsNotFound() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + final S3Exception exception = assertThrows(NoSuchUploadException.class, + () -> s3Client.listParts(b -> b.bucket(bucketName).key(keyName).uploadId("nonexist"))); + assertEquals(404, exception.statusCode()); + } + + @Test + public void testGetParticularPart(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (15 * MB)); + multipartUpload(bucketName, keyName, multipartUploadFile, (int) (5 * MB), new HashMap<>(), Collections.emptyList()); + + final long allPartsLength = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)) + .response().contentLength(); + // part number 0 is not a part: the whole object is returned + assertEquals(allPartsLength, s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName).partNumber(0)) + .response().contentLength()); + long partsLength = 0; + for (int part = 1; part <= 3; part++) { + final int partNumber = part; + final long partLength = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName).partNumber(partNumber)) + .response().contentLength(); + assertEquals(allPartsLength / 3, partLength, "part " + partNumber); + partsLength += partLength; + } + assertEquals(allPartsLength, partsLength); + } + + private static String emptyToNull(String value) { + return value == null || value.isEmpty() ? null : value; + } + private String getBucketName() { return getBucketName(""); } diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties b/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties index 977e356b7ad3..27935677a2fa 100644 --- a/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties +++ b/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties @@ -23,8 +23,5 @@ log4j.logger.org.apache.hadoop.hdds.utils.db.managed=TRACE log4j.logger.org.apache.hadoop.hdds.utils.db.CodecBuffer=DEBUG log4j.logger.org.apache.hadoop.ozone.client.OzoneClientFactory=DEBUG -log4j.logger.com.amazonaws=WARN -log4j.logger.com.amazonaws.request=DEBUG - log4j.logger.software.amazon.awssdk=WARN log4j.logger.software.amazon.awssdk.request=DEBUG diff --git a/pom.xml b/pom.xml index 77991cabdf0b..af831d2544f2 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,6 @@ 1.9.25.1 1.9.7 3.27.7 - 1.12.788 2.55.6 0.8.0.RELEASE 1.86 @@ -363,23 +362,6 @@ - - com.amazonaws - aws-java-sdk-core - ${aws-java-sdk.version} - - - - commons-logging - commons-logging - - - - - com.amazonaws - aws-java-sdk-s3 - ${aws-java-sdk.version} - com.bettercloud vault-java-driver @@ -2126,6 +2108,23 @@ + + ban-aws-sdk-v1 + + enforce + + validate + + + + + com.amazonaws + + AWS SDK for Java v1 is end of life, use AWS SDK for Java v2 (software.amazon.awssdk) + + + + ban-imports @@ -2199,6 +2198,17 @@ ${project.build.directory}/generated-sources/protobuf/java + + true + AWS SDK for Java v1 is end of life, use AWS SDK for Java v2 (software.amazon.awssdk) + + com.amazonaws.** + + + ${project.build.directory}/generated-sources/java + ${project.build.directory}/generated-sources/protobuf/java + + true true