From 1db9b86d25805af49017e5736bdb37f3d1a9dd52 Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Wed, 4 Nov 2026 15:22:24 -0500 Subject: [PATCH 1/3] HDDS-17707. Remove AWS SDK for Java v1 Co-authored-by: Claude Opus 5.5 --- .../dist/src/main/license/bin/LICENSE.txt | 35 +- .../dist/src/main/license/bin/NOTICE.txt | 30 + .../dist/src/main/license/jar-report.txt | 45 +- .../dist/src/shell/conf/log4j.properties | 4 +- hadoop-ozone/freon/pom.xml | 51 +- ...va => FreonS3TraceContextInterceptor.java} | 19 +- .../hadoop/ozone/freon/S3BucketGenerator.java | 2 +- .../hadoop/ozone/freon/S3EntityGenerator.java | 40 +- .../hadoop/ozone/freon/S3KeyGenerator.java | 53 +- .../TestFreonS3TraceContextInterceptor.java | 73 + hadoop-ozone/integration-test-s3/pom.xml | 10 - .../hadoop/ozone/s3/S3ClientFactory.java | 60 - .../ozone/s3/awssdk/OzoneS3SDKTests.java | 9 - .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 3167 ----------------- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 481 +++ .../src/test/resources/log4j.properties | 3 - pom.xml | 46 +- 17 files changed, 776 insertions(+), 3352 deletions(-) rename hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/{FreonS3TraceContextRequestHandler.java => FreonS3TraceContextInterceptor.java} (64%) create mode 100644 hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java delete mode 100644 hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index 5dab10019565..b8c36618f2a1 100644 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@ -266,15 +266,10 @@ Apache License 2.0 ===================== ch.qos.reload4j:reload4j - com.amazonaws:aws-java-sdk-core - com.amazonaws:aws-java-sdk-kms - com.amazonaws:aws-java-sdk-s3 - com.amazonaws:jmespath-java com.carrotsearch:hppc com.fasterxml.jackson.core:jackson-annotations com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.dataformat:jackson-dataformat-cbor com.fasterxml.jackson.dataformat:jackson-dataformat-xml com.fasterxml.jackson.datatype:jackson-datatype-jsr310 com.fasterxml.jackson.jaxrs:jackson-jaxrs-base @@ -488,6 +483,36 @@ Apache License 2.0 org.xerial.snappy:snappy-java org.xerial:sqlite-jdbc org.yaml:snakeyaml + software.amazon.awssdk:annotations + software.amazon.awssdk:apache-client + software.amazon.awssdk:arns + software.amazon.awssdk:auth + software.amazon.awssdk:aws-core + software.amazon.awssdk:aws-query-protocol + software.amazon.awssdk:aws-xml-protocol + software.amazon.awssdk:checksums + software.amazon.awssdk:checksums-spi + software.amazon.awssdk:crt-core + software.amazon.awssdk:endpoints-spi + software.amazon.awssdk:http-auth + software.amazon.awssdk:http-auth-aws + software.amazon.awssdk:http-auth-aws-eventstream + software.amazon.awssdk:http-auth-spi + software.amazon.awssdk:http-client-spi + software.amazon.awssdk:identity-spi + software.amazon.awssdk:json-utils + software.amazon.awssdk:metrics-spi + software.amazon.awssdk:profiles + software.amazon.awssdk:protocol-core + software.amazon.awssdk:regions + software.amazon.awssdk:retries + software.amazon.awssdk:retries-spi + software.amazon.awssdk:s3 + software.amazon.awssdk:sdk-core + software.amazon.awssdk:third-party-jackson-core + software.amazon.awssdk:utils + software.amazon.awssdk:utils-lite + software.amazon.eventstream:eventstream MIT ===================== diff --git a/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt b/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt index dd78c02888cf..c5c1fda48840 100644 --- a/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt @@ -448,6 +448,36 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +software.amazon.awssdk +==================== + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + + ==================== ratis-thirdparty-misc is a shaded dependency which includes additional 3rd party dependencies in shaded form. For the detailed list of the dependencies and the associated NOTICE file see licenses/NOTICE-ratis-thirdparty-misc.txt. diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 886623bde38f..28874de0aa07 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -1,26 +1,34 @@ share/ozone/lib/aircompressor.jar +share/ozone/lib/aircompressor.jar share/ozone/lib/animal-sniffer-annotations.jar share/ozone/lib/annotations.jar share/ozone/lib/annotations.jar +share/ozone/lib/annotations.jar +share/ozone/lib/annotations.jar share/ozone/lib/aopalliance-repackaged.jar share/ozone/lib/aopalliance.jar +share/ozone/lib/apache-client.jar share/ozone/lib/apache-log4j-extras.jar +share/ozone/lib/arns.jar share/ozone/lib/asm-analysis.jar share/ozone/lib/asm-commons.jar share/ozone/lib/asm-tree.jar share/ozone/lib/asm-util.jar share/ozone/lib/asm.jar share/ozone/lib/aspectjrt.jar +share/ozone/lib/auth.jar share/ozone/lib/avro.jar -share/ozone/lib/aws-java-sdk-core.jar -share/ozone/lib/aws-java-sdk-kms.jar -share/ozone/lib/aws-java-sdk-s3.jar +share/ozone/lib/aws-core.jar +share/ozone/lib/aws-query-protocol.jar +share/ozone/lib/aws-xml-protocol.jar share/ozone/lib/bcpkix-jdk18on.jar share/ozone/lib/bcprov-jdk18on.jar share/ozone/lib/bcutil-jdk18on.jar share/ozone/lib/bonecp.RELEASE.jar share/ozone/lib/caffeine.jar share/ozone/lib/cdi-api.SP1.jar +share/ozone/lib/checksums-spi.jar +share/ozone/lib/checksums.jar share/ozone/lib/commons-beanutils.jar share/ozone/lib/commons-cli.jar share/ozone/lib/commons-codec.jar @@ -38,12 +46,15 @@ share/ozone/lib/commons-pool2.jar share/ozone/lib/commons-text.jar share/ozone/lib/commons-validator.jar share/ozone/lib/converter-jackson.jar +share/ozone/lib/crt-core.jar share/ozone/lib/curator-client.jar share/ozone/lib/curator-framework.jar share/ozone/lib/derby.jar share/ozone/lib/disruptor.jar share/ozone/lib/dnsjava.jar +share/ozone/lib/endpoints-spi.jar share/ozone/lib/error_prone_annotations.jar +share/ozone/lib/eventstream.jar share/ozone/lib/failureaccess.jar share/ozone/lib/grpc-api.jar share/ozone/lib/grpc-context.jar @@ -84,6 +95,11 @@ share/ozone/lib/hk2-api.jar share/ozone/lib/hk2-locator.jar share/ozone/lib/hk2-utils.jar share/ozone/lib/hppc.jar +share/ozone/lib/http-auth-aws-eventstream.jar +share/ozone/lib/http-auth-aws.jar +share/ozone/lib/http-auth-spi.jar +share/ozone/lib/http-auth.jar +share/ozone/lib/http-client-spi.jar share/ozone/lib/httpclient.jar share/ozone/lib/httpcore.jar share/ozone/lib/iceberg-api.jar @@ -92,12 +108,12 @@ share/ozone/lib/iceberg-common.jar share/ozone/lib/iceberg-core.jar share/ozone/lib/iceberg-orc.jar share/ozone/lib/iceberg-parquet.jar +share/ozone/lib/identity-spi.jar share/ozone/lib/istack-commons-runtime.jar share/ozone/lib/j2objc-annotations.jar share/ozone/lib/jackson-annotations.jar share/ozone/lib/jackson-core.jar share/ozone/lib/jackson-databind.jar -share/ozone/lib/jackson-dataformat-cbor.jar share/ozone/lib/jackson-dataformat-xml.jar share/ozone/lib/jackson-datatype-jsr310.jar share/ozone/lib/jackson-jaxrs-base.jar @@ -105,6 +121,7 @@ share/ozone/lib/jackson-jaxrs-json-provider.jar share/ozone/lib/jackson-module-jaxb-annotations.jar share/ozone/lib/jakarta.activation-api.jar share/ozone/lib/jakarta.activation.jar +share/ozone/lib/jakarta.activation.jar share/ozone/lib/jakarta.annotation-api.jar share/ozone/lib/jakarta.inject-api.jar share/ozone/lib/jakarta.inject.jar @@ -161,7 +178,6 @@ share/ozone/lib/jgrapht-ext.jar share/ozone/lib/jgraphx.jar share/ozone/lib/jheaps.jar share/ozone/lib/jline.jar -share/ozone/lib/jmespath-java.jar share/ozone/lib/jnr-a64asm.jar share/ozone/lib/jnr-constants.jar share/ozone/lib/jnr-ffi.jar @@ -172,12 +188,13 @@ share/ozone/lib/jooq-codegen.jar share/ozone/lib/jooq-meta.jar share/ozone/lib/jooq.jar share/ozone/lib/jsch.jar +share/ozone/lib/json-utils.jar share/ozone/lib/jsp-api.jar share/ozone/lib/jspecify.jar share/ozone/lib/jsr311-api.jar +share/ozone/lib/jtokkit.jar share/ozone/lib/jts-core.jar share/ozone/lib/jul-to-slf4j.jar -share/ozone/lib/jtokkit.jar share/ozone/lib/kerb-core.jar share/ozone/lib/kerb-crypto.jar share/ozone/lib/kerb-util.jar @@ -196,6 +213,7 @@ share/ozone/lib/listenablefuture-empty-to-avoid-conflict-with-guava.jar share/ozone/lib/log4j-api.jar share/ozone/lib/log4j-core.jar share/ozone/lib/metrics-core.jar +share/ozone/lib/metrics-spi.jar share/ozone/lib/netty-buffer.Final.jar share/ozone/lib/netty-codec-http.Final.jar share/ozone/lib/netty-codec-http2.Final.jar @@ -221,6 +239,7 @@ share/ozone/lib/okhttp-jvm.jar share/ozone/lib/okhttp-sse.jar share/ozone/lib/okhttp.jar share/ozone/lib/okio-jvm.jar +share/ozone/lib/okio-jvm.jar share/ozone/lib/okio.jar share/ozone/lib/openai4j.jar share/ozone/lib/opentelemetry-api.jar @@ -276,12 +295,14 @@ share/ozone/lib/parquet-variant.jar share/ozone/lib/perfmark-api.jar share/ozone/lib/picocli-shell-jline3.jar share/ozone/lib/picocli.jar +share/ozone/lib/profiles.jar share/ozone/lib/proto-google-common-protos.jar share/ozone/lib/protobuf-java.jar +share/ozone/lib/protocol-core.jar +share/ozone/lib/r2dbc-spi.RELEASE.jar share/ozone/lib/ranger-audit-core.jar share/ozone/lib/ranger-authz-api.jar share/ozone/lib/ranger-common-utils.jar -share/ozone/lib/r2dbc-spi.RELEASE.jar share/ozone/lib/ranger-intg.jar share/ozone/lib/ranger-plugin-classloader.jar share/ozone/lib/ranger-plugins-common.jar @@ -300,11 +321,17 @@ share/ozone/lib/ratis-thirdparty-misc.jar share/ozone/lib/ratis-tools.jar share/ozone/lib/re2j.jar share/ozone/lib/reactive-streams.jar +share/ozone/lib/reactive-streams.jar share/ozone/lib/reflections.jar +share/ozone/lib/regions.jar share/ozone/lib/reload4j.jar +share/ozone/lib/retries-spi.jar +share/ozone/lib/retries.jar share/ozone/lib/retrofit.jar share/ozone/lib/rocksdb-checkpoint-differ.jar share/ozone/lib/rocksdbjni.jar +share/ozone/lib/s3.jar +share/ozone/lib/sdk-core.jar share/ozone/lib/simpleclient.jar share/ozone/lib/simpleclient_common.jar share/ozone/lib/simpleclient_dropwizard.jar @@ -318,8 +345,12 @@ share/ozone/lib/spring-jdbc.jar share/ozone/lib/spring-tx.jar share/ozone/lib/sqlite-jdbc.jar share/ozone/lib/stax2-api.jar +share/ozone/lib/stax2-api.jar +share/ozone/lib/third-party-jackson-core.jar share/ozone/lib/txw2.jar share/ozone/lib/ugsync-util.jar +share/ozone/lib/utils-lite.jar +share/ozone/lib/utils.jar share/ozone/lib/vault-java-driver.jar share/ozone/lib/weld-servlet-shaded.Final.jar share/ozone/lib/woodstox-core.jar diff --git a/hadoop-ozone/dist/src/shell/conf/log4j.properties b/hadoop-ozone/dist/src/shell/conf/log4j.properties index 5ed63ffd1ee5..eaa4f1343150 100644 --- a/hadoop-ozone/dist/src/shell/conf/log4j.properties +++ b/hadoop-ozone/dist/src/shell/conf/log4j.properties @@ -167,9 +167,7 @@ log4j.logger.http.requests=${ozone.http.request.logger} #log4j.logger.http.requests.webhdfs=INFO,HttpAccess # Custom Logging levels -# AWS SDK & S3A FileSystem -#log4j.logger.com.amazonaws=ERROR -log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR +# S3A FileSystem #log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN # Log levels of third-party libraries diff --git a/hadoop-ozone/freon/pom.xml b/hadoop-ozone/freon/pom.xml index aed70f18d5f4..7b329a3bf827 100644 --- a/hadoop-ozone/freon/pom.xml +++ b/hadoop-ozone/freon/pom.xml @@ -31,14 +31,6 @@ - - com.amazonaws - aws-java-sdk-core - - - com.amazonaws - aws-java-sdk-s3 - com.fasterxml.jackson.core jackson-annotations @@ -139,6 +131,49 @@ org.slf4j slf4j-api + + software.amazon.awssdk + apache-client + + + software.amazon.awssdk + auth + + + software.amazon.awssdk + aws-core + + + software.amazon.awssdk + http-client-spi + + + software.amazon.awssdk + regions + + + software.amazon.awssdk + s3 + + + + software.amazon.awssdk + apache5-client + + + software.amazon.awssdk + netty-nio-client + + + + + software.amazon.awssdk + sdk-core + + + software.amazon.awssdk + utils + org.apache.ozone hdds-annotation-processing diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextRequestHandler.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextInterceptor.java similarity index 64% rename from hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextRequestHandler.java rename to hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextInterceptor.java index a5f40c6412b0..58da089cb0f1 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextRequestHandler.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextInterceptor.java @@ -17,26 +17,29 @@ package org.apache.hadoop.ozone.freon; -import com.amazonaws.Request; -import com.amazonaws.handlers.RequestHandler2; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.Context; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.http.SdkHttpRequest; /** * Adds W3C trace context headers to each outgoing S3 request so the S3 Gateway * can attach its spans to the Freon task span, using {@link W3CTraceContextPropagator}. */ -public final class FreonS3TraceContextRequestHandler extends RequestHandler2 { +public final class FreonS3TraceContextInterceptor implements ExecutionInterceptor { @Override - public void beforeRequest(Request request) { + public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { if (!Span.current().getSpanContext().isValid()) { - return; + return context.httpRequest(); } + final SdkHttpRequest.Builder request = context.httpRequest().toBuilder(); W3CTraceContextPropagator.getInstance().inject( - Context.current(), + io.opentelemetry.context.Context.current(), request, - (carrier, key, value) -> carrier.addHeader(key, value)); + (carrier, key, value) -> carrier.putHeader(key, value)); + return request.build(); } } diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java index f3ebccce24f8..e6ba1c5ad3d9 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java @@ -68,7 +68,7 @@ public Void call() throws Exception { private void createBucket(long bucketNum) throws Exception { String bName = getPrefix() + bucketNum; timer.time(() -> { - getS3().createBucket(bName); + getS3().createBucket(r -> r.bucket(bName)); return null; }); } diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java index bc8660489876..fab89c68fecc 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java @@ -17,12 +17,15 @@ package org.apache.hadoop.ozone.freon; -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import java.net.URI; import picocli.CommandLine; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; +import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; /** * Initiliazing common aspects of S3BucketGenerator and S3KeyGenerator. @@ -33,33 +36,36 @@ public class S3EntityGenerator extends BaseFreonGenerator { defaultValue = "http://localhost:9878") private String endpoint; - private AmazonS3 s3; + private S3Client s3; protected void s3ClientInit() { init(); - AmazonS3ClientBuilder amazonS3ClientBuilder = - AmazonS3ClientBuilder.standard() - .withCredentials(new EnvironmentVariableCredentialsProvider()); + S3ClientBuilder s3ClientBuilder = S3Client.builder() + .httpClientBuilder(ApacheHttpClient.builder()) + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + // checksums only where S3 requires them: a load generator should not spend its time computing them + .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED) + .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED) + .overrideConfiguration(c -> c.addExecutionInterceptor(new FreonS3TraceContextInterceptor())); if (!endpoint.isEmpty()) { - amazonS3ClientBuilder - .withPathStyleAccessEnabled(true) - .withEndpointConfiguration( - new EndpointConfiguration(endpoint, "us-east-1")); + s3ClientBuilder + .forcePathStyle(true) + .endpointOverride(URI.create(endpoint)) + .region(Region.US_EAST_1); } else { - amazonS3ClientBuilder.withRegion(Regions.DEFAULT_REGION); + s3ClientBuilder.region(Region.US_WEST_2); } - amazonS3ClientBuilder.withRequestHandlers(new FreonS3TraceContextRequestHandler()); - s3 = amazonS3ClientBuilder.build(); + s3 = s3ClientBuilder.build(); } public String getEndpoint() { return endpoint; } - public AmazonS3 getS3() { + public S3Client getS3() { return s3; } } diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java index 05bb75100192..6f04181f3399 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java @@ -19,14 +19,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_MULTIPART_MIN_SIZE; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; import com.codahale.metrics.Timer; -import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -38,6 +31,9 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Option; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; /** * Generate random keys via the s3 interface. @@ -84,7 +80,7 @@ public class S3KeyGenerator extends S3EntityGenerator private Timer timer; - private String content; + private byte[] content; @Override public Void call() throws Exception { @@ -96,11 +92,10 @@ public Void call() throws Exception { s3ClientInit(); - content = RandomStringUtils.secure().nextAscii(fileSize); + content = RandomStringUtils.secure().nextAscii(fileSize).getBytes(StandardCharsets.UTF_8); timer = getMetrics().timer("key-create"); - System.setProperty("com.amazonaws.services.s3.disablePutObjectMD5Validation", "true"); runTests(this::createKey); return null; @@ -111,38 +106,24 @@ private void createKey(long counter) throws Exception { if (multiPart) { final String keyName = generateObjectName(counter); - final InitiateMultipartUploadRequest initiateRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); + final String uploadId = getS3().createMultipartUpload(r -> r.bucket(bucketName).key(keyName)).uploadId(); - final InitiateMultipartUploadResult initiateMultipartUploadResult = - getS3().initiateMultipartUpload(initiateRequest); - final String uploadId = initiateMultipartUploadResult.getUploadId(); - - List parts = new ArrayList<>(); + List parts = new ArrayList<>(); for (int i = 1; i <= numberOfParts; i++) { - - final UploadPartRequest uploadPartRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withPartNumber(i) - .withLastPart(i == numberOfParts) - .withUploadId(uploadId) - .withPartSize(fileSize) - .withInputStream(new ByteArrayInputStream(content.getBytes( - StandardCharsets.UTF_8))); - - final UploadPartResult uploadPartResult = - getS3().uploadPart(uploadPartRequest); - parts.add(uploadPartResult.getPartETag()); + final int partNumber = i; + final UploadPartResponse uploadPartResponse = getS3().uploadPart( + r -> r.bucket(bucketName).key(keyName).uploadId(uploadId).partNumber(partNumber) + .contentLength((long) content.length), + RequestBody.fromBytes(content)); + parts.add(CompletedPart.builder().partNumber(partNumber).eTag(uploadPartResponse.eTag()).build()); } - getS3().completeMultipartUpload( - new CompleteMultipartUploadRequest(bucketName, keyName, uploadId, - parts)); + getS3().completeMultipartUpload(r -> r.bucket(bucketName).key(keyName).uploadId(uploadId) + .multipartUpload(m -> m.parts(parts))); } else { - getS3().putObject(bucketName, generateObjectName(counter), - content); + getS3().putObject(r -> r.bucket(bucketName).key(generateObjectName(counter)), + RequestBody.fromBytes(content)); } return null; diff --git a/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java b/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java new file mode 100644 index 000000000000..8cb6b8199d1f --- /dev/null +++ b/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.freon; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Scope; +import java.net.URI; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; + +/** Tests for {@link FreonS3TraceContextInterceptor}. */ +class TestFreonS3TraceContextInterceptor { + private static final String TRACE_ID = "0af7651916cd43dd8448eb211c80319c"; + private static final String SPAN_ID = "b7ad6b7169203331"; + + private final FreonS3TraceContextInterceptor interceptor = new FreonS3TraceContextInterceptor(); + + private static Context.ModifyHttpRequest context(SdkHttpRequest request) { + final Context.ModifyHttpRequest context = mock(Context.ModifyHttpRequest.class); + when(context.httpRequest()).thenReturn(request); + return context; + } + + private static SdkHttpRequest request() { + return SdkHttpRequest.builder().method(SdkHttpMethod.PUT).uri(URI.create("http://localhost:9878/bucket/key")) + .build(); + } + + @Test + void addsTraceContextOfCurrentSpan() { + final Span span = Span.wrap( + SpanContext.create(TRACE_ID, SPAN_ID, TraceFlags.getSampled(), TraceState.getDefault())); + final SdkHttpRequest modified; + try (Scope ignored = span.makeCurrent()) { + modified = interceptor.modifyHttpRequest(context(request()), new ExecutionAttributes()); + } + + assertThat(modified.firstMatchingHeader("traceparent")).hasValue("00-" + TRACE_ID + "-" + SPAN_ID + "-01"); + } + + @Test + void leavesRequestUnchangedWithoutSpan() { + final SdkHttpRequest request = request(); + + assertSame(request, interceptor.modifyHttpRequest(context(request), new ExecutionAttributes())); + } +} diff --git a/hadoop-ozone/integration-test-s3/pom.xml b/hadoop-ozone/integration-test-s3/pom.xml index fb951dfbfa9a..283cada97836 100644 --- a/hadoop-ozone/integration-test-s3/pom.xml +++ b/hadoop-ozone/integration-test-s3/pom.xml @@ -26,16 +26,6 @@ Apache Ozone Integration Tests with S3 Gateway - - com.amazonaws - aws-java-sdk-core - test - - - com.amazonaws - aws-java-sdk-s3 - test - com.google.guava guava diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java index 9ad26f180f6f..3186b97b97cf 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java @@ -23,14 +23,6 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import java.net.URI; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.http.HttpConfig; @@ -61,58 +53,6 @@ public S3ClientFactory(OzoneConfiguration conf) { this.conf = conf; } - /** - * Creates an AmazonS3 client (AWS SDK V1) with path style access enabled. - * - * @return AmazonS3 client - */ - public AmazonS3 createS3Client() { - return createS3Client(true); - } - - /** - * Creates an AmazonS3 client (AWS SDK V1). - * - * @param enablePathStyle whether to enable path style access - * @return AmazonS3 client - */ - public AmazonS3 createS3Client(boolean enablePathStyle) { - final String accessKey = "user"; - final String secretKey = "password"; - final Regions region = Regions.DEFAULT_REGION; - - final String protocol; - final HttpConfig.Policy webPolicy = getHttpPolicy(conf); - String host; - - if (webPolicy.isHttpsEnabled()) { - // TODO: Currently HTTPS is disabled in the test, we can add HTTPS - // integration in the future - protocol = HTTPS_SCHEME; - host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY); - } else { - protocol = HTTP_SCHEME; - host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY); - } - - String endpoint = protocol + "://" + host; - - AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider( - new BasicAWSCredentials(accessKey, secretKey)); - - ClientConfiguration clientConfiguration = new ClientConfiguration(); - LOG.info("S3 Endpoint is {}", endpoint); - - return AmazonS3ClientBuilder.standard() - .withPathStyleAccessEnabled(enablePathStyle) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration( - endpoint, region.getName())) - .withClientConfiguration(clientConfiguration) - .withCredentials(credentials) - .build(); - } - /** * Creates an S3Client (AWS SDK V2) with path style access enabled. * diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java index 78e801ef9e0f..30ac8729eaf3 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java @@ -23,7 +23,6 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.s3.MultiS3GatewayService; import org.apache.hadoop.ozone.s3.S3GatewayService; -import org.apache.hadoop.ozone.s3.awssdk.v1.AbstractS3SDKV1Tests; import org.apache.hadoop.ozone.s3.awssdk.v2.AbstractS3SDKV2Tests; import org.apache.ozone.test.ClusterForTests; import org.junit.jupiter.api.Nested; @@ -48,14 +47,6 @@ protected MiniOzoneCluster createCluster() throws Exception { .build(); } - @Nested - class V1 extends AbstractS3SDKV1Tests { - @Override - public MiniOzoneCluster cluster() { - return getCluster(); - } - } - @Nested class V2 extends AbstractS3SDKV2Tests { @Override diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java deleted file mode 100644 index a8dfa914b128..000000000000 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ /dev/null @@ -1,3167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.s3.awssdk.v1; - -import static org.apache.hadoop.ozone.OzoneConsts.MB; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.getOmKeyModificationTime; -import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.getOmPartModificationTime; -import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; -import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.AmazonServiceException.ErrorType; -import com.amazonaws.HttpMethod; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.Headers; -import com.amazonaws.services.s3.model.AbortIncompleteMultipartUpload; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.AccessControlList; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; -import com.amazonaws.services.s3.model.BucketTaggingConfiguration; -import com.amazonaws.services.s3.model.CanonicalGrantee; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.CopyPartRequest; -import com.amazonaws.services.s3.model.CopyPartResult; -import com.amazonaws.services.s3.model.CreateBucketRequest; -import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest; -import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; -import com.amazonaws.services.s3.model.GetBucketAccelerateConfigurationRequest; -import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest; -import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest; -import com.amazonaws.services.s3.model.GetObjectAclRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.GetObjectTaggingRequest; -import com.amazonaws.services.s3.model.GetObjectTaggingResult; -import com.amazonaws.services.s3.model.Grantee; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ListBucketsPaginatedRequest; -import com.amazonaws.services.s3.model.ListBucketsPaginatedResult; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ListPartsRequest; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.MultipartUploadListing; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.ObjectTagging; -import com.amazonaws.services.s3.model.Owner; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PartListing; -import com.amazonaws.services.s3.model.PartSummary; -import com.amazonaws.services.s3.model.Permission; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.model.SetBucketLifecycleConfigurationRequest; -import com.amazonaws.services.s3.model.SetBucketTaggingConfigurationRequest; -import com.amazonaws.services.s3.model.SetObjectAclRequest; -import com.amazonaws.services.s3.model.SetObjectTaggingRequest; -import com.amazonaws.services.s3.model.Tag; -import com.amazonaws.services.s3.model.TagSet; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; -import com.amazonaws.services.s3.transfer.model.UploadResult; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.MessageDigest; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import javax.xml.bind.DatatypeConverter; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.hdds.client.OzoneQuota; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.client.BucketArgs; -import org.apache.hadoop.ozone.client.ObjectStore; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; -import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; -import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.service.KeyLifecycleService; -import org.apache.hadoop.ozone.s3.S3ClientFactory; -import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils; -import org.apache.hadoop.ozone.s3.endpoint.S3Owner; -import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; -import org.apache.hadoop.ozone.s3.util.S3Consts; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.NonHATests; -import org.apache.ozone.test.OzoneTestBase; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.function.Executable; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; - -/** - * This is an abstract class to test the AWS Java S3 SDK operations. - * This class should be extended for OM standalone and OM HA (Ratis) cluster setup. - * - * The test scenarios are adapted from - * - https://github.com/awsdocs/aws-doc-sdk-examples/tree/main/java/example_code/s3/ - * - https://github.com/ceph/s3-tests - * - */ -@TestMethodOrder(MethodOrderer.MethodName.class) -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public abstract class AbstractS3SDKV1Tests extends OzoneTestBase implements NonHATests.TestCase { - - // server-side limitation - private static final int MAX_UPLOADS_LIMIT = 1000; - private static final String READ_CONSISTENCY_HEADER = - "x-ozone-read-consistency"; - private static final String LOCAL_LEASE_LOG_LIMIT_HEADER = - "x-ozone-local-lease-log-limit"; - - /** - * There are still some unsupported S3 operations. - * Current unsupported S3 operations (non-exhaustive): - * - Cross Region Replication (CrossRegionReplication.java) - * - Versioned enabled buckets - * - DeleteObjectVersionEnabledBucket.java - * - DeleteMultipleObjectsVersionEnabledBucket.java - * - ListKeysVersioningEnabledBucket.java - * - Website configurations - * - WebsiteConfiguration.java - * - SetWebsiteConfiguration.java - * - GetWebsiteConfiguration.java - * - DeleteWebsiteConfiguration.java - * - S3 Event Notifications - * - EnableNotificationOnABucket.java - * - Object tags - * - GetObjectTags.java - * - GetObjectTags2.java - * - Bucket policy - * - SetBucketPolicy.java - * - GetBucketPolicy.java - * - DeleteBucketPolicy.java - * - Bucket lifecycle configuration - * - LifecycleConfiguration.java - * - Canned Bucket ACL - * - CreateBucketWithACL.java - * - Object ACL - * - SetAcl.java - * - ModifyACLExistingObject.java - * - GetAcl.java - * - S3 Encryption - * - S3Encrypt.java - * - S3EncryptV2.java - * - Client-side encryption - * - S3ClientSideEncryptionAsymmetricMasterKey.java - * - S3ClientSideEncryptionSymMasterKey.java - * - Server-side encryption - * - SpecifyServerSideEncryption.ajva - * - ServerSideEncryptionCopyObjectUsingHLWithSSEC.java - * - ServerSideEncryptionUsingClientSideEncryptionKey.java - * - Dual stack endpoints - * - DualStackEndpoints.java - * - Transfer acceleration - * - TransferAcceleration.java - * - Temp credentials - * - MakingRequestsWithFederatedTempCredentials.java - * - MakingRequestsWithIAMTempCredentials.java - * - Object archival - * - RestoreArchivedObject - * - KMS key - * - UploadObjectKMSKey.java - */ - - private MiniOzoneCluster cluster; - private AmazonS3 s3Client; - - @BeforeAll - void createClient() { - cluster = cluster(); - s3Client = new S3ClientFactory(cluster.getConf()) - .createS3Client(); - } - - @Test - public void testCreateBucket() { - final String bucketName = getBucketName(); - - Bucket b = s3Client.createBucket(bucketName); - - assertEquals(bucketName, b.getName()); - assertTrue(s3Client.doesBucketExist(bucketName)); - assertTrue(s3Client.doesBucketExistV2(bucketName)); - assertTrue(isBucketEmpty(b)); - } - - /** - * s3-tests: test_bucket_create_exists. - */ - @Test - public void testCreateBucketAlreadyOwnedByYou() { - final String bucketName = getBucketName("owned-by-you"); - s3Client.createBucket(bucketName); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.createBucket(bucketName)); - assertEquals(409, ase.getStatusCode()); - assertEquals(S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU.getCode(), ase.getErrorCode()); - assertEquals("application/xml", ase.getHttpHeaders().get("Content-Type")); - } - - @Test - public void testCreateBucketAlreadyExistsDifferentOwner() throws IOException { - final String bucketName = getBucketName("other-owner"); - final String otherOwner = "other-s3-owner"; - try (OzoneClient ozoneClient = cluster.newClient()) { - ozoneClient.getObjectStore().getS3Volume().createBucket(bucketName, - BucketArgs.newBuilder().setOwner(otherOwner).build()); - } - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.createBucket(bucketName)); - assertEquals(409, ase.getStatusCode()); - assertEquals(S3ErrorTable.BUCKET_ALREADY_EXISTS.getCode(), ase.getErrorCode()); - } - - @Test - public void testBucketACLOperations() { - // TODO HDDS-11738: Uncomment assertions when bucket S3 ACL logic has been fixed - final String bucketName = getBucketName(); - - AccessControlList aclList = new AccessControlList(); - Owner owner = new Owner("owner", "owner"); - aclList.withOwner(owner); - Grantee grantee = new CanonicalGrantee("testGrantee"); - aclList.grantPermission(grantee, Permission.Read); - - - CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName) - .withAccessControlList(aclList); - - s3Client.createBucket(createBucketRequest); - - //assertEquals(aclList, s3Client.getBucketAcl(bucketName)); - - aclList.grantPermission(grantee, Permission.Write); - s3Client.setBucketAcl(bucketName, aclList); - - //assertEquals(aclList, s3Client.getBucketAcl(bucketName)); - } - - /** - * Integration tests for ListBuckets (GET / ListAllMyBuckets). - */ - @Nested - class ListBucketsTests { - - @Test - public void testListBuckets() throws IOException { - List bucketNames = new ArrayList<>(); - for (int i = 0; i <= 5; i++) { - String bucketName = getBucketName(String.valueOf(i)); - s3Client.createBucket(bucketName); - bucketNames.add(bucketName); - } - - List bucketList = s3Client.listBuckets(); - List listBucketNames = bucketList.stream() - .map(Bucket::getName).collect(Collectors.toList()); - - assertThat(listBucketNames).containsAll(bucketNames); - - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - String expectOwner = ugi.getShortUserName(); - - Owner s3AccountOwner = s3Client.getS3AccountOwner(); - - assertThat(s3AccountOwner.getDisplayName()).isEqualTo(expectOwner); - assertThat(s3AccountOwner.getId()).isEqualTo(S3Owner.DEFAULT_S3OWNER_ID); - } - - /** - * Verifies {@code maxBuckets=1} returns one bucket per page and a continuation token - * when more buckets exist. - */ - @Test - public void testListBucketsPaginatedMaxBucketsOne() { - final String bucketA = uniqueObjectName("bucket-a"); - final String bucketB = uniqueObjectName("bucket-b"); - s3Client.createBucket(bucketA); - s3Client.createBucket(bucketB); - try { - List found = S3SDKTestUtils.collectBucketsOnePerPage((token, max) -> { - ListBucketsPaginatedRequest request = new ListBucketsPaginatedRequest() - .withMaxBuckets(max); - if (token != null) { - request.withContinuationToken(token); - } - - ListBucketsPaginatedResult page = s3Client.listBuckets(request); - return new S3SDKTestUtils.BucketListPage( - page.getBuckets().stream().map(Bucket::getName).collect(Collectors.toList()), - page.getContinuationToken()); - }); - List foundTestBuckets = S3SDKTestUtils.filterToExpectedBuckets( - found, bucketA, bucketB); - assertThat(foundTestBuckets).containsExactlyInAnyOrder(bucketA, bucketB); - } finally { - s3Client.deleteBucket(bucketA); - s3Client.deleteBucket(bucketB); - } - } - - /** - * Verifies pagination: listing buckets page-by-page using {@code maxBuckets} - * and the returned continuation token, until all buckets are retrieved. - */ - @Test - public void testListBucketsPaginationReturnsAllBuckets() { - final int totalBuckets = 5; - final int pageSize = 2; - List created = new ArrayList<>(); - - for (int i = 0; i < totalBuckets; i++) { - String name = uniqueObjectName("paginated-" + i); - s3Client.createBucket(name); - created.add(name); - } - - try { - List retrieved = new ArrayList<>(); - String continuationToken = null; - - do { - ListBucketsPaginatedRequest request = new ListBucketsPaginatedRequest() - .withMaxBuckets(pageSize); - if (continuationToken != null) { - request.withContinuationToken(continuationToken); - } - - ListBucketsPaginatedResult response = s3Client.listBuckets(request); - - response.getBuckets().stream() - .map(Bucket::getName) - .filter(created::contains) - .forEach(retrieved::add); - - continuationToken = response.getContinuationToken(); - } while (continuationToken != null); - - assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created); - } finally { - for (String name : created) { - s3Client.deleteBucket(name); - } - } - } - - /** - * verifies that Page 1 uses maxBuckets=1; - * later pages send only continuationToken (no maxBuckets). - */ - @Test - public void testListBucketsContinuationTokenWithoutMaxBuckets() { - List created = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - String name = uniqueObjectName("token-only-" + i); - s3Client.createBucket(name); - created.add(name); - } - - try { - List retrieved = new ArrayList<>(); - String continuationToken = null; - boolean firstPage = true; - - do { - ListBucketsPaginatedRequest request = new ListBucketsPaginatedRequest(); - if (firstPage) { - request.withMaxBuckets(1); - firstPage = false; - } else { - request.withContinuationToken(continuationToken); - } - - ListBucketsPaginatedResult response = s3Client.listBuckets(request); - if (continuationToken == null) { - assertEquals(1, response.getBuckets().size()); - assertNotNull(response.getContinuationToken()); - } else { - assertFalse(response.getBuckets().isEmpty()); - } - - response.getBuckets().stream() - .map(Bucket::getName) - .filter(created::contains) - .forEach(retrieved::add); - - continuationToken = response.getContinuationToken(); - } while (continuationToken != null); - - assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created); - } finally { - for (String name : created) { - s3Client.deleteBucket(name); - } - } - } - } - - @Test - public void testDeleteBucket() { - final String bucketName = getBucketName(); - - s3Client.createBucket(bucketName); - - s3Client.deleteBucket(bucketName); - - assertFalse(s3Client.doesBucketExist(bucketName)); - assertFalse(s3Client.doesBucketExistV2(bucketName)); - } - - @Test - public void testDeleteBucketNotExist() { - final String bucketName = getBucketName(); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucket(bucketName)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchBucket", ase.getErrorCode()); - } - - @Test - public void testDeleteBucketNonEmptyWithKeys() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Upload some objects to the bucket - for (int i = 1; i <= 10; i++) { - s3Client.putObject(bucketName, "key-" + i, RandomStringUtils.secure().nextAlphanumeric(1024)); - } - - // Bucket deletion should fail if there are still keys in the bucket - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucket(bucketName) - ); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(409, ase.getStatusCode()); - assertEquals("BucketNotEmpty", ase.getErrorCode()); - - // Delete all the keys - ObjectListing objectListing = s3Client.listObjects(bucketName); - while (true) { - for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { - s3Client.deleteObject(bucketName, summary.getKey()); - } - - // more object_listing to retrieve? - if (objectListing.isTruncated()) { - objectListing = s3Client.listNextBatchOfObjects(objectListing); - } else { - break; - } - } - } - - @Test - public void testDeleteBucketNonEmptyWithIncompleteMultipartUpload(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (5 * MB)); - - // Create an incomplete multipart upload by initiating multipart upload, - // uploading some parts, but not actually completing it. - String uploadId = initiateMultipartUpload(bucketName, keyName, null, null, null); - - uploadParts(bucketName, keyName, uploadId, multipartUploadFile, 1 * MB); - - // Bucket deletion should fail if there are still keys in the bucket - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucket(bucketName) - ); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(409, ase.getStatusCode()); - assertEquals("BucketNotEmpty", ase.getErrorCode()); - - // After the multipart upload is aborted, the bucket deletion should succeed - abortMultipartUpload(bucketName, keyName, uploadId); - - s3Client.deleteBucket(bucketName); - - assertFalse(s3Client.doesBucketExistV2(bucketName)); - } - - @Test - public void testPutObject() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - } - - @ParameterizedTest - @ValueSource(strings = {"follower-stale", "follower-linearizable", - "leader-only"}) - public void testGetObjectWithReadConsistencyHeader(String readConsistency) - throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, content); - - GetObjectRequest request = new GetObjectRequest(bucketName, keyName); - request.putCustomRequestHeader(READ_CONSISTENCY_HEADER, - readConsistency); - if ("follower-stale".equals(readConsistency)) { - request.putCustomRequestHeader(LOCAL_LEASE_LOG_LIMIT_HEADER, "10"); - } - - try (S3Object object = s3Client.getObject(request); - S3ObjectInputStream objectContent = object.getObjectContent()) { - assertEquals(content, IOUtils.toString(objectContent, - StandardCharsets.UTF_8)); - } - } - - @Test - public void testGetObjectWithInvalidReadConsistencyHeader() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, "bar"); - - GetObjectRequest request = new GetObjectRequest(bucketName, keyName); - request.putCustomRequestHeader(READ_CONSISTENCY_HEADER, - "invalid"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals("InvalidArgument", ase.getErrorCode()); - } - - @Test - public void testPutObjectWithEmptyContentType() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentType(""); - InputStream inputStream = new ByteArrayInputStream( - content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject( - bucketName, keyName, inputStream, metadata); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", - putObjectResult.getETag()); - } - - @Test - public void testGetObjectWithMalformedReadConsistencyHeader() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, "bar"); - - GetObjectRequest request = new GetObjectRequest(bucketName, keyName); - request.putCustomRequestHeader(READ_CONSISTENCY_HEADER, - "follower-stale;logLimit=10"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals("InvalidArgument", ase.getErrorCode()); - } - - @Test - public void testPutObjectIfNoneMatch() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is, new ObjectMetadata()).ifNoneMatch("*"); - - PutObjectResult putObjectResult = s3Client.putObject(request); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - } - - @Test - public void testPutObjectIfNoneMatchFail() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - InputStream is2 = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is2, new ObjectMetadata()).ifNoneMatch("*"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - } - - @Test - public void testPutObjectIfMatch() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - String etag = putObjectResult.getETag(); - - InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is2, new ObjectMetadata()).ifMatch(etag); - - PutObjectResult putObjectResult2 = s3Client.putObject(request); - assertNotNull(putObjectResult2.getETag()); - assertNotEquals(etag, putObjectResult2.getETag()); - - ObjectMetadata updatedObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(putObjectResult2.getETag(), updatedObjectMetadata.getETag()); - } - - @Test - public void testPutObjectIfMatchFail() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult initialResult = - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is2, new ObjectMetadata()).ifMatch("wrong-etag"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - - ObjectMetadata existingObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(initialResult.getETag(), existingObjectMetadata.getETag()); - } - - @ParameterizedTest - @ValueSource(strings = {"*", "\"some-etag\""}) - public void testPutObjectIfMatchMissingKeyFail(String ifMatch) { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream("bar2".getBytes( - StandardCharsets.UTF_8)); - PutObjectRequest request = new PutObjectRequest( - bucketName, keyName, is, new ObjectMetadata()).ifMatch(ifMatch); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(request)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchKey", ase.getErrorCode()); - - AmazonServiceException missingKey = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(bucketName, keyName)); - assertEquals(ErrorType.Client, missingKey.getErrorType()); - assertEquals(404, missingKey.getStatusCode()); - assertEquals("NoSuchKey", missingKey.getErrorCode()); - } - - @Test - public void testDeleteObjectIfMatch() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, putObjectResult.getETag()); - - assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testDeleteObjectIfMatchFail() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, "wrong-etag"); - - assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, responseCode); - ObjectMetadata existingObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(putObjectResult.getETag(), existingObjectMetadata.getETag()); - } - - @Test - public void testDeleteObjectIfMatchWildcard() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, "*"); - - assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testDeleteObjectIfMatchWildcardMissingKeyFail() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - int responseCode = deleteObjectWithIfMatch(bucketName, keyName, "*"); - - assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, responseCode); - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - private int deleteObjectWithIfMatch(String bucketName, String keyName, String ifMatch) throws IOException { - GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.DELETE) - .withExpiration(Date.from(Instant.now().plusMillis(1000 * 60 * 60))); - request.putCustomRequestHeader(Headers.GET_OBJECT_IF_MATCH, ifMatch); - URL presignedUrl = s3Client.generatePresignedUrl(request); - Map> headers = Collections.singletonMap(Headers.GET_OBJECT_IF_MATCH, - Collections.singletonList(ifMatch)); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "DELETE", headers, null); - return connection.getResponseCode(); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - } - - @Test - public void testCopyObject() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putResult.getETag()); - - CopyObjectResult copyResult = s3Client.copyObject(sourceBucketName, sourceKey, destBucketName, destKey); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", copyResult.getETag()); - } - - @Test - public void testCopyObjectLastModifiedMatchesOmDb() throws Exception { - final String sourceBucketName = getBucketName(); - final String destBucketName = getBucketName(); - final String sourceKey = getKeyName(); - final String destKey = getKeyName(); - final String content = "copy-last-modified-test-content"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - final InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(sourceBucketName, sourceKey, inputStream, new ObjectMetadata()); - - final long sourceModificationTime = getOmKeyModificationTime(cluster, sourceBucketName, sourceKey); - - Thread.sleep(100); - - final CopyObjectResult copyResult = s3Client.copyObject(sourceBucketName, sourceKey, destBucketName, destKey); - - assertNotNull(copyResult.getLastModifiedDate()); - final long responseModificationTime = copyResult.getLastModifiedDate().getTime(); - final long destModificationTime = getOmKeyModificationTime(cluster, destBucketName, destKey); - - assertEquals(destModificationTime, responseModificationTime); - assertNotEquals(sourceModificationTime, responseModificationTime); - } - - @Test - public void testUploadPartCopyLastModifiedMatchesOmDb() throws Exception { - final String bucketName = getBucketName(); - final String sourceKey = getKeyName(); - final String destKey = getKeyName(); - final String content = "copy-last-modified-test-content"; - s3Client.createBucket(bucketName); - - final InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, sourceKey, inputStream, new ObjectMetadata()); - - final long sourceModificationTime = getOmKeyModificationTime(cluster, bucketName, sourceKey); - - Thread.sleep(100); - - final InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucketName, destKey)); - final String uploadId = initResponse.getUploadId(); - - final CopyPartResult copyPartResult = s3Client.copyPart( - new CopyPartRequest() - .withSourceBucketName(bucketName) - .withSourceKey(sourceKey) - .withDestinationBucketName(bucketName) - .withDestinationKey(destKey) - .withUploadId(uploadId) - .withPartNumber(1)); - - assertNotNull(copyPartResult.getLastModifiedDate()); - final long responseModificationTime = copyPartResult.getLastModifiedDate().getTime(); - final long destPartModificationTime = getOmPartModificationTime(cluster, bucketName, destKey, uploadId, 1); - - assertEquals(destPartModificationTime, responseModificationTime); - assertNotEquals(sourceModificationTime, responseModificationTime); - } - - @Test - public void testCopyObjectWithSourceIfMatch() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - String sourceETag = putResult.getETag(); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withMatchingETagConstraint(sourceETag); - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertEquals(sourceETag, copyResult.getETag()); - } - - @Test - public void testCopyObjectWithSourceIfMatchFail() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withMatchingETagConstraint("wrong-etag"); - - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertNull(copyResult); - } - - @Test - public void testCopyObjectWithSourceIfNoneMatch() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - String sourceETag = putResult.getETag(); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withNonmatchingETagConstraint("different-etag"); - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertEquals(sourceETag, copyResult.getETag()); - } - - @Test - public void testCopyObjectWithSourceIfNoneMatchFail() { - final String sourceBucketName = getBucketName("source"); - final String destBucketName = getBucketName("dest"); - final String sourceKey = getKeyName("source"); - final String destKey = getKeyName("dest"); - final String content = "bar"; - s3Client.createBucket(sourceBucketName); - s3Client.createBucket(destBucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata()); - String sourceETag = putResult.getETag(); - - CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey) - .withNonmatchingETagConstraint(sourceETag); - - CopyObjectResult copyResult = s3Client.copyObject(copyRequest); - assertNull(copyResult); - } - - @Test - public void testPutObjectWithMD5Header() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - byte[] md5Bytes = calculateDigest(new ByteArrayInputStream(contentBytes), 0, contentBytes.length); - String md5Base64 = Base64.getEncoder().encodeToString(md5Bytes); - - InputStream is = new ByteArrayInputStream(contentBytes); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentMD5(md5Base64); - objectMetadata.setContentLength(contentBytes.length); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, objectMetadata); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - - S3Object object = s3Client.getObject(bucketName, keyName); - assertEquals(content.length(), object.getObjectMetadata().getContentLength()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", object.getObjectMetadata().getETag()); - } - - static Stream wrongContentMD5Provider() throws Exception { - byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); - byte[] wrongMd5Bytes = MessageDigest.getInstance("MD5").digest(wrongContentBytes); - String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); - - return Stream.of( - Arguments.of(wrongMd5Base64, S3ErrorTable.BAD_DIGEST.getCode()), - Arguments.of("invalid-base64", S3ErrorTable.INVALID_DIGEST.getCode()) - ); - } - - @ParameterizedTest - @MethodSource("wrongContentMD5Provider") - public void testPutObjectWithWrongMD5Header(String wrongMd5Base64, String expectedErrorCode) { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - - byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - InputStream is = new ByteArrayInputStream(contentBytes); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentMD5(wrongMd5Base64); - objectMetadata.setContentLength(contentBytes.length); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(bucketName, keyName, is, objectMetadata)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals(expectedErrorCode, ase.getErrorCode()); - - // Verify the object was not uploaded - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testMultipartUploadWithMD5Header() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate multipart upload - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Prepare part data - String part1Content = "part1data"; - byte[] part1Bytes = part1Content.getBytes(StandardCharsets.UTF_8); - byte[] part1Md5Bytes = calculateDigest(new ByteArrayInputStream(part1Bytes), 0, part1Bytes.length); - String part1Md5Base64 = Base64.getEncoder().encodeToString(part1Md5Bytes); - - // Upload part 1 with MD5 - InputStream part1InputStream = new ByteArrayInputStream(part1Bytes); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentMD5(part1Md5Base64); - metadata.setContentLength(part1Bytes.length); - - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(part1InputStream) - .withPartSize(part1Bytes.length) - .withObjectMetadata(metadata); - - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Verify ETag - String expectedETag = DatatypeConverter.printHexBinary(part1Md5Bytes).toLowerCase(); - assertEquals(expectedETag, uploadResult.getPartETag().getETag()); - - // Complete multipart upload - List partETags = new ArrayList<>(); - partETags.add(uploadResult.getPartETag()); - - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - s3Client.completeMultipartUpload(completeRequest); - - // Verify object was uploaded - S3Object object = s3Client.getObject(bucketName, keyName); - try (S3ObjectInputStream s3is = object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream()) { - IOUtils.copy(s3is, bos); - assertEquals(part1Content, bos.toString("UTF-8")); - } - } - - @ParameterizedTest - @MethodSource("wrongContentMD5Provider") - public void testMultipartUploadPartWithWrongMD5Header(String wrongMd5Base64, String expectedErrorCode) { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate multipart upload - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Prepare part data with wrong MD5 - String partContent = "partdata"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - - // Upload part with wrong/invalid MD5 should fail - InputStream partInputStream = new ByteArrayInputStream(partBytes); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentMD5(wrongMd5Base64); - metadata.setContentLength(partBytes.length); - - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(partInputStream) - .withPartSize(partBytes.length) - .withObjectMetadata(metadata); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.uploadPart(uploadRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals(expectedErrorCode, ase.getErrorCode()); - - // Abort the multipart upload - AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(bucketName, keyName, uploadId); - s3Client.abortMultipartUpload(abortRequest); - - // Verify object was not created - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testCompleteMultipartUploadIfNoneMatch() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate and complete multipart upload with If-None-Match: * - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "part1data"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-None-Match: * (key doesn't exist, should succeed) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-None-Match", "*"); - - CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(completeRequest); - assertNotNull(result.getETag()); - - // Verify object was created - assertTrue(s3Client.doesObjectExist(bucketName, keyName)); - } - - @Test - public void testCompleteMultipartUploadIfNoneMatchFail() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // First, create an existing key - s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream("existing".getBytes(StandardCharsets.UTF_8)), - new ObjectMetadata()); - - // Initiate multipart upload for same key - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "part1data"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-None-Match: * (key exists, should fail) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-None-Match", "*"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.completeMultipartUpload(completeRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - } - - @Test - public void testCompleteMultipartUploadIfMatch() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // First, create an existing key and get its ETag - PutObjectResult existingResult = s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream("existing".getBytes(StandardCharsets.UTF_8)), - new ObjectMetadata()); - String existingETag = existingResult.getETag(); - - // Initiate multipart upload for same key - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "newcontent"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-Match: (should succeed) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-Match", "\"" + existingETag + "\""); - - CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(completeRequest); - assertNotNull(result.getETag()); - assertNotEquals(existingETag, stripQuotes(result.getETag())); - } - - @Test - public void testCompleteMultipartUploadIfMatchFail() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // First, create an existing key - s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream("existing".getBytes(StandardCharsets.UTF_8)), - new ObjectMetadata()); - - // Initiate multipart upload for same key - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "newcontent"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-Match: wrong-etag (should fail) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-Match", "\"wrong-etag\""); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.completeMultipartUpload(completeRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(412, ase.getStatusCode()); - assertEquals("PreconditionFailed", ase.getErrorCode()); - } - - @ParameterizedTest - @ValueSource(strings = {"*", "\"some-etag\""}) - public void testCompleteMultipartUploadIfMatchMissingKeyFail(String ifMatch) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - - // Initiate multipart upload (key doesn't exist) - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(bucketName, keyName); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Upload a part - String partContent = "newcontent"; - byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(keyName) - .withUploadId(uploadId) - .withPartNumber(1) - .withInputStream(new ByteArrayInputStream(partBytes)) - .withPartSize(partBytes.length); - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - - // Complete with If-Match on non-existent key (should fail) - List partETags = Collections.singletonList(uploadResult.getPartETag()); - CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( - bucketName, keyName, uploadId, partETags); - completeRequest.putCustomRequestHeader("If-Match", ifMatch); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.completeMultipartUpload(completeRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchKey", ase.getErrorCode()); - } - - @Test - public void testPutDoubleSlashPrefixObject() throws IOException { - final String bucketName = getBucketName(); - final String keyName = "//dir1"; - final String content = "bar"; - // Create a FSO bucket for test - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - OzoneVolume volume = store.getS3Volume(); - OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder() - .setVolumeName(volume.getName()) - .setBucketName(bucketName) - .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); - OzoneManagerProtocol ozoneManagerProtocol = store.getClientProxy().getOzoneManagerClient(); - ozoneManagerProtocol.createBucket(bucketInfo.build()); - } - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); - - S3Object object = s3Client.getObject(bucketName, keyName); - assertEquals(content.length(), object.getObjectMetadata().getContentLength()); - } - - @Test - public void testPutObjectEmpty() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = ""; - s3Client.createBucket(bucketName); - - long initialAllocatedBlocks = - cluster.getStorageContainerManager().getPipelineManager() - .getMetrics().getTotalNumBlocksAllocated(); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - assertEquals("d41d8cd98f00b204e9800998ecf8427e", putObjectResult.getETag()); - - // Verify via Ozone client that the key has no block locations - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - OzoneVolume volume = store.getS3Volume(); - OzoneBucket bucket = volume.getBucket(bucketName); - - OzoneKeyDetails keyDetails = bucket.getKey(keyName); - assertNotNull(keyDetails); - assertEquals(0, keyDetails.getDataSize(), - "Empty S3 object should have dataSize of 0"); - assertTrue(keyDetails.getOzoneKeyLocations().isEmpty(), - "Empty S3 object should have no block locations"); - } - - // createKey should skip block allocation if data size is 0 - long currentAllocatedBlocks = - cluster.getStorageContainerManager().getPipelineManager().getMetrics().getTotalNumBlocksAllocated(); - assertEquals(initialAllocatedBlocks, currentAllocatedBlocks); - } - - @Test - public void testPutObjectACL() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - final byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - String originalObjectETag = putObjectResult.getETag(); - assertTrue(s3Client.doesObjectExist(bucketName, keyName)); - - AccessControlList aclList = new AccessControlList(); - Owner owner = new Owner("owner", "owner"); - aclList.withOwner(owner); - Grantee grantee = new CanonicalGrantee("testGrantee"); - aclList.grantPermission(grantee, Permission.Read); - - SetObjectAclRequest setObjectAclRequest = new SetObjectAclRequest(bucketName, keyName, aclList); - - assertNotImplemented(() -> s3Client.setObjectAcl(setObjectAclRequest)); - - // Ensure that the object content remains unchanged - ObjectMetadata updatedObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(originalObjectETag, updatedObjectMetadata.getETag()); - S3Object updatedObject = s3Client.getObject(bucketName, keyName); - - try (S3ObjectInputStream s3is = updatedObject.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(contentBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(content, bos.toString("UTF-8")); - } - } - - @Test - public void testGetObjectAclNotImplemented() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); - s3Client.putObject( - bucketName, keyName, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); - - assertNotImplemented(() -> s3Client.getObjectAcl(new GetObjectAclRequest(bucketName, keyName))); - } - - @Test - public void testGetBucketAccelerateConfigurationNotImplemented() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - assertNotImplemented( - () -> s3Client.getBucketAccelerateConfiguration(new GetBucketAccelerateConfigurationRequest(bucketName))); - } - - @Test - public void testGetObject() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - final byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - s3Client.createBucket(bucketName); - - InputStream is = new ByteArrayInputStream(contentBytes); - ObjectMetadata objectMetadata = new ObjectMetadata(); - Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); - objectMetadata.setUserMetadata(userMetadata); - - List tags = Arrays.asList(new Tag("tag1", "value1"), new Tag("tag2", "value2")); - ObjectTagging objectTagging = new ObjectTagging(tags); - - - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, is, objectMetadata) - .withTagging(objectTagging); - - s3Client.putObject(putObjectRequest); - - S3Object s3Object = s3Client.getObject(bucketName, keyName); - assertEquals(tags.size(), s3Object.getTaggingCount()); - - try (S3ObjectInputStream s3is = s3Object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(contentBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(content, bos.toString("UTF-8")); - } - } - - /** - * Adapted from ceph s3-tests test_object_read_unreadable. - */ - @Test - public void testGetObjectUnreadableKey() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, - StandardCharsets.ISO_8859_1); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(bucketName, unreadableKey)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(400, ase.getStatusCode()); - assertEquals(S3ErrorTable.INVALID_URI.getCode(), ase.getErrorCode()); - assertEquals(S3ErrorTable.INVALID_URI.getErrorMessage(), ase.getErrorMessage()); - } - - static Stream onlyTagKeyCasesV1() { - Map fooBarEmptyBar = new HashMap<>(); - fooBarEmptyBar.put("foo", "bar"); - fooBarEmptyBar.put("bar", ""); - return Stream.of( - Arguments.of( - new ObjectTagging(Collections.singletonList(new Tag("tag1", null))), - Collections.singletonMap("tag1", "")), - Arguments.of( - new ObjectTagging(Arrays.asList(new Tag("foo", "bar"), new Tag("bar", null))), - fooBarEmptyBar) - ); - } - - @ParameterizedTest - @MethodSource("onlyTagKeyCasesV1") - public void testPutObjectWithOnlyTagKey(ObjectTagging objectTagging, - Map expectedTags) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "0123456789"; - s3Client.createBucket(bucketName); - - try (InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) { - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, is, new ObjectMetadata()) - .withTagging(objectTagging); - s3Client.putObject(putObjectRequest); - } - - GetObjectTaggingResult taggingResult = s3Client.getObjectTagging( - new GetObjectTaggingRequest(bucketName, keyName)); - Map actualTags = taggingResult.getTagSet().stream() - .collect(Collectors.toMap(Tag::getKey, Tag::getValue)); - assertEquals(expectedTags, actualTags); - } - - @Test - public void testHeadObjectReturnsTaggingCount() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "head-object-tag-count"; - s3Client.createBucket(bucketName); - - s3Client.putObject(bucketName, keyName, - new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); - - List tags = Arrays.asList(new Tag("tag1", "v1"), new Tag("tag2", "v2")); - s3Client.setObjectTagging( - new SetObjectTaggingRequest(bucketName, keyName, new ObjectTagging(tags))); - - ObjectMetadata head = s3Client.getObjectMetadata(bucketName, keyName); - // AWS SDK v1: getTaggingCount() exists on S3Object (GET), not on ObjectMetadata (HEAD). - // x-amz-tagging-count is exposed via raw metadata. - Object tagCountHeader = head.getRawMetadataValue(Headers.S3_TAGGING_COUNT); - assertNotNull(tagCountHeader); - assertEquals(tags.size(), Integer.parseInt(tagCountHeader.toString())); - } - - @Test - public void testGetObjectTaggingReturnsTagsSortedByKey() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, ""); - - List tagsPutOrder = Arrays.asList(new Tag("key2", "val2"), new Tag("key", "val")); - s3Client.setObjectTagging( - new SetObjectTaggingRequest(bucketName, keyName, new ObjectTagging(tagsPutOrder))); - - GetObjectTaggingResult taggingResult = - s3Client.getObjectTagging(new GetObjectTaggingRequest(bucketName, keyName)); - List tagSet = taggingResult.getTagSet(); - assertEquals(2, tagSet.size()); - assertEquals("key", tagSet.get(0).getKey()); - assertEquals("val", tagSet.get(0).getValue()); - assertEquals("key2", tagSet.get(1).getKey()); - assertEquals("val2", tagSet.get(1).getValue()); - } - - @Test - public void testBucketTaggingPutGetDelete() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // AWS SDK v1 returns null when no bucket tagging is configured. - assertNull(s3Client.getBucketTaggingConfiguration( - new GetBucketTaggingConfigurationRequest(bucketName))); - - TagSet tagSet = new TagSet(); - tagSet.setTag("tag-key1", "tag-value1"); - tagSet.setTag("tag-key2", "tag-value2"); - s3Client.setBucketTaggingConfiguration(new SetBucketTaggingConfigurationRequest(bucketName, - new BucketTaggingConfiguration(Collections.singletonList(tagSet)))); - - BucketTaggingConfiguration taggingConfiguration = - s3Client.getBucketTaggingConfiguration(new GetBucketTaggingConfigurationRequest(bucketName)); - Map actualTags = taggingConfiguration.getTagSet().getAllTags(); - assertEquals(2, actualTags.size()); - assertEquals("tag-value1", actualTags.get("tag-key1")); - assertEquals("tag-value2", actualTags.get("tag-key2")); - - s3Client.deleteBucketTaggingConfiguration(new DeleteBucketTaggingConfigurationRequest(bucketName)); - - assertNull(s3Client.getBucketTaggingConfiguration( - new GetBucketTaggingConfigurationRequest(bucketName))); - } - - @Test - public void testGetObjectWithoutETag() throws Exception { - // Object uploaded using other protocols (e.g. ofs / ozone cli) will not - // have ETag. Ensure that ETag will not do ETag validation on GetObject if there - // is no ETag present. - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - String value = "sample value"; - byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); - - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - - OzoneVolume volume = store.getS3Volume(); - OzoneBucket bucket = volume.getBucket(bucketName); - - try (OzoneOutputStream out = bucket.createKey(keyName, - valueBytes.length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), - Collections.emptyMap())) { - out.write(valueBytes); - } - } - - S3Object s3Object = s3Client.getObject(bucketName, keyName); - assertNull(s3Object.getObjectMetadata().getETag()); - - try (S3ObjectInputStream s3is = s3Object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(valueBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(value, bos.toString("UTF-8")); - } - } - - @Test - public void testListObjectsMany() throws Exception { - testListObjectsMany(false); - } - - @Test - public void testListObjectsManyV2() throws Exception { - testListObjectsMany(true); - } - - @Test - public void testListObjectsSpecialKeyNames() throws Exception { - final String bucketName = getBucketName("special-keys"); - final String content = "x"; - s3Client.createBucket(bucketName); - - for (String keyName : S3SDKTestUtils.S3_SPECIAL_KEY_NAMES) { - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); - try (S3Object object = s3Client.getObject(bucketName, keyName)) { - assertEquals(content, IOUtils.toString(object.getObjectContent(), StandardCharsets.UTF_8)); - } - } - - ObjectListing listObjectsResponse = s3Client.listObjects( - new ListObjectsRequest().withBucketName(bucketName)); - List listedKeys = listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey) - .collect(Collectors.toList()); - assertEquals(S3SDKTestUtils.S3_SPECIAL_KEY_NAMES, listedKeys); - } - - private void testListObjectsMany(boolean isListV2) throws Exception { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - final List keyNames = Arrays.asList( - getKeyName("1"), - getKeyName("2"), - getKeyName("3") - ); - final List keyNamesWithoutETag = Arrays.asList( - getKeyName("4"), - getKeyName("5") - ); - - final Map keyToEtag = new HashMap<>(); - for (String keyName: keyNames) { - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, - RandomStringUtils.secure().nextAlphanumeric(5)); - keyToEtag.put(keyName, putObjectResult.getETag()); - } - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - - OzoneVolume volume = store.getS3Volume(); - OzoneBucket bucket = volume.getBucket(bucketName); - - for (String keyNameWithoutETag : keyNamesWithoutETag) { - byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); - try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, - valueBytes.length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), - Collections.emptyMap())) { - out.write(valueBytes); - } - } - } - - List objectSummaries; - String continuationToken; - if (isListV2) { - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(2); - ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertTrue(listObjectsResponse.isTruncated()); - continuationToken = listObjectsResponse.getNextContinuationToken(); - } else { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(2); - ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertTrue(listObjectsResponse.isTruncated()); - continuationToken = listObjectsResponse.getNextMarker(); - } - assertThat(objectSummaries).hasSize(2); - assertEquals(objectSummaries.stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(0, 2)); - for (S3ObjectSummary objectSummary : objectSummaries) { - assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); - } - - // Include both keys with and without ETag - if (isListV2) { - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(5) - .withContinuationToken(continuationToken); - ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertFalse(listObjectsResponse.isTruncated()); - } else { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(5) - .withMarker(continuationToken); - ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); - objectSummaries = listObjectsResponse.getObjectSummaries(); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertFalse(listObjectsResponse.isTruncated()); - } - - assertThat(objectSummaries).hasSize(3); - assertEquals(keyNames.get(2), objectSummaries.get(0).getKey()); - assertEquals(keyNamesWithoutETag.get(0), objectSummaries.get(1).getKey()); - assertEquals(keyNamesWithoutETag.get(1), objectSummaries.get(2).getKey()); - for (S3ObjectSummary objectSummary : objectSummaries) { - assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); - } - } - - @Test - public void testListObjectsBucketNotExist() { - final String bucketName = getBucketName(); - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName); - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.listObjects(listObjectsRequest)); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchBucket", ase.getErrorCode()); - } - - @Test - public void testListObjectsV2BucketNotExist() { - final String bucketName = getBucketName(); - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName); - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.listObjectsV2(listObjectsRequest)); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchBucket", ase.getErrorCode()); - } - - @Test - public void testListObjectsV2FetchOwner() { - final String bucketName = getBucketName("fetch-owner"); - final String keyName = getKeyName("obj"); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, RandomStringUtils.secure().nextAlphanumeric(5)); - - ListObjectsV2Result defaultResponse = s3Client.listObjectsV2( - new ListObjectsV2Request().withBucketName(bucketName)); - assertThat(defaultResponse.getObjectSummaries()).isNotEmpty(); - assertNull(defaultResponse.getObjectSummaries().get(0).getOwner()); - - ListObjectsV2Result falseResponse = s3Client.listObjectsV2( - new ListObjectsV2Request().withBucketName(bucketName).withFetchOwner(false)); - assertNull(falseResponse.getObjectSummaries().get(0).getOwner()); - - ListObjectsV2Result trueResponse = s3Client.listObjectsV2( - new ListObjectsV2Request().withBucketName(bucketName).withFetchOwner(true)); - Owner owner = trueResponse.getObjectSummaries().get(0).getOwner(); - assertNotNull(owner); - assertNotNull(owner.getDisplayName()); - assertEquals(S3Owner.DEFAULT_S3OWNER_ID, owner.getId()); - } - - @Test - public void testHighLevelMultipartUpload(@TempDir Path tempDir) throws Exception { - TransferManager tm = TransferManagerBuilder.standard() - .withS3Client(s3Client) - .build(); - - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - // The minimum file size to for TransferManager to initiate multipart upload is 16MB, so create a file - // larger than the threshold. - // See TransferManagerConfiguration#getMultipartUploadThreshold - int fileSize = (int) (20 * MB); - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, fileSize); - - // TransferManager processes all transfers asynchronously, - // so this call returns immediately. - Upload upload = tm.upload(bucketName, keyName, multipartUploadFile); - - upload.waitForCompletion(); - UploadResult uploadResult = upload.waitForUploadResult(); - assertEquals(bucketName, uploadResult.getBucketName()); - assertEquals(keyName, uploadResult.getKey()); - } - - @Test - public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); - - List tags = Arrays.asList(new Tag("tag1", "value1"), new Tag("tag2", "value2")); - - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (25 * MB)); - - multipartUpload(bucketName, keyName, multipartUploadFile, 5 * MB, null, userMetadata, tags); - - S3Object s3Object = s3Client.getObject(bucketName, keyName); - assertEquals(keyName, s3Object.getKey()); - assertEquals(bucketName, s3Object.getBucketName()); - assertEquals(tags.size(), s3Object.getTaggingCount()); - - ObjectMetadata objectMetadata = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(userMetadata, objectMetadata.getUserMetadata()); - } - - @Test - public void testListMultipartUploads() { - final String bucketName = getBucketName(); - final String multipartKey1 = getKeyName("multipart1"); - final String multipartKey2 = getKeyName("multipart2"); - - s3Client.createBucket(bucketName); - - List uploadIds = new ArrayList<>(); - - String uploadId1 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); - uploadIds.add(uploadId1); - String uploadId2 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); - uploadIds.add(uploadId2); - - String uploadId3 = initiateMultipartUpload(bucketName, multipartKey2, null, null, null); - uploadIds.add(uploadId3); - - ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName); - listMultipartUploadsRequest.setMaxUploads(5000); - - MultipartUploadListing result = s3Client.listMultipartUploads(listMultipartUploadsRequest); - - List listUploadIds = result.getMultipartUploads().stream() - .map(MultipartUpload::getUploadId) - .collect(Collectors.toList()); - - assertEquals(uploadIds, listUploadIds); - } - - @ParameterizedTest - @ValueSource(ints = {10, 5000}) - public void testListMultipartUploadsPagination(int requestedMaxUploads) { - final String bucketName = getBucketName() + "-" + requestedMaxUploads; - final String multipartKeyPrefix = getKeyName("multipart"); - - s3Client.createBucket(bucketName); - - // Create multipart uploads to test pagination - List allKeys = new ArrayList<>(); - Map keyToUploadId = new HashMap<>(); - - final int effectiveMaxUploads = Math.min(requestedMaxUploads, MAX_UPLOADS_LIMIT); - final int uploadsCreated = 2 * effectiveMaxUploads + 5; - final int expectedPages = uploadsCreated / effectiveMaxUploads + 1; - - for (int i = 0; i < uploadsCreated; i++) { - String key = String.format("%s-%04d", multipartKeyPrefix, i); - allKeys.add(key); - String uploadId = initiateMultipartUpload(bucketName, key, null, null, null); - keyToUploadId.put(key, uploadId); - } - Collections.sort(allKeys); - - // Test pagination - Set retrievedKeys = new HashSet<>(); - String keyMarker = null; - String uploadIdMarker = null; - boolean truncated = true; - int pageCount = 0; - - do { - ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(bucketName) - .withMaxUploads(requestedMaxUploads) - .withKeyMarker(keyMarker) - .withUploadIdMarker(uploadIdMarker); - - MultipartUploadListing result = s3Client.listMultipartUploads(request); - pageCount++; - - // Verify page size - if (pageCount < expectedPages) { - assertEquals(effectiveMaxUploads, result.getMultipartUploads().size()); - assertTrue(result.isTruncated()); - } else { - assertEquals(uploadsCreated % effectiveMaxUploads, result.getMultipartUploads().size()); - assertFalse(result.isTruncated()); - } - - // Collect keys and verify uploadIds - for (MultipartUpload upload : result.getMultipartUploads()) { - String key = upload.getKey(); - retrievedKeys.add(key); - assertEquals(keyToUploadId.get(key), upload.getUploadId()); - } - - // Verify response - assertNull(result.getPrefix()); - assertEquals(result.getUploadIdMarker(), uploadIdMarker); - assertEquals(result.getKeyMarker(), keyMarker); - assertEquals(effectiveMaxUploads, result.getMaxUploads()); - - // Verify next markers content - if (result.isTruncated()) { - MultipartUpload lastUploadOnPage = result.getMultipartUploads() - .get(result.getMultipartUploads().size() - 1); - assertEquals(lastUploadOnPage.getKey(), result.getNextKeyMarker()); - assertEquals(lastUploadOnPage.getUploadId(), result.getNextUploadIdMarker()); - } else { - assertNull(result.getNextKeyMarker()); - assertNull(result.getNextUploadIdMarker()); - } - - // Update markers for next page - keyMarker = result.getNextKeyMarker(); - uploadIdMarker = result.getNextUploadIdMarker(); - - truncated = result.isTruncated(); - - } while (truncated); - - // Verify pagination results - assertEquals(expectedPages, pageCount); - assertEquals(uploadsCreated, retrievedKeys.size(), "Should retrieve all uploads"); - assertEquals( - allKeys, - retrievedKeys.stream().sorted().collect(Collectors.toList()), - "Retrieved keys should match expected keys in order"); - - // Test with prefix - String prefix = multipartKeyPrefix + "-001"; - ListMultipartUploadsRequest prefixRequest = new ListMultipartUploadsRequest(bucketName) - .withPrefix(prefix); - - MultipartUploadListing prefixResult = s3Client.listMultipartUploads(prefixRequest); - - assertEquals(prefix, prefixResult.getPrefix()); - assertEquals( - IntStream.rangeClosed(0, 9) - .mapToObj(i -> prefix + i) - .collect(Collectors.toList()), - prefixResult.getMultipartUploads().stream() - .map(MultipartUpload::getKey) - .collect(Collectors.toList())); - } - - @Test - public void testListMultipartUploadsPaginationCornerCases() { - final String bucketName = getBucketName(); - final String keyA = getKeyName("samekey"); - final String keyB = getKeyName("after"); - - s3Client.createBucket(bucketName); - - // Create multiple MPUs for the same key to verify upload-id-marker semantics - List keyAUploadIds = new ArrayList<>(); - keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null, null)); - keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null, null)); - keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null, null)); - // Also create another key to ensure listing proceeds past keyA - initiateMultipartUpload(bucketName, keyB, null, null, null); - - // Sort upload IDs lexicographically to match listing order for the same key - Collections.sort(keyAUploadIds); - - // Case 1: key-marker=keyA and upload-id-marker set to the lowest uploadId - // Per spec, same-key uploads MAY be included if uploadId > marker - ListMultipartUploadsRequest request1 = new ListMultipartUploadsRequest(bucketName) - .withKeyMarker(keyA) - .withUploadIdMarker(keyAUploadIds.get(0)) - .withMaxUploads(100); - MultipartUploadListing result1 = s3Client.listMultipartUploads(request1); - - List uploads1 = result1.getMultipartUploads(); - // Collect same-key uploads and verify none are <= marker - List sameKeyIds1 = uploads1.stream() - .filter(u -> keyA.equals(u.getKey())) - .map(MultipartUpload::getUploadId) - .collect(Collectors.toList()); - assertThat(sameKeyIds1).allSatisfy(id -> assertTrue(id.compareTo(keyAUploadIds.get(0)) > 0)); - - // Case 2: key-marker=keyA and upload-id-marker set to the highest uploadId - // Expect no same-key (keyA) uploads to be returned - ListMultipartUploadsRequest request2 = new ListMultipartUploadsRequest(bucketName) - .withKeyMarker(keyA) - .withUploadIdMarker(keyAUploadIds.get(2)) - .withMaxUploads(100); - MultipartUploadListing result2 = s3Client.listMultipartUploads(request2); - assertTrue(result2.getMultipartUploads().stream().noneMatch(u -> keyA.equals(u.getKey()))); - } - - @Test - public void testListParts(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final long fileSize = 5 * MB; - final long partSize = 1 * MB; - final int maxParts = 2; - - s3Client.createBucket(bucketName); - - String uploadId = initiateMultipartUpload(bucketName, keyName, null, null, null); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) fileSize); - - List partETags = uploadParts(bucketName, keyName, uploadId, multipartUploadFile, partSize); - - List listPartETags = new ArrayList<>(); - int partNumberMarker = 0; - int expectedNumOfParts = 5; - PartListing listPartsResult; - do { - ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, keyName, uploadId) - .withMaxParts(maxParts) - .withPartNumberMarker(partNumberMarker); - listPartsResult = s3Client.listParts(listPartsRequest); - if (expectedNumOfParts > maxParts) { - assertTrue(listPartsResult.isTruncated()); - partNumberMarker = listPartsResult.getNextPartNumberMarker(); - expectedNumOfParts -= maxParts; - } else { - assertFalse(listPartsResult.isTruncated()); - } - for (PartSummary partSummary : listPartsResult.getParts()) { - listPartETags.add(new PartETag(partSummary.getPartNumber(), partSummary.getETag())); - } - } while (listPartsResult.isTruncated()); - - assertEquals(partETags.size(), listPartETags.size()); - for (int i = 0; i < partETags.size(); i++) { - assertEquals(partETags.get(i).getPartNumber(), listPartETags.get(i).getPartNumber()); - assertEquals(partETags.get(i).getETag(), listPartETags.get(i).getETag()); - } - } - - @Test - public void testGetParticularPart(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (15 * MB)); - - multipartUpload(bucketName, keyName, multipartUploadFile, 5 * MB, null, null, null); - - GetObjectRequest getObjectRequestAll = new GetObjectRequest(bucketName, keyName); - getObjectRequestAll.setPartNumber(0); - S3Object s3ObjectAll = s3Client.getObject(getObjectRequestAll); - long allPartContentLength = s3ObjectAll.getObjectMetadata().getContentLength(); - - GetObjectRequest getObjectRequestOne = new GetObjectRequest(bucketName, keyName); - getObjectRequestOne.setPartNumber(1); - S3Object s3ObjectOne = s3Client.getObject(getObjectRequestOne); - long partOneContentLength = s3ObjectOne.getObjectMetadata().getContentLength(); - assertEquals(allPartContentLength / 3, partOneContentLength); - - GetObjectRequest getObjectRequestTwo = new GetObjectRequest(bucketName, keyName); - getObjectRequestTwo.setPartNumber(2); - S3Object s3ObjectTwo = s3Client.getObject(getObjectRequestTwo); - long partTwoContentLength = s3ObjectTwo.getObjectMetadata().getContentLength(); - assertEquals(allPartContentLength / 3, partTwoContentLength); - - GetObjectRequest getObjectRequestThree = new GetObjectRequest(bucketName, keyName); - getObjectRequestThree.setPartNumber(1); - S3Object s3ObjectThree = s3Client.getObject(getObjectRequestTwo); - long partThreeContentLength = s3ObjectThree.getObjectMetadata().getContentLength(); - assertEquals(allPartContentLength / 3, partThreeContentLength); - - assertEquals(allPartContentLength, (partOneContentLength + partTwoContentLength + partThreeContentLength)); - } - - @Test - public void testGetNotExistedPart(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - - createFile(multipartUploadFile, (int) (15 * MB)); - - multipartUpload(bucketName, keyName, multipartUploadFile, 5 * MB, null, null, null); - - GetObjectRequest getObjectRequestOne = new GetObjectRequest(bucketName, keyName); - getObjectRequestOne.setPartNumber(4); - // Reading a part number beyond the object's part count must fail with - // InvalidPart, instead of returning an empty (0-byte) object. - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(getObjectRequestOne)); - assertEquals(400, ase.getStatusCode()); - assertEquals("InvalidPart", ase.getErrorCode()); - } - - @Test - public void testListPartsNotFound() { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - ListPartsRequest listPartsRequest = - new ListPartsRequest(bucketName, keyName, "nonexist"); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.listParts(listPartsRequest)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchUpload", ase.getErrorCode()); - } - - @Test - public void testQuotaExceeded() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - - s3Client.createBucket(bucketName); - - cluster.newClient().getObjectStore() - .getVolume("s3v") - .getBucket(bucketName) - .setQuota(OzoneQuota.parseQuota("1", "10")); - - // Upload some objects to the bucket - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(bucketName, keyName, - RandomStringUtils.secure().nextAlphanumeric(1024))); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(403, ase.getStatusCode()); - assertEquals("QuotaExceeded", ase.getErrorCode()); - } - - @Test - public void testS3LifecycleConfigurationCreateSuccessfully() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule rule1 = new BucketLifecycleConfiguration.Rule() - .withId("expire-logs-after-365-days") - .withPrefix("logs/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(365); - rules.add(rule1); - - configuration.setRules(rules); - - // Set lifecycle configuration - SetBucketLifecycleConfigurationRequest request = - new SetBucketLifecycleConfigurationRequest(bucketName, configuration); - s3Client.setBucketLifecycleConfiguration(request); - - // Verify the configuration was set - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - assertEquals(1, retrievedConfig.getRules().size()); - - // Verify rule 1 - BucketLifecycleConfiguration.Rule retrievedRule1 = retrievedConfig.getRules().get(0); - assertEquals("expire-logs-after-365-days", retrievedRule1.getId()); - assertEquals("logs/", retrievedRule1.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule1.getStatus()); - assertEquals(365, retrievedRule1.getExpirationInDays()); - } - - @Test - public void testS3LifecycleConfigurationCreationFailed() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Test 1: Invalid configuration - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - - BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule() - .withId("invalid") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rules.add(rule); - configuration.setRules(rules); - SetBucketLifecycleConfigurationRequest request = - new SetBucketLifecycleConfigurationRequest(bucketName, configuration); - - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.setBucketLifecycleConfiguration(request)); - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, ase.getStatusCode()); - - // Test 2: Non-existent bucket - final String nonExistentBucket = getBucketName("nonexistent"); - BucketLifecycleConfiguration validConfig = new BucketLifecycleConfiguration(); - List validRules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule validRule = new BucketLifecycleConfiguration.Rule() - .withId("test-rule") - .withPrefix("test/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(30); - validRules.add(validRule); - validConfig.setRules(validRules); - - SetBucketLifecycleConfigurationRequest nonExistentRequest = - new SetBucketLifecycleConfigurationRequest(nonExistentBucket, validConfig); - - AmazonServiceException ase2 = assertThrows(AmazonServiceException.class, - () -> s3Client.setBucketLifecycleConfiguration(nonExistentRequest)); - assertEquals(ErrorType.Client, ase2.getErrorType()); - assertEquals(HttpURLConnection.HTTP_NOT_FOUND, ase2.getStatusCode()); - assertEquals("NoSuchBucket", ase2.getErrorCode()); - - BucketLifecycleConfiguration pastDateConfig = new BucketLifecycleConfiguration(); - List pastDateRules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule pastDateRule = new BucketLifecycleConfiguration.Rule() - .withId("past-date") - .withPrefix("logs/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationDate(Date.from(Instant.parse("2020-01-01T00:00:00Z"))); - pastDateRules.add(pastDateRule); - pastDateConfig.setRules(pastDateRules); - SetBucketLifecycleConfigurationRequest pastDateRequest = - new SetBucketLifecycleConfigurationRequest(bucketName, pastDateConfig); - - AmazonServiceException ase3 = assertThrows(AmazonServiceException.class, - () -> s3Client.setBucketLifecycleConfiguration(pastDateRequest)); - assertEquals(ErrorType.Client, ase3.getErrorType()); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, ase3.getStatusCode()); - assertEquals(S3ErrorTable.INVALID_REQUEST.getCode(), ase3.getErrorCode()); - assertTrue(ase3.getErrorMessage().contains("must be in the future"), - "The client should receive OM's detailed message explaining the date is in the past, got: " - + ase3.getErrorMessage()); - } - - @Test - public void testS3LifecycleConfigurationDelete() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Test delete lifecycle for a bucket, while doesn't have lifecycle - assertNull(s3Client.getBucketLifecycleConfiguration(bucketName)); - // Idempotent delete: no exception expected even without an existing config - s3Client.deleteBucketLifecycleConfiguration(bucketName); - - // First create a lifecycle configuration - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule() - .withId("test-rule") - .withPrefix("test/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(30); - rules.add(rule); - configuration.setRules(rules); - - s3Client.setBucketLifecycleConfiguration(bucketName, configuration); - // Verify it exists - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - assertEquals(1, retrievedConfig.getRules().size()); - // Delete the lifecycle configuration - s3Client.deleteBucketLifecycleConfiguration(bucketName); - assertNull(s3Client.getBucketLifecycleConfiguration(bucketName)); - // Test delete on non-existent bucket - final String nonExistentBucket = getBucketName("nonexistent"); - assertThrows(AmazonServiceException.class, - () -> s3Client.deleteBucketLifecycleConfiguration(nonExistentBucket)); - } - - @Test - public void testS3LifecycleConfigurationGet() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - // Test get on bucket without lifecycle configuration - assertNull(s3Client.getBucketLifecycleConfiguration(bucketName)); - - // Create a comprehensive lifecycle configuration - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - - // Rule with expiration and prefix - BucketLifecycleConfiguration.Rule rule1 = new BucketLifecycleConfiguration.Rule() - .withId("expire-old-files") - .withPrefix("old/") - .withStatus(BucketLifecycleConfiguration.ENABLED) - .withExpirationInDays(365); - - rules.add(rule1); - configuration.setRules(rules); - - // Set the configuration - s3Client.setBucketLifecycleConfiguration(bucketName, configuration); - - // Get and verify the configuration - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - - assertEquals(1, retrievedConfig.getRules().size()); - - // Verify first rule - BucketLifecycleConfiguration.Rule retrievedRule1 = retrievedConfig.getRules().get(0); - assertEquals("expire-old-files", retrievedRule1.getId()); - assertEquals("old/", retrievedRule1.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule1.getStatus()); - assertEquals(365, retrievedRule1.getExpirationInDays()); - - // Test getting configuration using GetBucketLifecycleConfigurationRequest - GetBucketLifecycleConfigurationRequest getRequest = - new GetBucketLifecycleConfigurationRequest(bucketName); - BucketLifecycleConfiguration configFromRequest = - s3Client.getBucketLifecycleConfiguration(getRequest); - assertEquals(retrievedConfig.getRules().size(), configFromRequest.getRules().size()); - } - - @Test - public void testGetLifecycleWithAbortIncompleteMultipartUpload() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration(); - List rules = new ArrayList<>(); - - BucketLifecycleConfiguration.Rule rule1 = new BucketLifecycleConfiguration.Rule() - .withId("abort-incomplete-mpu-with-prefix") - .withPrefix("uploads/") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule1.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(7)); - - BucketLifecycleConfiguration.Rule rule2 = new BucketLifecycleConfiguration.Rule() - .withId("abort-incomplete-mpu-temp") - .withPrefix("temp/") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule2.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(3)); - - BucketLifecycleConfiguration.Rule rule3 = new BucketLifecycleConfiguration.Rule() - .withId("abort-incomplete-mpu-no-prefix") - .withPrefix("") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule3.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(29)); - - rules.add(rule1); - rules.add(rule2); - rules.add(rule3); - configuration.setRules(rules); - - // Set lifecycle configuration - s3Client.setBucketLifecycleConfiguration(bucketName, configuration); - - // Get and verify the configuration - BucketLifecycleConfiguration retrievedConfig = - s3Client.getBucketLifecycleConfiguration(bucketName); - - assertEquals(3, retrievedConfig.getRules().size()); - - BucketLifecycleConfiguration.Rule retrievedRule1 = retrievedConfig.getRules().get(0); - assertEquals("abort-incomplete-mpu-with-prefix", retrievedRule1.getId()); - assertEquals("uploads/", retrievedRule1.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule1.getStatus()); - assertEquals(7, retrievedRule1.getAbortIncompleteMultipartUpload().getDaysAfterInitiation()); - - BucketLifecycleConfiguration.Rule retrievedRule2 = retrievedConfig.getRules().get(1); - assertEquals("abort-incomplete-mpu-temp", retrievedRule2.getId()); - assertEquals("temp/", retrievedRule2.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule2.getStatus()); - assertEquals(3, retrievedRule2.getAbortIncompleteMultipartUpload().getDaysAfterInitiation()); - - BucketLifecycleConfiguration.Rule retrievedRule3 = retrievedConfig.getRules().get(2); - assertEquals("abort-incomplete-mpu-no-prefix", retrievedRule3.getId()); - assertEquals("", retrievedRule3.getPrefix()); - assertEquals(BucketLifecycleConfiguration.ENABLED, retrievedRule3.getStatus()); - assertEquals(29, retrievedRule3.getAbortIncompleteMultipartUpload().getDaysAfterInitiation()); - } - - /** - * End-to-end test verifying that KeyLifecycleService correctly aborts - * incomplete multipart uploads based on lifecycle configuration. - */ - @Test - void testAbortIncompleteMultipartUploadE2E() throws Exception { - OzoneManager ozoneManager = cluster().getOzoneManager(); - KeyLifecycleService lifecycleService = ozoneManager.getKeyManager().getKeyLifecycleService(); - if (lifecycleService == null) { - return; - } - - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - - String s3VolumeName; - try (OzoneClient ozoneClient = cluster().newClient()) { - s3VolumeName = ozoneClient.getObjectStore().getS3Volume().getName(); - } - - OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); - - // Create 3 MPUs - String matchingOldKey = "temp/old-file.txt"; - InitiateMultipartUploadResult mpu1 = s3Client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucketName, matchingOldKey)); - - String nonMatchingOldKey = "permanent/old-file.txt"; - InitiateMultipartUploadResult mpu2 = s3Client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucketName, nonMatchingOldKey)); - - String matchingRecentKey = "temp/recent-file.txt"; - s3Client.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, matchingRecentKey)); - - MultipartUploadListing listBefore = s3Client.listMultipartUploads( - new ListMultipartUploadsRequest(bucketName)); - assertEquals(3, listBefore.getMultipartUploads().size()); - - // Backdate 2 MPUs to 2 days ago - long oldCreationTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2); - updateMpuCreationTime(metadataManager, s3VolumeName, bucketName, - matchingOldKey, mpu1.getUploadId(), oldCreationTime); - updateMpuCreationTime(metadataManager, s3VolumeName, bucketName, - nonMatchingOldKey, mpu2.getUploadId(), oldCreationTime); - - // Set lifecycle rule: abort MPUs with prefix "temp/" after 1 day - BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule() - .withId("abort-temp-uploads") - .withPrefix("temp/") - .withStatus(BucketLifecycleConfiguration.ENABLED); - rule.setAbortIncompleteMultipartUpload( - new AbortIncompleteMultipartUpload().withDaysAfterInitiation(1)); - - BucketLifecycleConfiguration config = new BucketLifecycleConfiguration(); - config.setRules(Collections.singletonList(rule)); - s3Client.setBucketLifecycleConfiguration(bucketName, config); - - // Trigger lifecycle service - lifecycleService.runPeriodicalTaskNow(); - - // Wait for abort - GenericTestUtils.waitFor(() -> { - MultipartUploadListing listing = s3Client.listMultipartUploads( - new ListMultipartUploadsRequest(bucketName)); - return listing.getMultipartUploads().size() == 2; - }, 500, 30000); - - // Verify results - MultipartUploadListing listAfter = s3Client.listMultipartUploads( - new ListMultipartUploadsRequest(bucketName)); - List remainingKeys = listAfter.getMultipartUploads().stream() - .map(MultipartUpload::getKey) - .collect(Collectors.toList()); - - assertFalse(remainingKeys.contains(matchingOldKey), "Old MPU with matching prefix should be aborted"); - assertTrue(remainingKeys.contains(nonMatchingOldKey), "Old MPU with non-matching prefix should remain"); - assertTrue(remainingKeys.contains(matchingRecentKey), "Recent MPU should remain"); - } - - private void updateMpuCreationTime(OMMetadataManager metadataManager, - String volumeName, String bucketName, String keyName, - String uploadId, long newCreationTime) throws Exception { - String multipartKey = metadataManager.getMultipartKey(volumeName, bucketName, keyName, uploadId); - OmMultipartKeyInfo existingInfo = metadataManager.getMultipartInfoTable().get(multipartKey); - if (existingInfo == null) { - throw new RuntimeException("Multipart key info not found: " + multipartKey); - } - - OmMultipartKeyInfo updatedInfo = new OmMultipartKeyInfo.Builder() - .setUploadID(existingInfo.getUploadID()) - .setCreationTime(newCreationTime) - .setReplicationConfig(existingInfo.getReplicationConfig()) - .setObjectID(existingInfo.getObjectID()) - .setUpdateID(existingInfo.getUpdateID()) - .setParentID(existingInfo.getParentID()) - .build(); - - metadataManager.getMultipartInfoTable().put(multipartKey, updatedInfo); - } - - @Nested - @TestInstance(TestInstance.Lifecycle.PER_CLASS) - class PresignedUrlTests { - private static final String BUCKET_NAME = "v1-presigned-url-bucket"; - private static final String CONTENT = "bar"; - // Set the presigned URL to expire after one hour. - private final Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); - - @BeforeAll - public void setup() { - s3Client.createBucket(BUCKET_NAME); - } - - @Test - public void testPresignedUrlGet() throws IOException { - final String keyName = getKeyName(); - - InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); - - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - - // Generate the presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.GET).withExpiration(expiration); - generatePresignedUrlRequest.addRequestParameter("x-custom-parameter", "custom-value"); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - // Download the object using HttpUrlConnection (since v1.1) - // Capture the response body to a byte array. - HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); - // Download the result of executing the request. - try (InputStream s3is = connection.getInputStream(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) { - IOUtils.copy(s3is, bos); - assertEquals(CONTENT, bos.toString("UTF-8")); - } - } - - @ParameterizedTest - @ValueSource(strings = {"annotation", "torrent"}) - public void testPresignedUrlGetObjectSubresourceNotImplemented(String subresource) throws Exception { - final String keyName = getKeyName(); - - InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - - // AmazonS3 (SDK v1) has no getObjectTorrent or listObjectAnnotations API, so exercise the same HTTP behavior - // via a presigned URL with the subresource query parameter, as with other request shapes - // the typed v1 API doesn't expose. - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.GET).withExpiration(expiration); - generatePresignedUrlRequest.addRequestParameter(subresource, ""); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); - assertEquals(HttpURLConnection.HTTP_NOT_IMPLEMENTED, connection.getResponseCode()); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // object must be untouched - ObjectMetadata metadata = s3Client.getObjectMetadata(BUCKET_NAME, keyName); - assertEquals(CONTENT.length(), metadata.getContentLength()); - } - - @Test - public void testPresignedUrlHead() throws IOException { - final String keyName = getKeyName(); - - InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - - // Test HeadObject presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.HEAD).withExpiration(expiration); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "HEAD", null, null); - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, "HeadObject presigned URL should return 200 OK"); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // Test HeadBucket presigned URL - GeneratePresignedUrlRequest generateBucketPresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, null).withMethod(HttpMethod.HEAD).withExpiration(expiration); - URL presignedBucketUrl = s3Client.generatePresignedUrl(generateBucketPresignedUrlRequest); - - HttpURLConnection bucketConnection = null; - try { - bucketConnection = S3SDKTestUtils.openHttpURLConnection(presignedBucketUrl, "HEAD", null, null); - int bucketResponseCode = bucketConnection.getResponseCode(); - assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should return 200 OK"); - } finally { - if (bucketConnection != null) { - bucketConnection.disconnect(); - } - } - } - - @Test - public void testPresignedUrlPutObject() throws Exception { - final String keyName = getKeyName(); - - // Test PutObjectRequest presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT).withExpiration(expiration); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", - null, CONTENT.getBytes(StandardCharsets.UTF_8)); - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, "PutObject presigned URL should return 200 OK"); - String actualContent; - S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName); - try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { - actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } - assertEquals(CONTENT, actualContent, "Downloaded content should match uploaded content"); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - } - - @Test - public void testPresignedUrlPutSingleChunkWithWrongSha256() throws Exception { - final String keyName = getKeyName(); - - // Test PutObjectRequest presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT).withExpiration(expiration); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - Map> headers = new HashMap<>(); - List sha256Value = new ArrayList<>(); - sha256Value.add("wrong-sha256-value"); - headers.put("x-amz-content-sha256", sha256Value); - - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", - headers, CONTENT.getBytes(StandardCharsets.UTF_8)); - int responseCode = connection.getResponseCode(); - assertEquals(400, responseCode, "PutObject presigned URL should return 400 because of wrong SHA256"); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // Verify the object was not uploaded - AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.getObject(BUCKET_NAME, keyName)); - - assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchKey", ase.getErrorCode()); - } - - @Test - public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception { - final String keyName = getKeyName(); - final Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); - final Map tags = new HashMap<>(); - tags.put("tag1", "value1"); - tags.put("tag2", "value2"); - - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - createFile(multipartUploadFile, (int) (10 * MB)); - - // create MPU using presigned URL - GeneratePresignedUrlRequest initMPUPresignedUrlRequest = - createInitMPUPresignedUrlRequest(keyName, userMetadata, tags); - - - String uploadId = initMultipartUpload(initMPUPresignedUrlRequest); - - // upload parts using presigned URL - List completedParts = uploadParts(multipartUploadFile, keyName, uploadId); - - // Complete multipart upload using presigned URL - completeMPU(keyName, uploadId, completedParts); - - // Verify upload result - ObjectMetadata objectMeta = s3Client.getObjectMetadata(BUCKET_NAME, keyName); - assertEquals(userMetadata, objectMeta.getUserMetadata()); - - // Verify content - S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName); - assertEquals(tags.size(), s3Object.getTaggingCount()); - String actualContent; - try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { - actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } - String expectedContent = new String(Files.readAllBytes(multipartUploadFile.toPath()), StandardCharsets.UTF_8); - assertEquals(expectedContent, actualContent, "Downloaded content should match uploaded content"); - } - - private void completeMPU(String keyName, String uploadId, List completedParts) throws IOException { - GeneratePresignedUrlRequest request = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.POST).withExpiration(expiration); - request.addRequestParameter("uploadId", uploadId); - - URL presignedUrl = s3Client.generatePresignedUrl(request); - - // Generate completion XML payload - StringBuilder completionXml = new StringBuilder(); - completionXml.append("\n"); - for (PartETag part : completedParts) { - completionXml.append(" \n") - .append(" ").append(part.getPartNumber()).append("\n") - .append(" ").append(stripQuotes(part.getETag())).append("\n") - .append(" \n"); - } - completionXml.append(""); - - byte[] completionPayloadBytes = completionXml.toString().getBytes(StandardCharsets.UTF_8); - - HttpURLConnection httpConnection = null; - try { - httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "POST", null, completionPayloadBytes); - int responseCode = httpConnection.getResponseCode(); - assertEquals(200, responseCode, "Complete multipart upload should return 200 OK"); - } finally { - if (httpConnection != null) { - httpConnection.disconnect(); - } - } - } - - private List uploadParts(File multipartUploadFile, String keyName, String uploadId) throws IOException { - List completedParts = new ArrayList<>(); - ByteBuffer byteBuffer = ByteBuffer.allocate((int) (5 * MB)); - long filePosition = 0; - long fileLength = multipartUploadFile.length(); - int partNumber = 1; - - try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { - while (filePosition < fileLength) { - file.seek(filePosition); - long bytesRead = file.getChannel().read(byteBuffer); - byteBuffer.flip(); - - // generate presigned URL for each part - GeneratePresignedUrlRequest request = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT) - .withExpiration(expiration); - request.addRequestParameter("partNumber", String.valueOf(partNumber)); - request.addRequestParameter("uploadId", uploadId); - - URL presignedUrl = s3Client.generatePresignedUrl(request); - - // upload each part using presigned URL - HttpURLConnection connection = null; - try { - Map> headers = new HashMap<>(); - List header = Collections.singletonList(String.valueOf(byteBuffer.remaining())); - headers.put("Content-Length", header); - byte[] body = new byte[byteBuffer.remaining()]; - byteBuffer.get(body); - - connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", headers, body); - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, String.format("Upload part %d should return 200 OK", partNumber)); - - String etag = connection.getHeaderField("ETag"); - PartETag partETag = new PartETag(partNumber, etag); - completedParts.add(partETag); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - byteBuffer.clear(); - filePosition += bytesRead; - partNumber++; - } - } - return completedParts; - } - - private String initMultipartUpload(GeneratePresignedUrlRequest request) throws IOException { - URL presignedUrl = s3Client.generatePresignedUrl(request); - String uploadId; - HttpURLConnection httpConnection = null; - try { - Map customRequestHeaders = request.getCustomRequestHeaders(); - Map> headers = new HashMap<>(); - if (customRequestHeaders != null) { - for (Map.Entry entry : customRequestHeaders.entrySet()) { - headers.put(entry.getKey(), Collections.singletonList(entry.getValue())); - } - } - httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "POST", headers, null); - int initMPUConnectionResponseCode = httpConnection.getResponseCode(); - assertEquals(200, initMPUConnectionResponseCode); - - try (InputStream is = httpConnection.getInputStream()) { - String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); - uploadId = S3SDKTestUtils.extractUploadId(responseXml); - } - } finally { - if (httpConnection != null) { - httpConnection.disconnect(); - } - } - return uploadId; - } - - private GeneratePresignedUrlRequest createInitMPUPresignedUrlRequest(String keyName, - Map userMetadata, - Map tags) throws Exception { - GeneratePresignedUrlRequest initMPUPresignUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.POST).withExpiration(expiration); - - userMetadata.forEach((k, v) -> { - initMPUPresignUrlRequest.putCustomRequestHeader(CUSTOM_METADATA_HEADER_PREFIX + k, v); - }); - - StringBuilder tagValueBuilder = new StringBuilder(); - for (Map.Entry entry : tags.entrySet()) { - if (tagValueBuilder.length() > 0) { - tagValueBuilder.append('&'); - } - tagValueBuilder.append(entry.getKey()).append('=').append(URLEncoder.encode(entry.getValue(), "UTF-8")); - } - initMPUPresignUrlRequest.putCustomRequestHeader(S3Consts.TAG_HEADER, tagValueBuilder.toString()); - return initMPUPresignUrlRequest; - } - - @Test - public void testPresignedUrlDelete() throws IOException { - final String keyName = getKeyName(); - - try (InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8))) { - s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - } - - // Generate the presigned URL for DELETE - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.DELETE) - .withExpiration(expiration); - URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - // Execute the DELETE request using HttpUrlConnection - HttpURLConnection connection = null; - try { - connection = S3SDKTestUtils.openHttpURLConnection(url, "DELETE", null, null); - int responseCode = connection.getResponseCode(); - // Verify the response code is 204 (No Content) - assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - - // Verify the object is deleted - assertFalse(s3Client.doesObjectExist(BUCKET_NAME, keyName)); - } - } - - /** - * Tests the functionality to create a snapshot of an Ozone bucket and then read files - * from the snapshot directory using the S3 SDK. - * - *

The test follows these steps: - *

    - *
  1. Create a bucket and upload a file via the S3 client.
    2. - *
  2. Create a snapshot on the bucket using the Ozone client.
    3. - *
  3. Construct the snapshot object key using the ".snapshot" directory format.
    4. - *
  4. Retrieve the object from the snapshot and verify that its content matches - * the originally uploaded content.
    5. - *
- *

- * - * @throws Exception if the test fails due to any errors during bucket creation, snapshot creation, - * file upload, or retrieval. - */ - @Test - public void testReadSnapshotDirectoryUsingS3SDK() throws Exception { - final String bucketName = getBucketName("snapshot"); - final String keyName = getKeyName("snapshotfile"); - final String content = "snapshot test content"; - final byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - - // Create the bucket and upload an object using S3 SDK. - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.createBucket(bucketName); - s3Client.putObject(bucketName, keyName, - is, new ObjectMetadata()); - - String snapshotName = "snap1"; - // Create a snapshot using the Ozone client. - // Snapshots in Ozone are created on the bucket and are exposed via the ".snapshot" directory. - try (OzoneClient ozoneClient = cluster.newClient()) { - ObjectStore store = ozoneClient.getObjectStore(); - OzoneVolume volume = store.getS3Volume(); - - store.createSnapshot(volume.getName(), bucketName, snapshotName); - } - - // Use the S3 SDK to read the file from the snapshot directory. - // The key in the snapshot is constructed using the special ".snapshot" prefix. - String snapshotKey = ".snapshot/" + snapshotName + "/" + keyName; - - S3Object s3Object = s3Client.getObject(bucketName, snapshotKey); - try (S3ObjectInputStream s3is = s3Object.getObjectContent(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(contentBytes.length)) { - byte[] readBuf = new byte[1024]; - int readLen = 0; - while ((readLen = s3is.read(readBuf)) > 0) { - bos.write(readBuf, 0, readLen); - } - assertEquals(content, bos.toString("UTF-8")); - } - } - - private boolean isBucketEmpty(Bucket bucket) { - ObjectListing objectListing = s3Client.listObjects(bucket.getName()); - return objectListing.getObjectSummaries().isEmpty(); - } - - private String getBucketName() { - return getBucketName(""); - } - - private String getBucketName(String ignored) { - return uniqueObjectName(); - } - - private String getKeyName() { - return getKeyName(""); - } - - private String getKeyName(String ignored) { - return uniqueObjectName(); - } - - private static void assertNotImplemented(Executable request) { - final AmazonServiceException ase = assertThrows(AmazonServiceException.class, request); - assertEquals(S3ErrorTable.NOT_IMPLEMENTED.getCode(), ase.getErrorCode()); - assertEquals(501, ase.getStatusCode()); - assertEquals(ErrorType.Service, ase.getErrorType()); - } - - private String multipartUpload(String bucketName, String key, File file, long partSize, String contentType, - Map userMetadata, List tags) throws Exception { - String uploadId = initiateMultipartUpload(bucketName, key, contentType, userMetadata, tags); - - List partETags = uploadParts(bucketName, key, uploadId, file, partSize); - - completeMultipartUpload(bucketName, key, uploadId, partETags); - - return uploadId; - } - - private String initiateMultipartUpload(String bucketName, String key, String contentType, - Map metadata, List tags) { - InitiateMultipartUploadRequest initRequest; - if (metadata == null || metadata.isEmpty()) { - initRequest = new InitiateMultipartUploadRequest(bucketName, key); - } else { - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setUserMetadata(metadata); - if (contentType != null) { - objectMetadata.setContentType(contentType); - } - - initRequest = new InitiateMultipartUploadRequest(bucketName, key, objectMetadata) - .withTagging(new ObjectTagging(tags)); - } - - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - assertEquals(bucketName, initResponse.getBucketName()); - assertEquals(key, initResponse.getKey()); - // TODO: Once bucket lifecycle configuration is supported, should check for "abortDate" and "abortRuleId" - - return initResponse.getUploadId(); - } - - // TODO: Also support async upload parts (similar to v2 asyncClient) - private List uploadParts(String bucketName, String key, String uploadId, File file, long partSize) - throws Exception { - // Create a list of ETag objects. You retrieve ETags for each object part - // uploaded, - // then, after each individual part has been uploaded, pass the list of ETags to - // the request to complete the upload. - List partETags = new ArrayList<>(); - - // Upload the file parts. - long filePosition = 0; - long fileLength = file.length(); - try (InputStream fileInputStream = Files.newInputStream(file.toPath())) { - for (int i = 1; filePosition < fileLength; i++) { - // Because the last part could be less than 5 MB, adjust the part size as - // needed. - partSize = Math.min(partSize, (fileLength - filePosition)); - - // Create the request to upload a part. - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName) - .withKey(key) - .withUploadId(uploadId) - .withPartNumber(i) - .withFileOffset(filePosition) - .withFile(file) - .withPartSize(partSize); - - // Upload the part and add the response's ETag to our list. - UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); - PartETag partETag = uploadResult.getPartETag(); - assertEquals(i, partETag.getPartNumber()); - assertEquals(DatatypeConverter.printHexBinary( - calculateDigest(fileInputStream, 0, (int) partSize)).toLowerCase(), partETag.getETag()); - partETags.add(partETag); - - filePosition += partSize; - } - } - - return partETags; - } - - private void completeMultipartUpload(String bucketName, String key, String uploadId, List partETags) { - CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, key, - uploadId, partETags); - CompleteMultipartUploadResult compResponse = s3Client.completeMultipartUpload(compRequest); - assertEquals(bucketName, compResponse.getBucketName()); - assertEquals(key, compResponse.getKey()); - } - - private void abortMultipartUpload(String bucketName, String key, String uploadId) { - AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId); - s3Client.abortMultipartUpload(abortRequest); - } -} diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index d42284b08b0a..a3938fe91c34 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -63,18 +63,22 @@ import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import javax.xml.bind.DatatypeConverter; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; @@ -169,6 +173,8 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; import software.amazon.awssdk.services.s3.model.GetObjectTaggingResponse; +import software.amazon.awssdk.services.s3.model.Grant; +import software.amazon.awssdk.services.s3.model.Grantee; import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -188,12 +194,19 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.ListPartsRequest; +import software.amazon.awssdk.services.s3.model.ListPartsResponse; import software.amazon.awssdk.services.s3.model.MetadataDirective; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.NoSuchUploadException; import software.amazon.awssdk.services.s3.model.ObjectAttributes; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; import software.amazon.awssdk.services.s3.model.ObjectVersion; +import software.amazon.awssdk.services.s3.model.Owner; +import software.amazon.awssdk.services.s3.model.Part; +import software.amazon.awssdk.services.s3.model.Permission; import software.amazon.awssdk.services.s3.model.PutBucketAclRequest; import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -203,6 +216,7 @@ import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.Tag; import software.amazon.awssdk.services.s3.model.Tagging; +import software.amazon.awssdk.services.s3.model.Type; import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; import software.amazon.awssdk.services.s3.model.UploadPartCopyResponse; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -225,6 +239,7 @@ import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest; import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedFileUpload; import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; @@ -254,6 +269,7 @@ public abstract class AbstractS3SDKV2Tests extends OzoneTestBase implements NonH */ public static final String EC_PARALLEL_READ_S3G_HTTP_ADDRESS_KEY = "ozone.test.s3g.ec.parallel.read.http.address"; + private static final int MAX_UPLOADS_LIMIT = 1000; private static final String READ_CONSISTENCY_HEADER = "x-ozone-read-consistency"; private static final String LOCAL_LEASE_LOG_LIMIT_HEADER = @@ -314,6 +330,189 @@ public void testCreateBucketAlreadyExistsDifferentOwner() throws IOException { exception.awsErrorDetails().errorCode()); } + @Test + public void testCreateBucket() { + final String bucketName = getBucketName(); + + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.headBucket(b -> b.bucket(bucketName)); + s3Client.getBucketAcl(b -> b.bucket(bucketName)); + assertThat(s3Client.listObjectsV2(b -> b.bucket(bucketName)).contents()).isEmpty(); + } + + @Test + public void testBucketAclOperations() { + // TODO HDDS-11738: Assert the bucket ACL when bucket S3 ACL logic has been fixed + final String bucketName = getBucketName(); + final Owner owner = Owner.builder().id("owner").displayName("owner").build(); + final Grantee grantee = Grantee.builder().type(Type.CANONICAL_USER).id("testGrantee").build(); + + s3Client.createBucket(b -> b.bucket(bucketName).grantRead("id=testGrantee")); + + s3Client.putBucketAcl(b -> b.bucket(bucketName).accessControlPolicy(p -> p.owner(owner).grants( + Grant.builder().grantee(grantee).permission(Permission.READ).build(), + Grant.builder().grantee(grantee).permission(Permission.WRITE).build()))); + } + + @Test + public void testDeleteBucket() { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.deleteBucket(b -> b.bucket(bucketName)); + + final S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.headBucket(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + } + + @Test + public void testDeleteBucketNotExist() { + final String bucketName = getBucketName(); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.deleteBucket(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testDeleteBucketNonEmptyWithKeys() { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + for (int i = 1; i <= 10; i++) { + final String keyName = "key-" + i; + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(RandomStringUtils.secure().nextAlphanumeric(1024))); + } + + // Bucket deletion should fail if there are still keys in the bucket + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.deleteBucket(b -> b.bucket(bucketName))); + assertEquals(409, exception.statusCode()); + assertEquals(S3ErrorTable.BUCKET_NOT_EMPTY.getCode(), exception.awsErrorDetails().errorCode()); + + for (S3Object object : s3Client.listObjectsV2Paginator(b -> b.bucket(bucketName)).contents()) { + s3Client.deleteObject(b -> b.bucket(bucketName).key(object.key())); + } + } + + @Test + public void testDeleteBucketNonEmptyWithIncompleteMultipartUpload(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (5 * MB)); + + // An incomplete multipart upload: parts uploaded, but not completed + final String uploadId = initiateMultipartUpload(bucketName, keyName, new HashMap<>(), Collections.emptyList()); + uploadParts(bucketName, keyName, uploadId, multipartUploadFile, (int) MB); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.deleteBucket(b -> b.bucket(bucketName))); + assertEquals(409, exception.statusCode()); + assertEquals(S3ErrorTable.BUCKET_NOT_EMPTY.getCode(), exception.awsErrorDetails().errorCode()); + + // After the multipart upload is aborted, the bucket deletion should succeed + s3Client.abortMultipartUpload(b -> b.bucket(bucketName).key(keyName).uploadId(uploadId)); + s3Client.deleteBucket(b -> b.bucket(bucketName)); + assertThrows(NoSuchBucketException.class, () -> s3Client.headBucket(b -> b.bucket(bucketName))); + } + + @Test + public void testQuotaExceeded() throws IOException { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + try (OzoneClient ozoneClient = cluster.newClient()) { + ozoneClient.getObjectStore().getS3Volume().getBucket(bucketName).setQuota(OzoneQuota.parseQuota("1", "10")); + } + + final String content = RandomStringUtils.secure().nextAlphanumeric(1024); + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content))); + assertEquals(403, exception.statusCode()); + assertEquals(S3ErrorTable.QUOTA_EXCEEDED.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testGetObject() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + final Map userMetadata = new HashMap<>(); + userMetadata.put("key1", "value1"); + userMetadata.put("key2", "value2"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName).metadata(userMetadata) + .tagging("tag1=value1&tag2=value2"), RequestBody.fromString(content)); + + final ResponseBytes object = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)); + assertEquals(2, object.response().tagCount()); + assertEquals(userMetadata, object.response().metadata()); + assertEquals(content, object.asUtf8String()); + } + + @Test + public void testGetObjectWithoutETag() throws Exception { + // Object uploaded using other protocols (e.g. ofs / ozone cli) will not have ETag. + // Ensure that GetObject does not validate the ETag if there is none. + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String value = "sample value"; + final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); + s3Client.createBucket(b -> b.bucket(bucketName)); + try (OzoneClient ozoneClient = cluster.newClient()) { + final OzoneBucket bucket = ozoneClient.getObjectStore().getS3Volume().getBucket(bucketName); + try (OzoneOutputStream out = bucket.createKey(keyName, valueBytes.length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), Collections.emptyMap())) { + out.write(valueBytes); + } + } + + final ResponseBytes object = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)); + assertNull(object.response().eTag()); + assertEquals(value, object.asUtf8String()); + } + + @Test + public void testPutDoubleSlashPrefixObject() throws Exception { + final String bucketName = getBucketName(); + final String keyName = "//dir1"; + final String content = "bar"; + createFsoBucket(bucketName); + + final PutObjectResponse putObjectResponse = s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", putObjectResponse.eTag()); + + final ResponseBytes object = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)); + assertEquals(content.length(), object.response().contentLength()); + assertEquals(content, object.asUtf8String()); + } + + @Test + public void testPutObjectAcl() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + final String originalETag = s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)).eTag(); + final Owner owner = Owner.builder().id("owner").displayName("owner").build(); + final Grantee grantee = Grantee.builder().type(Type.CANONICAL_USER).id("testGrantee").build(); + + assertNotImplemented(() -> s3Client.putObjectAcl(b -> b.bucket(bucketName).key(keyName) + .accessControlPolicy(p -> p.owner(owner) + .grants(Grant.builder().grantee(grantee).permission(Permission.READ).build())))); + + // Ensure that the object remains unchanged + assertEquals(originalETag, s3Client.headObject(b -> b.bucket(bucketName).key(keyName)).eTag()); + assertEquals(content, s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)).asUtf8String()); + } + @Test public void testPutObject() { final String bucketName = getBucketName(); @@ -2960,6 +3159,27 @@ public void tearDown() { sdkHttpClient.close(); } + @Test + public void testPresignedUrlGetWithCustomQueryParameter() throws Exception { + final String keyName = getKeyName(); + s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName), RequestBody.fromString(CONTENT)); + + // a query parameter unknown to S3, signed with the request + final GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(BUCKET_NAME).key(keyName) + .overrideConfiguration(o -> o.putRawQueryParameter("x-custom-parameter", "custom-value")) + .build(); + final URL presignedUrl = presigner.presignGetObject(b -> b.signatureDuration(duration) + .getObjectRequest(objectRequest)).url(); + assertThat(presignedUrl.getQuery()).contains("x-custom-parameter=custom-value"); + + final HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); + try (InputStream s3is = connection.getInputStream()) { + assertEquals(CONTENT, IoUtils.toUtf8String(s3is)); + } finally { + connection.disconnect(); + } + } + @Test public void testPresignedUrlGet() throws Exception { final String keyName = getKeyName(); @@ -3958,6 +4178,267 @@ private void updateMpuCreationTime(OMMetadataManager metadataManager, metadataManager.getMultipartInfoTable().put(multipartKey, updatedInfo); } + @Test + public void testListObjectsSpecialKeyNames() { + final String bucketName = getBucketName("special-keys"); + final String content = "x"; + s3Client.createBucket(b -> b.bucket(bucketName)); + for (String keyName : S3SDKTestUtils.S3_SPECIAL_KEY_NAMES) { + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + } + + // ListObjects (the original API, not ListObjectsV2) + final ListObjectsResponse response = s3Client.listObjects(b -> b.bucket(bucketName)); + assertEquals(S3SDKTestUtils.S3_SPECIAL_KEY_NAMES, + response.contents().stream().map(S3Object::key).collect(Collectors.toList())); + } + + @Test + public void testListObjectsBucketNotExist() { + final String bucketName = getBucketName(); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.listObjects(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testListObjectsV2BucketNotExist() { + final String bucketName = getBucketName(); + + final S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.listObjectsV2(b -> b.bucket(bucketName))); + assertEquals(404, exception.statusCode()); + assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), exception.awsErrorDetails().errorCode()); + } + + @Test + public void testHighLevelMultipartUpload(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final long fileSize = 20 * MB; + final long partSize = 5 * MB; + s3Client.createBucket(b -> b.bucket(bucketName)); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) fileSize); + + // the transfer manager uploads in parallel parts when its client has multipart upload enabled + try (S3AsyncClient multipartClient = S3AsyncClient.builder() + .multipartEnabled(true) + .multipartConfiguration(c -> c.thresholdInBytes(partSize).minimumPartSizeInBytes(partSize)) + .forcePathStyle(true) + .disableS3ExpressSessionAuth(true) + .endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get()) + .region(s3Client.serviceClientConfiguration().region()) + .credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()) + .build(); + S3TransferManager transferManager = S3TransferManager.builder().s3Client(multipartClient).build()) { + final CompletedFileUpload upload = transferManager.uploadFile(b -> b + .putObjectRequest(r -> r.bucket(bucketName).key(keyName)) + .source(multipartUploadFile)).completionFuture().get(); + + // a multipart upload ETag ends with the number of parts + assertThat(upload.response().eTag()).endsWith("-" + fileSize / partSize + "\""); + } + assertEquals(fileSize, s3Client.headObject(b -> b.bucket(bucketName).key(keyName)).contentLength()); + } + + @Test + public void testListMultipartUploads() { + final String bucketName = getBucketName(); + final String multipartKey1 = getKeyName("multipart1"); + final String multipartKey2 = getKeyName("multipart2"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + final List uploadIds = new ArrayList<>(); + uploadIds.add(initiateMultipartUpload(bucketName, multipartKey1, new HashMap<>(), Collections.emptyList())); + uploadIds.add(initiateMultipartUpload(bucketName, multipartKey1, new HashMap<>(), Collections.emptyList())); + uploadIds.add(initiateMultipartUpload(bucketName, multipartKey2, new HashMap<>(), Collections.emptyList())); + + final ListMultipartUploadsResponse response = s3Client.listMultipartUploads(b -> b + .bucket(bucketName).maxUploads(5000)); + assertEquals(uploadIds, response.uploads().stream().map(MultipartUpload::uploadId).collect(Collectors.toList())); + } + + @ParameterizedTest + @ValueSource(ints = {10, 5000}) + public void testListMultipartUploadsPagination(int requestedMaxUploads) { + final String bucketName = getBucketName() + "-" + requestedMaxUploads; + final String multipartKeyPrefix = getKeyName("multipart"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + final List allKeys = new ArrayList<>(); + final Map keyToUploadId = new HashMap<>(); + final int effectiveMaxUploads = Math.min(requestedMaxUploads, MAX_UPLOADS_LIMIT); + final int uploadsCreated = 2 * effectiveMaxUploads + 5; + final int expectedPages = uploadsCreated / effectiveMaxUploads + 1; + for (int i = 0; i < uploadsCreated; i++) { + final String key = String.format("%s-%04d", multipartKeyPrefix, i); + allKeys.add(key); + keyToUploadId.put(key, initiateMultipartUpload(bucketName, key, new HashMap<>(), Collections.emptyList())); + } + Collections.sort(allKeys); + + final Set retrievedKeys = new HashSet<>(); + String keyMarker = null; + String uploadIdMarker = null; + boolean truncated; + int pageCount = 0; + do { + final String requestKeyMarker = keyMarker; + final String requestUploadIdMarker = uploadIdMarker; + final ListMultipartUploadsResponse result = s3Client.listMultipartUploads(b -> b.bucket(bucketName) + .maxUploads(requestedMaxUploads).keyMarker(requestKeyMarker).uploadIdMarker(requestUploadIdMarker)); + pageCount++; + + if (pageCount < expectedPages) { + assertEquals(effectiveMaxUploads, result.uploads().size()); + assertTrue(result.isTruncated()); + } else { + assertEquals(uploadsCreated % effectiveMaxUploads, result.uploads().size()); + assertFalse(result.isTruncated()); + } + for (MultipartUpload upload : result.uploads()) { + retrievedKeys.add(upload.key()); + assertEquals(keyToUploadId.get(upload.key()), upload.uploadId()); + } + + assertThat(result.prefix()).isNullOrEmpty(); + assertEquals(requestUploadIdMarker, emptyToNull(result.uploadIdMarker())); + assertEquals(requestKeyMarker, emptyToNull(result.keyMarker())); + assertEquals(effectiveMaxUploads, result.maxUploads()); + if (result.isTruncated()) { + final MultipartUpload lastUploadOnPage = result.uploads().get(result.uploads().size() - 1); + assertEquals(lastUploadOnPage.key(), result.nextKeyMarker()); + assertEquals(lastUploadOnPage.uploadId(), result.nextUploadIdMarker()); + } else { + assertNull(emptyToNull(result.nextKeyMarker())); + assertNull(emptyToNull(result.nextUploadIdMarker())); + } + + keyMarker = result.nextKeyMarker(); + uploadIdMarker = result.nextUploadIdMarker(); + truncated = result.isTruncated(); + } while (truncated); + + assertEquals(expectedPages, pageCount); + assertEquals(uploadsCreated, retrievedKeys.size(), "Should retrieve all uploads"); + assertEquals(allKeys, retrievedKeys.stream().sorted().collect(Collectors.toList()), + "Retrieved keys should match expected keys in order"); + + final String prefix = multipartKeyPrefix + "-001"; + final ListMultipartUploadsResponse prefixResult = s3Client.listMultipartUploads(b -> b + .bucket(bucketName).prefix(prefix)); + assertEquals(prefix, prefixResult.prefix()); + assertEquals(IntStream.rangeClosed(0, 9).mapToObj(i -> prefix + i).collect(Collectors.toList()), + prefixResult.uploads().stream().map(MultipartUpload::key).collect(Collectors.toList())); + } + + @Test + public void testListMultipartUploadsPaginationCornerCases() { + final String bucketName = getBucketName(); + final String keyA = getKeyName("samekey"); + final String keyB = getKeyName("after"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + // Several uploads for the same key, to verify the upload-id-marker semantics + final List keyAUploadIds = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, new HashMap<>(), Collections.emptyList())); + } + // Another key, to ensure that the listing proceeds past keyA + initiateMultipartUpload(bucketName, keyB, new HashMap<>(), Collections.emptyList()); + // Uploads of the same key are listed in upload id order + Collections.sort(keyAUploadIds); + + // upload-id-marker at the lowest upload id: only the larger upload ids of keyA are returned + final List sameKeyIds = s3Client.listMultipartUploads(b -> b.bucket(bucketName) + .keyMarker(keyA).uploadIdMarker(keyAUploadIds.get(0)).maxUploads(100)) + .uploads().stream().filter(u -> keyA.equals(u.key())).map(MultipartUpload::uploadId) + .collect(Collectors.toList()); + assertThat(sameKeyIds).allSatisfy(id -> assertThat(id).isGreaterThan(keyAUploadIds.get(0))); + + // upload-id-marker at the highest upload id: no upload of keyA is returned + assertThat(s3Client.listMultipartUploads(b -> b.bucket(bucketName) + .keyMarker(keyA).uploadIdMarker(keyAUploadIds.get(2)).maxUploads(100)).uploads()) + .noneMatch(u -> keyA.equals(u.key())); + } + + @Test + public void testListParts(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final int maxParts = 2; + s3Client.createBucket(b -> b.bucket(bucketName)); + final String uploadId = initiateMultipartUpload(bucketName, keyName, new HashMap<>(), Collections.emptyList()); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (5 * MB)); + final List uploadedParts = uploadParts(bucketName, keyName, uploadId, multipartUploadFile, + (int) MB); + + final List listedParts = new ArrayList<>(); + int partNumberMarker = 0; + int remainingParts = uploadedParts.size(); + ListPartsResponse response; + do { + final int marker = partNumberMarker; + response = s3Client.listParts(b -> b.bucket(bucketName).key(keyName).uploadId(uploadId) + .maxParts(maxParts).partNumberMarker(marker)); + if (remainingParts > maxParts) { + assertTrue(response.isTruncated()); + partNumberMarker = response.nextPartNumberMarker(); + remainingParts -= maxParts; + } else { + assertFalse(response.isTruncated()); + } + listedParts.addAll(response.parts()); + } while (response.isTruncated()); + + assertEquals(uploadedParts.size(), listedParts.size()); + for (int i = 0; i < uploadedParts.size(); i++) { + assertEquals(uploadedParts.get(i).partNumber(), listedParts.get(i).partNumber()); + assertEquals(uploadedParts.get(i).eTag(), stripQuotes(listedParts.get(i).eTag())); + } + } + + @Test + public void testListPartsNotFound() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + final S3Exception exception = assertThrows(NoSuchUploadException.class, + () -> s3Client.listParts(b -> b.bucket(bucketName).key(keyName).uploadId("nonexist"))); + assertEquals(404, exception.statusCode()); + } + + @Test + public void testGetParticularPart(@TempDir Path tempDir) throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + final File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (15 * MB)); + multipartUpload(bucketName, keyName, multipartUploadFile, (int) (5 * MB), new HashMap<>(), Collections.emptyList()); + + final long allPartsLength = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)) + .response().contentLength(); + long partsLength = 0; + for (int part = 1; part <= 3; part++) { + final int partNumber = part; + final long partLength = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName).partNumber(partNumber)) + .response().contentLength(); + assertEquals(allPartsLength / 3, partLength, "part " + partNumber); + partsLength += partLength; + } + assertEquals(allPartsLength, partsLength); + } + + private static String emptyToNull(String value) { + return value == null || value.isEmpty() ? null : value; + } + private String getBucketName() { return getBucketName(""); } diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties b/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties index 977e356b7ad3..27935677a2fa 100644 --- a/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties +++ b/hadoop-ozone/integration-test-s3/src/test/resources/log4j.properties @@ -23,8 +23,5 @@ log4j.logger.org.apache.hadoop.hdds.utils.db.managed=TRACE log4j.logger.org.apache.hadoop.hdds.utils.db.CodecBuffer=DEBUG log4j.logger.org.apache.hadoop.ozone.client.OzoneClientFactory=DEBUG -log4j.logger.com.amazonaws=WARN -log4j.logger.com.amazonaws.request=DEBUG - log4j.logger.software.amazon.awssdk=WARN log4j.logger.software.amazon.awssdk.request=DEBUG diff --git a/pom.xml b/pom.xml index a5d5c345c115..dcd5ca8bddee 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,6 @@ 1.9.25.1 1.9.7 3.27.7 - 1.12.788 2.55.6 0.8.0.RELEASE 1.86 @@ -363,23 +362,6 @@ - - com.amazonaws - aws-java-sdk-core - ${aws-java-sdk.version} - - - - commons-logging - commons-logging - - - - - com.amazonaws - aws-java-sdk-s3 - ${aws-java-sdk.version} - com.bettercloud vault-java-driver @@ -2121,6 +2103,23 @@ + + ban-aws-sdk-v1 + + enforce + + validate + + + + + com.amazonaws + + AWS SDK for Java v1 is end of life, use AWS SDK for Java v2 (software.amazon.awssdk) + + + + ban-imports @@ -2194,6 +2193,17 @@ ${project.build.directory}/generated-sources/protobuf/java + + true + AWS SDK for Java v1 is end of life, use AWS SDK for Java v2 (software.amazon.awssdk) + + com.amazonaws.** + + + ${project.build.directory}/generated-sources/java + ${project.build.directory}/generated-sources/protobuf/java + + true true From 19aac8dbfae8090a1172e59c9ce2e4bca996fca3 Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Wed, 4 Nov 2026 22:34:12 -0500 Subject: [PATCH 2/3] Addressed review comments --- .../dist/src/main/smoketest/s3/freon.robot | 3 + .../FreonEnvironmentCredentialsProvider.java | 67 ++++++++++++ .../hadoop/ozone/freon/S3EntityGenerator.java | 3 +- .../hadoop/ozone/freon/S3KeyGenerator.java | 3 +- ...stFreonEnvironmentCredentialsProvider.java | 101 ++++++++++++++++++ .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 28 +++++ 6 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java create mode 100644 hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot index 24ba88bba62f..f722f396c9ce 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot @@ -52,6 +52,9 @@ Run Freon S3BG Run Freon S3KG [Setup] Setup aws credentials Freon S3KG s3kg-${BUCKET} + # the content type that freon set for its keys with AWS SDK v1 + ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key s3kg-${BUCKET}/0 + Should contain ${result} "ContentType": "text/plain" Run Freon S3KG MPU [Setup] Setup aws credentials diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java new file mode 100644 index 000000000000..cc9ad72fd10b --- /dev/null +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.freon; + +import com.google.common.annotations.VisibleForTesting; +import java.util.function.UnaryOperator; +import org.apache.commons.lang3.StringUtils; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.core.exception.SdkClientException; + +/** + * Credentials from environment variables, read the way AWS SDK for Java v1 read them, which freon used before: + * the access key from {@code AWS_ACCESS_KEY_ID}, or else {@code AWS_ACCESS_KEY}, the secret key from + * {@code AWS_SECRET_KEY}, or else {@code AWS_SECRET_ACCESS_KEY}, and the session token from {@code AWS_SESSION_TOKEN}, + * if set. The environment provider of SDK v2 only reads {@code AWS_ACCESS_KEY_ID} and {@code AWS_SECRET_ACCESS_KEY}. + */ +public final class FreonEnvironmentCredentialsProvider implements AwsCredentialsProvider { + + private final UnaryOperator environment; + + public FreonEnvironmentCredentialsProvider() { + this(System::getenv); + } + + @VisibleForTesting + FreonEnvironmentCredentialsProvider(UnaryOperator environment) { + this.environment = environment; + } + + @Override + public AwsCredentials resolveCredentials() { + final String accessKey = StringUtils.trim(get("AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY")); + final String secretKey = StringUtils.trim(get("AWS_SECRET_KEY", "AWS_SECRET_ACCESS_KEY")); + if (StringUtils.isEmpty(accessKey) || StringUtils.isEmpty(secretKey)) { + throw SdkClientException.create("Unable to load AWS credentials from environment variables " + + "(AWS_ACCESS_KEY_ID (or AWS_ACCESS_KEY) and AWS_SECRET_KEY (or AWS_SECRET_ACCESS_KEY))"); + } + final String sessionToken = StringUtils.trim(environment.apply("AWS_SESSION_TOKEN")); + return StringUtils.isEmpty(sessionToken) + ? AwsBasicCredentials.create(accessKey, secretKey) + : AwsSessionCredentials.create(accessKey, secretKey, sessionToken); + } + + /** @return the value of the variable, or of the alternate one if the variable is not set at all */ + private String get(String variable, String alternate) { + final String value = environment.apply(variable); + return value != null ? value : environment.apply(alternate); + } +} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java index fab89c68fecc..fa988c30ea1c 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java @@ -19,7 +19,6 @@ import java.net.URI; import picocli.CommandLine; -import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; import software.amazon.awssdk.http.apache.ApacheHttpClient; @@ -43,7 +42,7 @@ protected void s3ClientInit() { S3ClientBuilder s3ClientBuilder = S3Client.builder() .httpClientBuilder(ApacheHttpClient.builder()) - .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .credentialsProvider(new FreonEnvironmentCredentialsProvider()) // checksums only where S3 requires them: a load generator should not spend its time computing them .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED) .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java index 6f04181f3399..835777ee6e31 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java @@ -122,7 +122,8 @@ private void createKey(long counter) throws Exception { .multipartUpload(m -> m.parts(parts))); } else { - getS3().putObject(r -> r.bucket(bucketName).key(generateObjectName(counter)), + // the content type that AWS SDK v1 set for the String content freon uploaded before + getS3().putObject(r -> r.bucket(bucketName).key(generateObjectName(counter)).contentType("text/plain"), RequestBody.fromBytes(content)); } diff --git a/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java b/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java new file mode 100644 index 000000000000..25c92a976e9a --- /dev/null +++ b/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.freon; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.core.exception.SdkClientException; + +/** Tests for {@link FreonEnvironmentCredentialsProvider}: the variables and precedence of AWS SDK v1. */ +class TestFreonEnvironmentCredentialsProvider { + + private final Map environment = new HashMap<>(); + private final FreonEnvironmentCredentialsProvider provider = + new FreonEnvironmentCredentialsProvider(environment::get); + + @Test + void readsStandardVariables() { + environment.put("AWS_ACCESS_KEY_ID", "access"); + environment.put("AWS_SECRET_ACCESS_KEY", "secret"); + + final AwsCredentials credentials = provider.resolveCredentials(); + assertEquals("access", credentials.accessKeyId()); + assertEquals("secret", credentials.secretAccessKey()); + assertThat(credentials).isNotInstanceOf(AwsSessionCredentials.class); + } + + @Test + void readsAlternateVariables() { + environment.put("AWS_ACCESS_KEY", "access"); + environment.put("AWS_SECRET_KEY", "secret"); + + final AwsCredentials credentials = provider.resolveCredentials(); + assertEquals("access", credentials.accessKeyId()); + assertEquals("secret", credentials.secretAccessKey()); + } + + /** As in SDK v1: AWS_ACCESS_KEY_ID before AWS_ACCESS_KEY, but AWS_SECRET_KEY before AWS_SECRET_ACCESS_KEY. */ + @Test + void precedenceOfSdkV1() { + environment.put("AWS_ACCESS_KEY_ID", "access-id"); + environment.put("AWS_ACCESS_KEY", "access"); + environment.put("AWS_SECRET_ACCESS_KEY", "secret-access"); + environment.put("AWS_SECRET_KEY", "secret"); + + final AwsCredentials credentials = provider.resolveCredentials(); + assertEquals("access-id", credentials.accessKeyId()); + assertEquals("secret", credentials.secretAccessKey()); + } + + @Test + void trimsValuesAndReadsSessionToken() { + environment.put("AWS_ACCESS_KEY_ID", " access "); + environment.put("AWS_SECRET_ACCESS_KEY", " secret\n"); + environment.put("AWS_SESSION_TOKEN", " token "); + + final AwsCredentials credentials = provider.resolveCredentials(); + assertEquals("access", credentials.accessKeyId()); + assertEquals("secret", credentials.secretAccessKey()); + assertThat(credentials).isInstanceOf(AwsSessionCredentials.class); + assertEquals("token", ((AwsSessionCredentials) credentials).sessionToken()); + } + + @Test + void failsWithoutSecretKey() { + environment.put("AWS_ACCESS_KEY_ID", "access"); + + assertThrows(SdkClientException.class, provider::resolveCredentials); + } + + /** As in SDK v1, a variable that is set but empty does not fall back to the alternate variable. */ + @Test + void emptyVariableDoesNotFallBack() { + environment.put("AWS_ACCESS_KEY_ID", " "); + environment.put("AWS_ACCESS_KEY", "access"); + environment.put("AWS_SECRET_ACCESS_KEY", "secret"); + + assertThrows(SdkClientException.class, provider::resolveCredentials); + } +} diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index a3938fe91c34..32296aa3955a 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -3159,6 +3159,31 @@ public void tearDown() { sdkHttpClient.close(); } + @ParameterizedTest + @ValueSource(strings = {"annotation", "torrent"}) + public void testPresignedUrlGetObjectSubresourceNotImplemented(String subresource) throws Exception { + final String keyName = getKeyName(); + s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName), RequestBody.fromString(CONTENT)); + + // a presigned GetObject with the subresource query parameter, as clients without the typed API send it + final GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(BUCKET_NAME).key(keyName) + .overrideConfiguration(o -> o.putRawQueryParameter(subresource, "")) + .build(); + final URL presignedUrl = presigner.presignGetObject(b -> b.signatureDuration(duration) + .getObjectRequest(objectRequest)).url(); + assertThat(presignedUrl.getQuery()).contains(subresource); + + final HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); + try { + assertEquals(HttpURLConnection.HTTP_NOT_IMPLEMENTED, connection.getResponseCode()); + } finally { + connection.disconnect(); + } + + // the object must be untouched + assertEquals(CONTENT.length(), s3Client.headObject(b -> b.bucket(BUCKET_NAME).key(keyName)).contentLength()); + } + @Test public void testPresignedUrlGetWithCustomQueryParameter() throws Exception { final String keyName = getKeyName(); @@ -4424,6 +4449,9 @@ public void testGetParticularPart(@TempDir Path tempDir) throws Exception { final long allPartsLength = s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName)) .response().contentLength(); + // part number 0 is not a part: the whole object is returned + assertEquals(allPartsLength, s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(keyName).partNumber(0)) + .response().contentLength()); long partsLength = 0; for (int part = 1; part <= 3; part++) { final int partNumber = part; From 4192052f37ce178faaebfe7d8798b2c42cd2804b Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Thu, 5 Nov 2026 04:26:40 -0500 Subject: [PATCH 3/3] Removed aws sdk 2 usage from freon and made aws sdk 2 test only dependency Co-authored-by: Claude Opus 5.5 --- .../dist/src/main/license/bin/LICENSE.txt | 30 ---- .../dist/src/main/license/bin/NOTICE.txt | 30 ---- .../dist/src/main/license/jar-report.txt | 40 +----- .../src/main/smoketest/fcq/s3_om_fcq.robot | 27 ++-- .../dist/src/main/smoketest/s3/freon.robot | 61 -------- hadoop-ozone/freon/pom.xml | 43 ------ .../FreonEnvironmentCredentialsProvider.java | 67 --------- .../freon/FreonS3TraceContextInterceptor.java | 45 ------ .../hadoop/ozone/freon/S3BucketGenerator.java | 75 ---------- .../hadoop/ozone/freon/S3EntityGenerator.java | 70 --------- .../hadoop/ozone/freon/S3KeyGenerator.java | 133 ------------------ ...stFreonEnvironmentCredentialsProvider.java | 101 ------------- .../TestFreonS3TraceContextInterceptor.java | 73 ---------- 13 files changed, 14 insertions(+), 781 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/freon.robot delete mode 100644 hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java delete mode 100644 hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextInterceptor.java delete mode 100644 hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java delete mode 100644 hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java delete mode 100644 hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java delete mode 100644 hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java delete mode 100644 hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index b8c36618f2a1..3f27f2610673 100644 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@ -483,36 +483,6 @@ Apache License 2.0 org.xerial.snappy:snappy-java org.xerial:sqlite-jdbc org.yaml:snakeyaml - software.amazon.awssdk:annotations - software.amazon.awssdk:apache-client - software.amazon.awssdk:arns - software.amazon.awssdk:auth - software.amazon.awssdk:aws-core - software.amazon.awssdk:aws-query-protocol - software.amazon.awssdk:aws-xml-protocol - software.amazon.awssdk:checksums - software.amazon.awssdk:checksums-spi - software.amazon.awssdk:crt-core - software.amazon.awssdk:endpoints-spi - software.amazon.awssdk:http-auth - software.amazon.awssdk:http-auth-aws - software.amazon.awssdk:http-auth-aws-eventstream - software.amazon.awssdk:http-auth-spi - software.amazon.awssdk:http-client-spi - software.amazon.awssdk:identity-spi - software.amazon.awssdk:json-utils - software.amazon.awssdk:metrics-spi - software.amazon.awssdk:profiles - software.amazon.awssdk:protocol-core - software.amazon.awssdk:regions - software.amazon.awssdk:retries - software.amazon.awssdk:retries-spi - software.amazon.awssdk:s3 - software.amazon.awssdk:sdk-core - software.amazon.awssdk:third-party-jackson-core - software.amazon.awssdk:utils - software.amazon.awssdk:utils-lite - software.amazon.eventstream:eventstream MIT ===================== diff --git a/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt b/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt index c5c1fda48840..dd78c02888cf 100644 --- a/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/NOTICE.txt @@ -448,36 +448,6 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -software.amazon.awssdk -==================== - -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). - -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary - -The licenses for these third party components are included in LICENSE.txt - -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). - - ==================== ratis-thirdparty-misc is a shaded dependency which includes additional 3rd party dependencies in shaded form. For the detailed list of the dependencies and the associated NOTICE file see licenses/NOTICE-ratis-thirdparty-misc.txt. diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 28874de0aa07..661b9c065cfc 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -1,34 +1,23 @@ share/ozone/lib/aircompressor.jar -share/ozone/lib/aircompressor.jar share/ozone/lib/animal-sniffer-annotations.jar share/ozone/lib/annotations.jar share/ozone/lib/annotations.jar -share/ozone/lib/annotations.jar -share/ozone/lib/annotations.jar share/ozone/lib/aopalliance-repackaged.jar share/ozone/lib/aopalliance.jar -share/ozone/lib/apache-client.jar share/ozone/lib/apache-log4j-extras.jar -share/ozone/lib/arns.jar share/ozone/lib/asm-analysis.jar share/ozone/lib/asm-commons.jar share/ozone/lib/asm-tree.jar share/ozone/lib/asm-util.jar share/ozone/lib/asm.jar share/ozone/lib/aspectjrt.jar -share/ozone/lib/auth.jar share/ozone/lib/avro.jar -share/ozone/lib/aws-core.jar -share/ozone/lib/aws-query-protocol.jar -share/ozone/lib/aws-xml-protocol.jar share/ozone/lib/bcpkix-jdk18on.jar share/ozone/lib/bcprov-jdk18on.jar share/ozone/lib/bcutil-jdk18on.jar share/ozone/lib/bonecp.RELEASE.jar share/ozone/lib/caffeine.jar share/ozone/lib/cdi-api.SP1.jar -share/ozone/lib/checksums-spi.jar -share/ozone/lib/checksums.jar share/ozone/lib/commons-beanutils.jar share/ozone/lib/commons-cli.jar share/ozone/lib/commons-codec.jar @@ -46,15 +35,12 @@ share/ozone/lib/commons-pool2.jar share/ozone/lib/commons-text.jar share/ozone/lib/commons-validator.jar share/ozone/lib/converter-jackson.jar -share/ozone/lib/crt-core.jar share/ozone/lib/curator-client.jar share/ozone/lib/curator-framework.jar share/ozone/lib/derby.jar share/ozone/lib/disruptor.jar share/ozone/lib/dnsjava.jar -share/ozone/lib/endpoints-spi.jar share/ozone/lib/error_prone_annotations.jar -share/ozone/lib/eventstream.jar share/ozone/lib/failureaccess.jar share/ozone/lib/grpc-api.jar share/ozone/lib/grpc-context.jar @@ -95,11 +81,6 @@ share/ozone/lib/hk2-api.jar share/ozone/lib/hk2-locator.jar share/ozone/lib/hk2-utils.jar share/ozone/lib/hppc.jar -share/ozone/lib/http-auth-aws-eventstream.jar -share/ozone/lib/http-auth-aws.jar -share/ozone/lib/http-auth-spi.jar -share/ozone/lib/http-auth.jar -share/ozone/lib/http-client-spi.jar share/ozone/lib/httpclient.jar share/ozone/lib/httpcore.jar share/ozone/lib/iceberg-api.jar @@ -108,7 +89,6 @@ share/ozone/lib/iceberg-common.jar share/ozone/lib/iceberg-core.jar share/ozone/lib/iceberg-orc.jar share/ozone/lib/iceberg-parquet.jar -share/ozone/lib/identity-spi.jar share/ozone/lib/istack-commons-runtime.jar share/ozone/lib/j2objc-annotations.jar share/ozone/lib/jackson-annotations.jar @@ -121,7 +101,6 @@ share/ozone/lib/jackson-jaxrs-json-provider.jar share/ozone/lib/jackson-module-jaxb-annotations.jar share/ozone/lib/jakarta.activation-api.jar share/ozone/lib/jakarta.activation.jar -share/ozone/lib/jakarta.activation.jar share/ozone/lib/jakarta.annotation-api.jar share/ozone/lib/jakarta.inject-api.jar share/ozone/lib/jakarta.inject.jar @@ -188,13 +167,12 @@ share/ozone/lib/jooq-codegen.jar share/ozone/lib/jooq-meta.jar share/ozone/lib/jooq.jar share/ozone/lib/jsch.jar -share/ozone/lib/json-utils.jar share/ozone/lib/jsp-api.jar share/ozone/lib/jspecify.jar share/ozone/lib/jsr311-api.jar -share/ozone/lib/jtokkit.jar share/ozone/lib/jts-core.jar share/ozone/lib/jul-to-slf4j.jar +share/ozone/lib/jtokkit.jar share/ozone/lib/kerb-core.jar share/ozone/lib/kerb-crypto.jar share/ozone/lib/kerb-util.jar @@ -213,7 +191,6 @@ share/ozone/lib/listenablefuture-empty-to-avoid-conflict-with-guava.jar share/ozone/lib/log4j-api.jar share/ozone/lib/log4j-core.jar share/ozone/lib/metrics-core.jar -share/ozone/lib/metrics-spi.jar share/ozone/lib/netty-buffer.Final.jar share/ozone/lib/netty-codec-http.Final.jar share/ozone/lib/netty-codec-http2.Final.jar @@ -239,7 +216,6 @@ share/ozone/lib/okhttp-jvm.jar share/ozone/lib/okhttp-sse.jar share/ozone/lib/okhttp.jar share/ozone/lib/okio-jvm.jar -share/ozone/lib/okio-jvm.jar share/ozone/lib/okio.jar share/ozone/lib/openai4j.jar share/ozone/lib/opentelemetry-api.jar @@ -295,14 +271,12 @@ share/ozone/lib/parquet-variant.jar share/ozone/lib/perfmark-api.jar share/ozone/lib/picocli-shell-jline3.jar share/ozone/lib/picocli.jar -share/ozone/lib/profiles.jar share/ozone/lib/proto-google-common-protos.jar share/ozone/lib/protobuf-java.jar -share/ozone/lib/protocol-core.jar -share/ozone/lib/r2dbc-spi.RELEASE.jar share/ozone/lib/ranger-audit-core.jar share/ozone/lib/ranger-authz-api.jar share/ozone/lib/ranger-common-utils.jar +share/ozone/lib/r2dbc-spi.RELEASE.jar share/ozone/lib/ranger-intg.jar share/ozone/lib/ranger-plugin-classloader.jar share/ozone/lib/ranger-plugins-common.jar @@ -321,17 +295,11 @@ share/ozone/lib/ratis-thirdparty-misc.jar share/ozone/lib/ratis-tools.jar share/ozone/lib/re2j.jar share/ozone/lib/reactive-streams.jar -share/ozone/lib/reactive-streams.jar share/ozone/lib/reflections.jar -share/ozone/lib/regions.jar share/ozone/lib/reload4j.jar -share/ozone/lib/retries-spi.jar -share/ozone/lib/retries.jar share/ozone/lib/retrofit.jar share/ozone/lib/rocksdb-checkpoint-differ.jar share/ozone/lib/rocksdbjni.jar -share/ozone/lib/s3.jar -share/ozone/lib/sdk-core.jar share/ozone/lib/simpleclient.jar share/ozone/lib/simpleclient_common.jar share/ozone/lib/simpleclient_dropwizard.jar @@ -345,12 +313,8 @@ share/ozone/lib/spring-jdbc.jar share/ozone/lib/spring-tx.jar share/ozone/lib/sqlite-jdbc.jar share/ozone/lib/stax2-api.jar -share/ozone/lib/stax2-api.jar -share/ozone/lib/third-party-jackson-core.jar share/ozone/lib/txw2.jar share/ozone/lib/ugsync-util.jar -share/ozone/lib/utils-lite.jar -share/ozone/lib/utils.jar share/ozone/lib/vault-java-driver.jar share/ozone/lib/weld-servlet-shaded.Final.jar share/ozone/lib/woodstox-core.jar diff --git a/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot b/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot index 256a9c3e8051..530aa2f55a4a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot +++ b/hadoop-ozone/dist/src/main/smoketest/fcq/s3_om_fcq.robot @@ -37,12 +37,6 @@ Setup headers Kinit test user testuser testuser.keytab Setup secure v4 headers -Setup aws credentials - ${accessKey} = Execute aws configure get aws_access_key_id - ${secret} = Execute aws configure get aws_secret_access_key - Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret} - Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey} - Verify endpoint is up [arguments] ${url} Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit HTTP user @@ -52,10 +46,16 @@ Verify endpoint is up Setup bucket1 Execute AWSS3APICli create-bucket --bucket bucket1 -Freon s3kg - [arguments] ${prefix}=s3bg ${n}=1000 ${threads}=10 ${args}=${EMPTY} - ${result} = Execute ozone freon s3kg -e ${S3_ENDPOINT_URL} -t ${threads} -n ${n} -p ${prefix} ${args} - Should contain ${result} Successful executions: ${n} +Generate S3 load + [arguments] ${n}=100 + # one aws process uploads all the files, several at a time + ${dir} = Execute mktemp -d + Execute for i in $(seq 1 ${n}); do echo "$i" > ${dir}/key-$i; done + ${result} = Execute AWSS3Cli cp --recursive ${dir} s3://bucket1/fcq/ + Should contain ${result} upload: + # count the objects: the CLI merges the pages of the listing, which drops KeyCount + ${count} = Execute AWSS3APICli list-objects-v2 --bucket bucket1 --prefix fcq/ --query 'length(Contents)' + Should Be Equal As Integers ${count} ${n} Check metrics caller ${result} = Execute curl --negotiate -u : -LSs ${OM_JMX_ENDPOINT} | sed -n '/${DECAY_RPC_SCHEDULER_METRICS_NAME}/,/}/p' | grep 'Caller(' @@ -65,9 +65,6 @@ Check metrics caller Test setup headers Setup headers -Test setup credentials - Setup aws credentials - Test create bucket 1 Setup bucket1 @@ -78,8 +75,8 @@ Test metrics are registered ${result} = Execute curl --negotiate -u : -LSs ${OM_JMX_ENDPOINT} | grep ${DECAY_RPC_SCHEDULER_METRICS_NAME} Should contain ${result} ${DECAY_RPC_SCHEDULER_METRICS_NAME} -Run freon s3kg - Freon s3kg +Run S3 load + Generate S3 load Test caller from metrics Check metrics caller diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot deleted file mode 100644 index f722f396c9ce..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation S3 gateway test with aws cli -Library OperatingSystem -Library String -Library BuiltIn -Resource ./commonawslib.robot -Suite Setup Setup s3 tests -Default Tags no-bucket-type - -*** Variables *** -${ENDPOINT_URL} http://s3g:9878 -${BUCKET} generated - -*** Keywords *** -# Export access key and secret to the environment -Setup aws credentials - ${accessKey} = Execute aws configure get aws_access_key_id - ${secret} = Execute aws configure get aws_secret_access_key - Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret} - Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey} - -Freon S3BG - [arguments] ${prefix}=s3bg ${n}=100 ${threads}=10 ${args}=${EMPTY} - ${result} = Execute ozone freon s3bg -e ${ENDPOINT_URL} -t ${threads} -n ${n} -p ${prefix} ${args} - Should contain ${result} Successful executions: ${n} - -Freon S3KG - [arguments] ${prefix}=s3kg ${n}=100 ${threads}=10 ${args}=${EMPTY} - ${result} = Execute ozone freon s3kg -e ${ENDPOINT_URL} -t ${threads} -n ${n} -p ${prefix} --bucket ${BUCKET} ${args} - Should contain ${result} Successful executions: ${n} - -*** Test Cases *** -Run Freon S3BG - [Setup] Setup aws credentials - Freon S3BG s3bg-${BUCKET} - -Run Freon S3KG - [Setup] Setup aws credentials - Freon S3KG s3kg-${BUCKET} - # the content type that freon set for its keys with AWS SDK v1 - ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key s3kg-${BUCKET}/0 - Should contain ${result} "ContentType": "text/plain" - -Run Freon S3KG MPU - [Setup] Setup aws credentials - Freon S3KG s3kg-mpu-${BUCKET} 10 1 --multi-part-upload --parts=2 --size=5242880 diff --git a/hadoop-ozone/freon/pom.xml b/hadoop-ozone/freon/pom.xml index 7b329a3bf827..7289cf2a202b 100644 --- a/hadoop-ozone/freon/pom.xml +++ b/hadoop-ozone/freon/pom.xml @@ -131,49 +131,6 @@ org.slf4j slf4j-api - - software.amazon.awssdk - apache-client - - - software.amazon.awssdk - auth - - - software.amazon.awssdk - aws-core - - - software.amazon.awssdk - http-client-spi - - - software.amazon.awssdk - regions - - - software.amazon.awssdk - s3 - - - - software.amazon.awssdk - apache5-client - - - software.amazon.awssdk - netty-nio-client - - - - - software.amazon.awssdk - sdk-core - - - software.amazon.awssdk - utils - org.apache.ozone hdds-annotation-processing diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java deleted file mode 100644 index cc9ad72fd10b..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonEnvironmentCredentialsProvider.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import com.google.common.annotations.VisibleForTesting; -import java.util.function.UnaryOperator; -import org.apache.commons.lang3.StringUtils; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.core.exception.SdkClientException; - -/** - * Credentials from environment variables, read the way AWS SDK for Java v1 read them, which freon used before: - * the access key from {@code AWS_ACCESS_KEY_ID}, or else {@code AWS_ACCESS_KEY}, the secret key from - * {@code AWS_SECRET_KEY}, or else {@code AWS_SECRET_ACCESS_KEY}, and the session token from {@code AWS_SESSION_TOKEN}, - * if set. The environment provider of SDK v2 only reads {@code AWS_ACCESS_KEY_ID} and {@code AWS_SECRET_ACCESS_KEY}. - */ -public final class FreonEnvironmentCredentialsProvider implements AwsCredentialsProvider { - - private final UnaryOperator environment; - - public FreonEnvironmentCredentialsProvider() { - this(System::getenv); - } - - @VisibleForTesting - FreonEnvironmentCredentialsProvider(UnaryOperator environment) { - this.environment = environment; - } - - @Override - public AwsCredentials resolveCredentials() { - final String accessKey = StringUtils.trim(get("AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY")); - final String secretKey = StringUtils.trim(get("AWS_SECRET_KEY", "AWS_SECRET_ACCESS_KEY")); - if (StringUtils.isEmpty(accessKey) || StringUtils.isEmpty(secretKey)) { - throw SdkClientException.create("Unable to load AWS credentials from environment variables " - + "(AWS_ACCESS_KEY_ID (or AWS_ACCESS_KEY) and AWS_SECRET_KEY (or AWS_SECRET_ACCESS_KEY))"); - } - final String sessionToken = StringUtils.trim(environment.apply("AWS_SESSION_TOKEN")); - return StringUtils.isEmpty(sessionToken) - ? AwsBasicCredentials.create(accessKey, secretKey) - : AwsSessionCredentials.create(accessKey, secretKey, sessionToken); - } - - /** @return the value of the variable, or of the alternate one if the variable is not set at all */ - private String get(String variable, String alternate) { - final String value = environment.apply(variable); - return value != null ? value : environment.apply(alternate); - } -} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextInterceptor.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextInterceptor.java deleted file mode 100644 index 58da089cb0f1..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/FreonS3TraceContextInterceptor.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import software.amazon.awssdk.core.interceptor.Context; -import software.amazon.awssdk.core.interceptor.ExecutionAttributes; -import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; -import software.amazon.awssdk.http.SdkHttpRequest; - -/** - * Adds W3C trace context headers to each outgoing S3 request so the S3 Gateway - * can attach its spans to the Freon task span, using {@link W3CTraceContextPropagator}. - */ -public final class FreonS3TraceContextInterceptor implements ExecutionInterceptor { - - @Override - public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { - if (!Span.current().getSpanContext().isValid()) { - return context.httpRequest(); - } - final SdkHttpRequest.Builder request = context.httpRequest().toBuilder(); - W3CTraceContextPropagator.getInstance().inject( - io.opentelemetry.context.Context.current(), - request, - (carrier, key, value) -> carrier.putHeader(key, value)); - return request.build(); - } -} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java deleted file mode 100644 index e6ba1c5ad3d9..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import com.codahale.metrics.Timer; -import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.kohsuke.MetaInfServices; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine.Command; - -/** - * Generate buckets via the s3 interface. - * - * For a secure cluster, - * $> init user keytab - * $> kinit -kt /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM - * $> eval $(ozone s3 getsecret -e) - * for getting and exporting access_key_id and secret_access_key - * to freon shell test environment - * secret access key. - */ -@Command(name = "s3bg", - aliases = "s3-bucket-generator", - description = "Create buckets via the s3 interface.%n" + - "Command requires AWS authentication environment " + - "variables set, AWS_ACCESS_KEY_ID and " + - "AWS_SECRET_ACCESS_KEY", - versionProvider = HddsVersionProvider.class, - mixinStandardHelpOptions = true, - showDefaultValues = true) -@MetaInfServices(FreonSubcommand.class) -public class S3BucketGenerator extends S3EntityGenerator - implements Callable { - - private static final Logger LOG = - LoggerFactory.getLogger(S3BucketGenerator.class); - - private Timer timer; - - @Override - public Void call() throws Exception { - s3ClientInit(); - - timer = getMetrics().timer("bucket-create"); - - runTests(this::createBucket); - - return null; - } - - private void createBucket(long bucketNum) throws Exception { - String bName = getPrefix() + bucketNum; - timer.time(() -> { - getS3().createBucket(r -> r.bucket(bName)); - return null; - }); - } -} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java deleted file mode 100644 index fa988c30ea1c..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import java.net.URI; -import picocli.CommandLine; -import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; -import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; -import software.amazon.awssdk.http.apache.ApacheHttpClient; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; - -/** - * Initiliazing common aspects of S3BucketGenerator and S3KeyGenerator. - */ -public class S3EntityGenerator extends BaseFreonGenerator { - @CommandLine.Option(names = {"-e", "--endpoint"}, - description = "S3 HTTP endpoint", - defaultValue = "http://localhost:9878") - private String endpoint; - - private S3Client s3; - - protected void s3ClientInit() { - init(); - - S3ClientBuilder s3ClientBuilder = S3Client.builder() - .httpClientBuilder(ApacheHttpClient.builder()) - .credentialsProvider(new FreonEnvironmentCredentialsProvider()) - // checksums only where S3 requires them: a load generator should not spend its time computing them - .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED) - .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED) - .overrideConfiguration(c -> c.addExecutionInterceptor(new FreonS3TraceContextInterceptor())); - - if (!endpoint.isEmpty()) { - s3ClientBuilder - .forcePathStyle(true) - .endpointOverride(URI.create(endpoint)) - .region(Region.US_EAST_1); - } else { - s3ClientBuilder.region(Region.US_WEST_2); - } - - s3 = s3ClientBuilder.build(); - } - - public String getEndpoint() { - return endpoint; - } - - public S3Client getS3() { - return s3; - } -} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java deleted file mode 100644 index 835777ee6e31..000000000000 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_MULTIPART_MIN_SIZE; - -import com.codahale.metrics.Timer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.kohsuke.MetaInfServices; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.model.CompletedPart; -import software.amazon.awssdk.services.s3.model.UploadPartResponse; - -/** - * Generate random keys via the s3 interface. - */ -@Command(name = "s3kg", - aliases = "s3-key-generator", - description = "Create random keys via the s3 interface." + - "Command requires AWS authentication environment " + - "variables set, AWS_ACCESS_KEY_ID and " + - "AWS_SECRET_ACCESS_KEY", - versionProvider = HddsVersionProvider.class, - mixinStandardHelpOptions = true, - showDefaultValues = true) -@MetaInfServices(FreonSubcommand.class) -@SuppressWarnings("java:S2245") // no need for secure random -public class S3KeyGenerator extends S3EntityGenerator - implements Callable { - - private static final Logger LOG = - LoggerFactory.getLogger(S3KeyGenerator.class); - - @Option(names = {"-b", "--bucket"}, - description = - "Name of the (S3!) bucket which contains the test data.", - defaultValue = "bucket1") - private String bucketName; - - @Option(names = {"-s", "--size"}, - description = "Size of the generated key (in bytes) or size of one " - + "multipart upload part (in case of multipart upload)", - defaultValue = "10240") - private int fileSize; - - @Option(names = {"--multi-part-upload"}, - description = "Use multi-part upload", - defaultValue = "false") - private boolean multiPart; - - @Option(names = {"--parts"}, - description = "Number of parts for multipart upload (final size = " - + "--size * --parts)", - defaultValue = "10") - private int numberOfParts; - - private Timer timer; - - private byte[] content; - - @Override - public Void call() throws Exception { - - if (multiPart && fileSize < OM_MULTIPART_MIN_SIZE) { - throw new IllegalArgumentException( - "Size of multipart upload parts should be at least 5MB (5242880)"); - } - - s3ClientInit(); - - content = RandomStringUtils.secure().nextAscii(fileSize).getBytes(StandardCharsets.UTF_8); - - timer = getMetrics().timer("key-create"); - - runTests(this::createKey); - - return null; - } - - private void createKey(long counter) throws Exception { - timer.time(() -> { - if (multiPart) { - - final String keyName = generateObjectName(counter); - final String uploadId = getS3().createMultipartUpload(r -> r.bucket(bucketName).key(keyName)).uploadId(); - - List parts = new ArrayList<>(); - for (int i = 1; i <= numberOfParts; i++) { - final int partNumber = i; - final UploadPartResponse uploadPartResponse = getS3().uploadPart( - r -> r.bucket(bucketName).key(keyName).uploadId(uploadId).partNumber(partNumber) - .contentLength((long) content.length), - RequestBody.fromBytes(content)); - parts.add(CompletedPart.builder().partNumber(partNumber).eTag(uploadPartResponse.eTag()).build()); - } - - getS3().completeMultipartUpload(r -> r.bucket(bucketName).key(keyName).uploadId(uploadId) - .multipartUpload(m -> m.parts(parts))); - - } else { - // the content type that AWS SDK v1 set for the String content freon uploaded before - getS3().putObject(r -> r.bucket(bucketName).key(generateObjectName(counter)).contentType("text/plain"), - RequestBody.fromBytes(content)); - } - - return null; - }); - } -} diff --git a/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java b/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java deleted file mode 100644 index 25c92a976e9a..000000000000 --- a/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonEnvironmentCredentialsProvider.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.HashMap; -import java.util.Map; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.core.exception.SdkClientException; - -/** Tests for {@link FreonEnvironmentCredentialsProvider}: the variables and precedence of AWS SDK v1. */ -class TestFreonEnvironmentCredentialsProvider { - - private final Map environment = new HashMap<>(); - private final FreonEnvironmentCredentialsProvider provider = - new FreonEnvironmentCredentialsProvider(environment::get); - - @Test - void readsStandardVariables() { - environment.put("AWS_ACCESS_KEY_ID", "access"); - environment.put("AWS_SECRET_ACCESS_KEY", "secret"); - - final AwsCredentials credentials = provider.resolveCredentials(); - assertEquals("access", credentials.accessKeyId()); - assertEquals("secret", credentials.secretAccessKey()); - assertThat(credentials).isNotInstanceOf(AwsSessionCredentials.class); - } - - @Test - void readsAlternateVariables() { - environment.put("AWS_ACCESS_KEY", "access"); - environment.put("AWS_SECRET_KEY", "secret"); - - final AwsCredentials credentials = provider.resolveCredentials(); - assertEquals("access", credentials.accessKeyId()); - assertEquals("secret", credentials.secretAccessKey()); - } - - /** As in SDK v1: AWS_ACCESS_KEY_ID before AWS_ACCESS_KEY, but AWS_SECRET_KEY before AWS_SECRET_ACCESS_KEY. */ - @Test - void precedenceOfSdkV1() { - environment.put("AWS_ACCESS_KEY_ID", "access-id"); - environment.put("AWS_ACCESS_KEY", "access"); - environment.put("AWS_SECRET_ACCESS_KEY", "secret-access"); - environment.put("AWS_SECRET_KEY", "secret"); - - final AwsCredentials credentials = provider.resolveCredentials(); - assertEquals("access-id", credentials.accessKeyId()); - assertEquals("secret", credentials.secretAccessKey()); - } - - @Test - void trimsValuesAndReadsSessionToken() { - environment.put("AWS_ACCESS_KEY_ID", " access "); - environment.put("AWS_SECRET_ACCESS_KEY", " secret\n"); - environment.put("AWS_SESSION_TOKEN", " token "); - - final AwsCredentials credentials = provider.resolveCredentials(); - assertEquals("access", credentials.accessKeyId()); - assertEquals("secret", credentials.secretAccessKey()); - assertThat(credentials).isInstanceOf(AwsSessionCredentials.class); - assertEquals("token", ((AwsSessionCredentials) credentials).sessionToken()); - } - - @Test - void failsWithoutSecretKey() { - environment.put("AWS_ACCESS_KEY_ID", "access"); - - assertThrows(SdkClientException.class, provider::resolveCredentials); - } - - /** As in SDK v1, a variable that is set but empty does not fall back to the alternate variable. */ - @Test - void emptyVariableDoesNotFallBack() { - environment.put("AWS_ACCESS_KEY_ID", " "); - environment.put("AWS_ACCESS_KEY", "access"); - environment.put("AWS_SECRET_ACCESS_KEY", "secret"); - - assertThrows(SdkClientException.class, provider::resolveCredentials); - } -} diff --git a/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java b/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java deleted file mode 100644 index 8cb6b8199d1f..000000000000 --- a/hadoop-ozone/freon/src/test/java/org/apache/hadoop/ozone/freon/TestFreonS3TraceContextInterceptor.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.freon; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.TraceFlags; -import io.opentelemetry.api.trace.TraceState; -import io.opentelemetry.context.Scope; -import java.net.URI; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.core.interceptor.Context; -import software.amazon.awssdk.core.interceptor.ExecutionAttributes; -import software.amazon.awssdk.http.SdkHttpMethod; -import software.amazon.awssdk.http.SdkHttpRequest; - -/** Tests for {@link FreonS3TraceContextInterceptor}. */ -class TestFreonS3TraceContextInterceptor { - private static final String TRACE_ID = "0af7651916cd43dd8448eb211c80319c"; - private static final String SPAN_ID = "b7ad6b7169203331"; - - private final FreonS3TraceContextInterceptor interceptor = new FreonS3TraceContextInterceptor(); - - private static Context.ModifyHttpRequest context(SdkHttpRequest request) { - final Context.ModifyHttpRequest context = mock(Context.ModifyHttpRequest.class); - when(context.httpRequest()).thenReturn(request); - return context; - } - - private static SdkHttpRequest request() { - return SdkHttpRequest.builder().method(SdkHttpMethod.PUT).uri(URI.create("http://localhost:9878/bucket/key")) - .build(); - } - - @Test - void addsTraceContextOfCurrentSpan() { - final Span span = Span.wrap( - SpanContext.create(TRACE_ID, SPAN_ID, TraceFlags.getSampled(), TraceState.getDefault())); - final SdkHttpRequest modified; - try (Scope ignored = span.makeCurrent()) { - modified = interceptor.modifyHttpRequest(context(request()), new ExecutionAttributes()); - } - - assertThat(modified.firstMatchingHeader("traceparent")).hasValue("00-" + TRACE_ID + "-" + SPAN_ID + "-01"); - } - - @Test - void leavesRequestUnchangedWithoutSpan() { - final SdkHttpRequest request = request(); - - assertSame(request, interceptor.modifyHttpRequest(context(request), new ExecutionAttributes())); - } -}