diff --git a/pom.xml b/pom.xml index 545984a9..ea3f98f5 100644 --- a/pom.xml +++ b/pom.xml @@ -44,11 +44,12 @@ 8 UTF-8 - 2.14.2 - 1.12.477 - 5.5.3 - 1.5.5-5 - 5.10.2 + 2.20.0 + 2.34.6 + 5.9.1 + 1.5.5-7 + 5.14.0 + 3.0.2 @@ -83,14 +84,19 @@ jackson-datatype-jdk8 ${jackson.version} + + com.google.code.findbugs + jsr305 + ${findbugs.version} + edu.ucar cdm-core ${netcdfJavaVersion} - com.amazonaws - aws-java-sdk-s3 + software.amazon.awssdk + s3 ${aws.version} diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 8afe254d..9368975f 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -1,10 +1,17 @@ package dev.zarr.zarrjava.store; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3ObjectInputStream; import dev.zarr.zarrjava.utils.Utils; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -16,13 +23,13 @@ public class S3Store implements Store, Store.ListableStore { @Nonnull - private final AmazonS3 s3client; + private final S3Client s3client; @Nonnull private final String bucketName; @Nullable private final String prefix; - public S3Store(@Nonnull AmazonS3 s3client, @Nonnull String bucketName, @Nullable String prefix) { + public S3Store(@Nonnull S3Client s3client, @Nonnull String bucketName, @Nullable String prefix) { this.s3client = s3client; this.bucketName = bucketName; this.prefix = prefix; @@ -40,8 +47,7 @@ String resolveKeys(String[] keys) { @Nullable ByteBuffer get(GetObjectRequest getObjectRequest) { - try (S3ObjectInputStream inputStream = s3client.getObject(getObjectRequest) - .getObjectContent()) { + try (ResponseInputStream inputStream = s3client.getObject(getObjectRequest)) { return Utils.asByteBuffer(inputStream); } catch (IOException e) { return null; @@ -50,31 +56,44 @@ ByteBuffer get(GetObjectRequest getObjectRequest) { @Override public boolean exists(String[] keys) { - return s3client.doesObjectExist(bucketName, resolveKeys(keys)); + HeadObjectRequest req = HeadObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(); + return s3client.headObject(req).sdkHttpResponse().statusCode() == 200; } @Nullable @Override public ByteBuffer get(String[] keys) { - return get(new GetObjectRequest(bucketName, resolveKeys(keys))); + return get(GetObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)) + .build()); } @Nullable @Override public ByteBuffer get(String[] keys, long start) { - return get(new GetObjectRequest(bucketName, resolveKeys(keys)).withRange(start)); + GetObjectRequest req = GetObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .range(String.valueOf(start)) + .build(); + return get(req); } @Nullable @Override public ByteBuffer get(String[] keys, long start, long end) { - return get(new GetObjectRequest(bucketName, resolveKeys(keys)).withRange(start, end)); + GetObjectRequest req = GetObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .range(String.valueOf(start)+"-"+String.valueOf(end)) + .build(); + return get(req); } @Override public void set(String[] keys, ByteBuffer bytes) { try (InputStream byteStream = new ByteArrayInputStream(Utils.toArray(bytes))) { - s3client.putObject(bucketName, resolveKeys(keys), byteStream, new ObjectMetadata()); + /*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/ + s3client.putObject(PutObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(), RequestBody.fromContentProvider(() -> byteStream, "application/octet-stream")); } catch (IOException e) { throw new RuntimeException(e); } @@ -82,16 +101,20 @@ public void set(String[] keys, ByteBuffer bytes) { @Override public void delete(String[] keys) { - s3client.deleteObject(bucketName, resolveKeys(keys)); + s3client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)) + .build()); } @Override public Stream list(String[] keys) { final String fullKey = resolveKeys(keys); - return s3client.listObjects(bucketName, fullKey) - .getObjectSummaries() + ListObjectsRequest req = ListObjectsRequest.builder() + .bucket(bucketName).prefix(fullKey) + .build(); + ListObjectsResponse res = s3client.listObjects(req); + return res.contents() .stream() - .map(p -> p.getKey().substring(fullKey.length() + 1)); + .map(p -> p.key().substring(fullKey.length() + 1)); } @Nonnull diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 647a3b3e..36274801 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,8 +1,5 @@ package dev.zarr.zarrjava; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.luben.zstd.Zstd; @@ -20,6 +17,10 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; + +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; import ucar.ma2.MAMath; import java.io.*; @@ -280,9 +281,9 @@ public void testFileSystemStores() throws IOException, ZarrException { @Test public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() - .withRegion("eu-west-1") - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + S3Store s3Store = new S3Store(S3Client.builder() + .region(Region.of("eu-west-1")) + .credentialsProvider(AnonymousCredentialsProvider.create()) .build(), "static.webknossos.org", "data"); System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); }