From 4195ac82d4e0771505959873409f3b97bbf433d0 Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Wed, 24 Jun 2026 13:48:43 -0400 Subject: [PATCH 1/4] superficial automated update --- THIRD-PARTY-LICENSES.txt | 8 +- toolbackup/pom.xml | 23 +++-- .../toolbackup/client/cli/S3Communicator.java | 99 ++++++++----------- .../client/cli/S3CommunicatorIT.java | 11 ++- 4 files changed, 65 insertions(+), 76 deletions(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 45dee3c9..1f5c1a72 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -1,5 +1,5 @@ -Lists of 404 third-party dependencies. +Lists of 400 third-party dependencies. (Apache License, Version 2.0) akka-actor (com.typesafe.akka:akka-actor_2.13:2.5.32 - https://akka.io/) (Apache License, Version 2.0) akka-protobuf (com.typesafe.akka:akka-protobuf_2.13:2.5.32 - https://akka.io/) (Apache License, Version 2.0) akka-slf4j (com.typesafe.akka:akka-slf4j_2.13:2.5.32 - https://akka.io/) @@ -66,6 +66,7 @@ Lists of 404 third-party dependencies. (Apache License, Version 2.0) AWS Java SDK :: Regions (software.amazon.awssdk:regions:2.31.56 - https://aws.amazon.com/sdkforjava/core/regions) (Apache License, Version 2.0) AWS Java SDK :: Retries (software.amazon.awssdk:retries:2.31.56 - https://aws.amazon.com/sdkforjava/core/retries) (Apache License, Version 2.0) AWS Java SDK :: Retries API (software.amazon.awssdk:retries-spi:2.31.56 - https://aws.amazon.com/sdkforjava/core/retries-spi) + (Apache License, Version 2.0) AWS Java SDK :: S3 :: Transfer Manager (software.amazon.awssdk:s3-transfer-manager:2.31.56 - https://aws.amazon.com/sdkforjava) (Apache License, Version 2.0) AWS Java SDK :: SDK Core (software.amazon.awssdk:sdk-core:2.31.56 - https://aws.amazon.com/sdkforjava) (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon Athena (software.amazon.awssdk:athena:2.31.56 - https://aws.amazon.com/sdkforjava) (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon CloudWatch (software.amazon.awssdk:cloudwatch:2.31.56 - https://aws.amazon.com/sdkforjava) @@ -77,9 +78,6 @@ Lists of 404 third-party dependencies. (Apache License, Version 2.0) AWS Java SDK :: Third Party :: Jackson-core (software.amazon.awssdk:third-party-jackson-core:2.31.56 - https://aws.amazon.com/sdkforjava) (Apache License, Version 2.0) AWS Java SDK :: Third Party :: Jackson-dataformat-cbor (software.amazon.awssdk:third-party-jackson-dataformat-cbor:2.31.56 - https://aws.amazon.com/sdkforjava) (Apache License, Version 2.0) AWS Java SDK :: Utilities (software.amazon.awssdk:utils:2.31.56 - https://aws.amazon.com/sdkforjava/utils) - (Apache License, Version 2.0) AWS Java SDK for Amazon S3 (com.amazonaws:aws-java-sdk-s3:1.11.83 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK for AWS KMS (com.amazonaws:aws-java-sdk-kms:1.11.83 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS SDK for Java - Core (com.amazonaws:aws-java-sdk-core:1.11.83 - https://aws.amazon.com/sdkforjava) (The Apache Software License, Version 2.0) Bean Validation API (javax.validation:validation-api:1.1.0.Final - http://beanvalidation.org) (MIT) better-files (com.github.pathikrit:better-files_2.13:3.9.1 - https://github.com/pathikrit/better-files) (Bouncy Castle Licence) Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.84 - https://www.bouncycastle.org/download/bouncy-castle-java/) @@ -271,7 +269,6 @@ Lists of 404 third-party dependencies. (Apache License, Version 2.0) JJWT :: Extensions :: Jackson (io.jsonwebtoken:jjwt-jackson:0.11.5 - https://github.com/jwtk/jjwt/jjwt-jackson) (Apache License, Version 2.0) JJWT :: Impl (io.jsonwebtoken:jjwt-impl:0.11.5 - https://github.com/jwtk/jjwt/jjwt-impl) (The BSD License) JLine Bundle (org.jline:jline:3.21.0 - http://nexus.sonatype.org/oss-repository-hosting.html/jline-parent/jline) - (Apache License, Version 2.0) JMES Path Query library (com.amazonaws:jmespath-java:1.11.83 - https://aws.amazon.com/sdkforjava) (Apache License, Version 2.0) Joda-Time (joda-time:joda-time:2.12.5 - https://www.joda.org/joda-time/) (Apache License, Version 2.0) jOOQ (org.jooq:jooq:3.19.7 - http://www.jooq.org/jooq) (The MIT License) JOpt Simple (net.sf.jopt-simple:jopt-simple:5.0.3 - http://pholser.github.io/jopt-simple) @@ -372,7 +369,6 @@ Lists of 404 third-party dependencies. (EPL 2.0) (GPL2 w/ CPE) ServiceLocator Default Implementation (org.glassfish.hk2:hk2-locator:3.0.6 - https://github.com/eclipse-ee4j/glassfish-hk2/hk2-locator) (MIT) SLF4J API Module (org.slf4j:slf4j-api:2.0.18 - http://www.slf4j.org) (Apache License, Version 2.0) SnakeYAML (org.yaml:snakeyaml:2.5 - https://bitbucket.org/snakeyaml/snakeyaml) - (The Apache License, Version 2.0) software.amazon.ion:ion-java (software.amazon.ion:ion-java:1.0.1 - https://github.com/amznlabs/ion-java/) (MIT) sourcecode_2.13 (com.lihaoyi:sourcecode_2.13:0.2.8 - https://github.com/lihaoyi/sourcecode) (Apache 2) spray-json (io.spray:spray-json_2.13:1.3.6 - https://github.com/spray/spray-json) (Apache-2.0) ssl-config-core (com.typesafe:ssl-config-core_2.13:0.3.8 - https://github.com/lightbend/ssl-config) diff --git a/toolbackup/pom.xml b/toolbackup/pom.xml index a6ee09ef..61fd1daa 100644 --- a/toolbackup/pom.xml +++ b/toolbackup/pom.xml @@ -143,25 +143,36 @@ - com.amazonaws - aws-java-sdk-s3 - ${aws.version} + software.amazon.awssdk + s3 - com.amazonaws - aws-java-sdk-core - ${aws.version} + software.amazon.awssdk + aws-core + + + software.amazon.awssdk + auth + + + software.amazon.awssdk + sdk-core + org.slf4j slf4j-api compile + + software.amazon.awssdk + s3-transfer-manager + diff --git a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java index ae2f0b36..74963df4 100644 --- a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java +++ b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java @@ -3,54 +3,53 @@ import static io.dockstore.toolbackup.client.cli.Client.COMMAND_ERROR; import static java.lang.System.out; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.S3ClientOptions; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CreateBucketRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.transfer.ObjectMetadataProvider; -import com.amazonaws.services.s3.transfer.TransferManager; import java.io.File; +import java.net.URI; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.ListBucketsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.config.DownloadFilter; +import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest; +import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest; /** * Created by kcao on 12/01/17. */ class S3Communicator { - private TransferManager transferManager; - private AmazonS3Client s3Client; + private S3TransferManager transferManager; + private S3AsyncClient s3Client; S3Communicator() { - s3Client = new AmazonS3Client(new ProfileCredentialsProvider().getCredentials()); - s3Client.setEndpoint("http://localhost:8080"); - s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).disableChunkedEncoding().build()); + s3Client = S3AsyncClient.builder().endpointOverride(URI.create("http://localhost:8080")).credentialsProvider(ProfileCredentialsProvider.builder().build()) + .forcePathStyle(true).build(); - transferManager = new TransferManager(s3Client); + transferManager = S3TransferManager.builder().s3Client(s3Client).build(); } S3Communicator(String section, String endpoint) { - ClientConfiguration opts = new ClientConfiguration(); - opts.setSignerOverride("S3SignerType"); - s3Client = new AmazonS3Client(new ProfileCredentialsProvider(section).getCredentials(), opts); + s3Client = S3AsyncClient.builder().credentialsProvider(ProfileCredentialsProvider.builder().build()).endpointOverride(URI.create(endpoint)).credentialsProvider(ProfileCredentialsProvider.builder().build()) + .forcePathStyle(true).build(); - s3Client.setEndpoint(endpoint); - s3Client.setS3ClientOptions(S3ClientOptions.builder().build()); - - transferManager = new TransferManager(s3Client); + transferManager = S3TransferManager.builder().s3Client(s3Client).build(); } //-----------------------Report----------------------- - long getCloudTotalInB(String bucketName, String prefix) { + long getCloudTotalInB(String bucketName, String prefix) throws ExecutionException, InterruptedException { long total = 0; - List objectSummaries = s3Client.listObjects(bucketName, prefix).getObjectSummaries(); - List sizes = objectSummaries.stream().map(S3ObjectSummary::getSize).collect(Collectors.toList()); + List objectSummaries = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix) + .build()).get().contents(); + List sizes = objectSummaries.stream().map(S3Object::size).toList(); for (long size : sizes) { total += size; @@ -60,48 +59,34 @@ long getCloudTotalInB(String bucketName, String prefix) { } //-----------------------Upload----------------------- - boolean doesBucketExist(String bucketName) { - return s3Client.doesBucketExist(bucketName); + boolean doesBucketExist(String bucketName) throws ExecutionException, InterruptedException { + return s3Client.listBuckets(ListBucketsRequest.builder().prefix(bucketName).build()).get().hasBuckets(); } - void createBucket(String bucketName) { + void createBucket(String bucketName) throws ExecutionException, InterruptedException { if (!doesBucketExist(bucketName)) { - s3Client.createBucket(new CreateBucketRequest(bucketName)); + s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName) + .build()); } } - Map getKeysToSizes(String bucketName, String prefix) { + Map getKeysToSizes(String bucketName, String prefix) throws ExecutionException, InterruptedException { createBucket(bucketName); - List objectSummaries = s3Client.listObjects(bucketName, prefix).getObjectSummaries(); - Map keysToSizes = objectSummaries.stream().collect(Collectors.toMap(S3ObjectSummary::getKey, S3ObjectSummary::getSize)); + List objectSummaries = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix) + .build()).get().contents(); + Map keysToSizes = objectSummaries.stream().collect(Collectors.toMap(S3Object::key, S3Object::size)); return keysToSizes; } - private static ObjectMetadataProvider encrypt() { - ObjectMetadataProvider objectMetadataProvider = new ObjectMetadataProvider() { - @Override - public void provideObjectMetadata(File file, ObjectMetadata objectMetadata) { - objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); - } - }; - return objectMetadataProvider; - } - - void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List files, boolean encrypt) { + void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List files, boolean encrypt) throws ExecutionException, InterruptedException { createBucket(bucketName); try { - if (encrypt) { - transferManager.uploadFileList(bucketName, keyPrefix, new File(dirPath), files, encrypt()).waitForCompletion(); - } else { - transferManager.uploadFileList(bucketName, keyPrefix, new File(dirPath), files).waitForCompletion(); - } + transferManager.uploadDirectory(UploadDirectoryRequest.builder().bucket(bucketName).s3Prefix(keyPrefix).build()); out.println("Uploaded necessary files in: " + dirPath); - } catch (InterruptedException e) { - throw new RuntimeException("Could not upload the directory: " + dirPath + " in its entirety"); - } catch (AmazonS3Exception e) { + } catch (S3Exception e) { ErrorExit.exceptionMessage(e, "MultiplePartUpload cannot finish. Check your keys and sign methods.", COMMAND_ERROR); } } @@ -113,17 +98,13 @@ void downloadDirectory(String bucketName, String keyPrefix, String dirPath) { if (!dir.isDirectory()) { throw new RuntimeException("Not a local directory thus nothing will be saved"); } else { - try { - transferManager.downloadDirectory(bucketName, keyPrefix, new File(dirPath), true).waitForCompletion(); - out.println("Downloaded the bucket(" + bucketName + ") with the prefix(" + keyPrefix + ") to the local directory: " + dirPath); - } catch (InterruptedException e) { - throw new RuntimeException("Could not download the bucket: " + bucketName + " in its entirety"); - } + transferManager.downloadDirectory(DownloadDirectoryRequest.builder().bucket(bucketName).filter(DownloadFilter.allObjects()).build()); + out.println("Downloaded the bucket(" + bucketName + ") with the prefix(" + keyPrefix + ") to the local directory: " + dirPath); } } //-----------------------Shutdown----------------------- void shutDown() { - transferManager.shutdownNow(); + transferManager.close(); } } diff --git a/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java b/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java index 2cf66117..584bfe7d 100644 --- a/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java +++ b/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java @@ -8,15 +8,16 @@ import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; -import com.amazonaws.services.s3.model.AmazonS3Exception; import io.dockstore.toolbackup.client.cli.common.AWSConfig; import io.dockstore.toolbackup.client.cli.common.DirCleaner; import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutionException; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import software.amazon.awssdk.services.s3.model.S3Exception; /** * Created by kcao on 24/01/17. @@ -26,7 +27,7 @@ public class S3CommunicatorIT { private static S3Communicator s3Communicator; @BeforeClass - public static void setUp() { + public static void setUp() throws ExecutionException, InterruptedException { AWSConfig.generateCredentials(); DirectoryGenerator.createDir(DIR); @@ -48,7 +49,7 @@ public void uploadDirectory() throws Exception { } @Test(expected = IllegalArgumentException.class) - public void uploadDirectoryNonexistentDirectory() { + public void uploadDirectoryNonexistentDirectory() throws ExecutionException, InterruptedException { s3Communicator.uploadDirectory(BUCKET, PREFIX, NON_EXISTING_DIR, null, false); } @@ -57,8 +58,8 @@ public void downloadDirectoryNotDir() { s3Communicator.downloadDirectory(BUCKET, PREFIX, NON_EXISTING_DIR); } - @Test(expected = AmazonS3Exception.class) - public void downloadDirectoryNoBucket() { + @Test(expected = S3Exception.class) + public void downloadDirectoryNoBucket() throws ExecutionException, InterruptedException { assumeFalse(s3Communicator.doesBucketExist(NON_EXISTING_BUCKET)); s3Communicator.downloadDirectory(NON_EXISTING_BUCKET, "", DIR); } From 87cf9d7b2c6b88a5799f2c0c86eca9893334fd2d Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Wed, 24 Jun 2026 14:14:29 -0400 Subject: [PATCH 2/4] fix? --- .../dockstore/toolbackup/client/cli/S3Communicator.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java index 74963df4..861dce18 100644 --- a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java +++ b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java @@ -5,6 +5,7 @@ import java.io.File; import java.net.URI; +import java.nio.file.Paths; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -84,7 +85,7 @@ void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List s3Object.key().startsWith(keyPrefix); + transferManager.downloadDirectory(DownloadDirectoryRequest.builder().bucket(bucketName).filter(filter).build()); out.println("Downloaded the bucket(" + bucketName + ") with the prefix(" + keyPrefix + ") to the local directory: " + dirPath); } } From 3cad07c74dce77686063d8b1cac8cc7f7982b53c Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Wed, 24 Jun 2026 14:31:29 -0400 Subject: [PATCH 3/4] more test fix --- .../io/dockstore/toolbackup/client/cli/S3Communicator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java index 861dce18..10917d6d 100644 --- a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java +++ b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java @@ -85,6 +85,9 @@ void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List s3Object.key().startsWith(keyPrefix); - transferManager.downloadDirectory(DownloadDirectoryRequest.builder().bucket(bucketName).filter(filter).build()); + transferManager.downloadDirectory(DownloadDirectoryRequest.builder().bucket(bucketName).destination(Paths.get(dirPath)).filter(filter).build()); out.println("Downloaded the bucket(" + bucketName + ") with the prefix(" + keyPrefix + ") to the local directory: " + dirPath); } } From b091b949d2b01cb53796d1b105818d9efd68068e Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Thu, 2 Jul 2026 10:38:00 -0400 Subject: [PATCH 4/4] remove unused parameter --- .../io/dockstore/toolbackup/client/cli/S3Communicator.java | 2 +- .../java/io/dockstore/toolbackup/client/cli/DownloaderIT.java | 2 +- .../io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java index 10917d6d..debba23e 100644 --- a/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java +++ b/toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java @@ -81,7 +81,7 @@ Map getKeysToSizes(String bucketName, String prefix) throws Execut return keysToSizes; } - void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List files, boolean encrypt) throws ExecutionException, InterruptedException { + void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List files) throws ExecutionException, InterruptedException { createBucket(bucketName); try { diff --git a/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/DownloaderIT.java b/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/DownloaderIT.java index 3dbae2ca..d849bf54 100644 --- a/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/DownloaderIT.java +++ b/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/DownloaderIT.java @@ -34,7 +34,7 @@ public void download() throws Exception { file.createNewFile(); files.add(file); - s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files, false); + s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files); new Downloader(null).download(BUCKET, PREFIX, DIR, s3Communicator); DirCleaner.deleteDir(DIR); diff --git a/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java b/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java index 584bfe7d..362649dd 100644 --- a/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java +++ b/toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java @@ -45,12 +45,12 @@ public void uploadDirectory() throws Exception { file.createNewFile(); files.add(file); - s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files, false); + s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files); } @Test(expected = IllegalArgumentException.class) public void uploadDirectoryNonexistentDirectory() throws ExecutionException, InterruptedException { - s3Communicator.uploadDirectory(BUCKET, PREFIX, NON_EXISTING_DIR, null, false); + s3Communicator.uploadDirectory(BUCKET, PREFIX, NON_EXISTING_DIR, null); } @Test(expected = RuntimeException.class)