superficial automated update, remove V1 AWS Java SDK

THIRD-PARTY-LICENSES.txt

@@ -1,5 +1,5 @@
 
-Lists of 404 third-party dependencies.
+Lists of 400 third-party dependencies.
      (Apache License, Version 2.0) akka-actor (com.typesafe.akka:akka-actor_2.13:2.5.32 - https://akka.io/)
      (Apache License, Version 2.0) akka-protobuf (com.typesafe.akka:akka-protobuf_2.13:2.5.32 - https://akka.io/)
      (Apache License, Version 2.0) akka-slf4j (com.typesafe.akka:akka-slf4j_2.13:2.5.32 - https://akka.io/)
@@ -66,6 +66,7 @@ Lists of 404 third-party dependencies.
      (Apache License, Version 2.0) AWS Java SDK :: Regions (software.amazon.awssdk:regions:2.31.56 - https://aws.amazon.com/sdkforjava/core/regions)
      (Apache License, Version 2.0) AWS Java SDK :: Retries (software.amazon.awssdk:retries:2.31.56 - https://aws.amazon.com/sdkforjava/core/retries)
      (Apache License, Version 2.0) AWS Java SDK :: Retries API (software.amazon.awssdk:retries-spi:2.31.56 - https://aws.amazon.com/sdkforjava/core/retries-spi)
+     (Apache License, Version 2.0) AWS Java SDK :: S3 :: Transfer Manager (software.amazon.awssdk:s3-transfer-manager:2.31.56 - https://aws.amazon.com/sdkforjava)
      (Apache License, Version 2.0) AWS Java SDK :: SDK Core (software.amazon.awssdk:sdk-core:2.31.56 - https://aws.amazon.com/sdkforjava)
      (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon Athena (software.amazon.awssdk:athena:2.31.56 - https://aws.amazon.com/sdkforjava)
      (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon CloudWatch (software.amazon.awssdk:cloudwatch:2.31.56 - https://aws.amazon.com/sdkforjava)
@@ -77,9 +78,6 @@ Lists of 404 third-party dependencies.
      (Apache License, Version 2.0) AWS Java SDK :: Third Party :: Jackson-core (software.amazon.awssdk:third-party-jackson-core:2.31.56 - https://aws.amazon.com/sdkforjava)
      (Apache License, Version 2.0) AWS Java SDK :: Third Party :: Jackson-dataformat-cbor (software.amazon.awssdk:third-party-jackson-dataformat-cbor:2.31.56 - https://aws.amazon.com/sdkforjava)
      (Apache License, Version 2.0) AWS Java SDK :: Utilities (software.amazon.awssdk:utils:2.31.56 - https://aws.amazon.com/sdkforjava/utils)
-     (Apache License, Version 2.0) AWS Java SDK for Amazon S3 (com.amazonaws:aws-java-sdk-s3:1.11.83 - https://aws.amazon.com/sdkforjava)
-     (Apache License, Version 2.0) AWS Java SDK for AWS KMS (com.amazonaws:aws-java-sdk-kms:1.11.83 - https://aws.amazon.com/sdkforjava)
-     (Apache License, Version 2.0) AWS SDK for Java - Core (com.amazonaws:aws-java-sdk-core:1.11.83 - https://aws.amazon.com/sdkforjava)
      (The Apache Software License, Version 2.0) Bean Validation API (javax.validation:validation-api:1.1.0.Final - http://beanvalidation.org)
      (MIT) better-files (com.github.pathikrit:better-files_2.13:3.9.1 - https://github.com/pathikrit/better-files)
      (Bouncy Castle Licence) Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.84 - https://www.bouncycastle.org/download/bouncy-castle-java/)
@@ -271,7 +269,6 @@ Lists of 404 third-party dependencies.
      (Apache License, Version 2.0) JJWT :: Extensions :: Jackson (io.jsonwebtoken:jjwt-jackson:0.11.5 - https://github.com/jwtk/jjwt/jjwt-jackson)
      (Apache License, Version 2.0) JJWT :: Impl (io.jsonwebtoken:jjwt-impl:0.11.5 - https://github.com/jwtk/jjwt/jjwt-impl)
      (The BSD License) JLine Bundle (org.jline:jline:3.21.0 - http://nexus.sonatype.org/oss-repository-hosting.html/jline-parent/jline)
-     (Apache License, Version 2.0) JMES Path Query library (com.amazonaws:jmespath-java:1.11.83 - https://aws.amazon.com/sdkforjava)
      (Apache License, Version 2.0) Joda-Time (joda-time:joda-time:2.12.5 - https://www.joda.org/joda-time/)
      (Apache License, Version 2.0) jOOQ (org.jooq:jooq:3.19.7 - http://www.jooq.org/jooq)
      (The MIT License) JOpt Simple (net.sf.jopt-simple:jopt-simple:5.0.3 - http://pholser.github.io/jopt-simple)
@@ -372,7 +369,6 @@ Lists of 404 third-party dependencies.
      (EPL 2.0) (GPL2 w/ CPE) ServiceLocator Default Implementation (org.glassfish.hk2:hk2-locator:3.0.6 - https://github.com/eclipse-ee4j/glassfish-hk2/hk2-locator)
      (MIT) SLF4J API Module (org.slf4j:slf4j-api:2.0.18 - http://www.slf4j.org)
      (Apache License, Version 2.0) SnakeYAML (org.yaml:snakeyaml:2.5 - https://bitbucket.org/snakeyaml/snakeyaml)
-     (The Apache License, Version 2.0) software.amazon.ion:ion-java (software.amazon.ion:ion-java:1.0.1 - https://github.com/amznlabs/ion-java/)
      (MIT) sourcecode_2.13 (com.lihaoyi:sourcecode_2.13:0.2.8 - https://github.com/lihaoyi/sourcecode)
      (Apache 2) spray-json (io.spray:spray-json_2.13:1.3.6 - https://github.com/spray/spray-json)
      (Apache-2.0) ssl-config-core (com.typesafe:ssl-config-core_2.13:0.3.8 - https://github.com/lightbend/ssl-config)

toolbackup/pom.xml

@@ -143,25 +143,36 @@
 
         <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3 -->
         <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-s3</artifactId>
-            <version>${aws.version}</version>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-core -->
         <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-core</artifactId>
-            <version>${aws.version}</version>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>auth</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>sdk-core</artifactId>
         </dependency>
 
 
+
         <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3-transfer-manager</artifactId>
+        </dependency>
 
         <!-- https://mvnrepository.com/artifact/net.sf.jopt-simple/jopt-simple -->
         <dependency>

toolbackup/src/main/java/io/dockstore/toolbackup/client/cli/S3Communicator.java

@@ -3,54 +3,54 @@
 import static io.dockstore.toolbackup.client.cli.Client.COMMAND_ERROR;
 import static java.lang.System.out;
 
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.profile.ProfileCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CreateBucketRequest;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.ObjectMetadataProvider;
-import com.amazonaws.services.s3.transfer.TransferManager;
 import java.io.File;
+import java.net.URI;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.ListBucketsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.config.DownloadFilter;
+import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest;
+import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest;
 
 /**
  * Created by kcao on 12/01/17.
  */
 class S3Communicator {
 
-    private TransferManager transferManager;
-    private AmazonS3Client s3Client;
+    private S3TransferManager transferManager;
+    private S3AsyncClient s3Client;
 
     S3Communicator() {
-        s3Client = new AmazonS3Client(new ProfileCredentialsProvider().getCredentials());
-        s3Client.setEndpoint("http://localhost:8080");
-        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).disableChunkedEncoding().build());
+        s3Client = S3AsyncClient.builder().endpointOverride(URI.create("http://localhost:8080")).credentialsProvider(ProfileCredentialsProvider.builder().build())
+            .forcePathStyle(true).build();
 
-        transferManager = new TransferManager(s3Client);
+        transferManager = S3TransferManager.builder().s3Client(s3Client).build();
     }
 
     S3Communicator(String section, String endpoint) {
-        ClientConfiguration opts = new ClientConfiguration();
-        opts.setSignerOverride("S3SignerType");
-        s3Client = new AmazonS3Client(new ProfileCredentialsProvider(section).getCredentials(), opts);
+        s3Client = S3AsyncClient.builder().credentialsProvider(ProfileCredentialsProvider.builder().build()).endpointOverride(URI.create(endpoint)).credentialsProvider(ProfileCredentialsProvider.builder().build())
+           .forcePathStyle(true).build();
 
-        s3Client.setEndpoint(endpoint);
-        s3Client.setS3ClientOptions(S3ClientOptions.builder().build());
-
-        transferManager = new TransferManager(s3Client);
+        transferManager = S3TransferManager.builder().s3Client(s3Client).build();
     }
 
     //-----------------------Report-----------------------
-    long getCloudTotalInB(String bucketName, String prefix) {
+    long getCloudTotalInB(String bucketName, String prefix) throws ExecutionException, InterruptedException {
         long total = 0;
 
-        List<S3ObjectSummary> objectSummaries = s3Client.listObjects(bucketName, prefix).getObjectSummaries();
-        List<Long> sizes = objectSummaries.stream().map(S3ObjectSummary::getSize).collect(Collectors.toList());
+        List<S3Object> objectSummaries = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
+                .build()).get().contents();
+        List<Long> sizes = objectSummaries.stream().map(S3Object::size).toList();
 
         for (long size : sizes) {
             total += size;
@@ -60,48 +60,37 @@
     }
 
     //-----------------------Upload-----------------------
-    boolean doesBucketExist(String bucketName) {
-        return s3Client.doesBucketExist(bucketName);
+    boolean doesBucketExist(String bucketName) throws ExecutionException, InterruptedException {
+        return s3Client.listBuckets(ListBucketsRequest.builder().prefix(bucketName).build()).get().hasBuckets();
     }
 
-    void createBucket(String bucketName) {
+    void createBucket(String bucketName) throws ExecutionException, InterruptedException {
         if (!doesBucketExist(bucketName)) {
-            s3Client.createBucket(new CreateBucketRequest(bucketName));
+            s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName)
+                    .build());
         }
     }
 
-    Map<String, Long> getKeysToSizes(String bucketName, String prefix) {
+    Map<String, Long> getKeysToSizes(String bucketName, String prefix) throws ExecutionException, InterruptedException {
         createBucket(bucketName);
 
-        List<S3ObjectSummary> objectSummaries = s3Client.listObjects(bucketName, prefix).getObjectSummaries();
-        Map<String, Long> keysToSizes = objectSummaries.stream().collect(Collectors.toMap(S3ObjectSummary::getKey, S3ObjectSummary::getSize));
+        List<S3Object> objectSummaries = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
+                .build()).get().contents();
+        Map<String, Long> keysToSizes = objectSummaries.stream().collect(Collectors.toMap(S3Object::key, S3Object::size));
 
         return keysToSizes;
     }
 
-    private static ObjectMetadataProvider encrypt() {
-        ObjectMetadataProvider objectMetadataProvider = new ObjectMetadataProvider() {
-            @Override
-            public void provideObjectMetadata(File file, ObjectMetadata objectMetadata) {
-                objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
-            }
-        };
-        return objectMetadataProvider;
-    }
-
-    void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List<File> files, boolean encrypt) {
+    void uploadDirectory(String bucketName, String keyPrefix, String dirPath, List<File> files) throws ExecutionException, InterruptedException {
         createBucket(bucketName);
 
         try {
-            if (encrypt) {
-                transferManager.uploadFileList(bucketName, keyPrefix, new File(dirPath), files, encrypt()).waitForCompletion();
-            } else {
-                transferManager.uploadFileList(bucketName, keyPrefix, new File(dirPath), files).waitForCompletion();
+            if (files == null) {
+                throw new IllegalArgumentException();
             }
+            transferManager.uploadDirectory(UploadDirectoryRequest.builder().source(Paths.get(dirPath)).bucket(bucketName).s3Prefix(keyPrefix).build());
             out.println("Uploaded necessary files in: " + dirPath);
-        } catch (InterruptedException e) {
-            throw new RuntimeException("Could not upload the directory: " + dirPath + " in its entirety");
-        } catch (AmazonS3Exception e) {
+        } catch (S3Exception e) {
             ErrorExit.exceptionMessage(e, "MultiplePartUpload cannot finish. Check your keys and sign methods.", COMMAND_ERROR);
         }
     }
@@ -113,17 +102,17 @@
         if (!dir.isDirectory()) {
             throw new RuntimeException("Not a local directory thus nothing will be saved");
         } else {
-            try {
-                transferManager.downloadDirectory(bucketName, keyPrefix, new File(dirPath), true).waitForCompletion();
-                out.println("Downloaded the bucket(" + bucketName + ") with the prefix(" + keyPrefix + ") to the local directory: " + dirPath);
-            } catch (InterruptedException e) {
-                throw new RuntimeException("Could not download the bucket: " + bucketName + " in its entirety");
+            if (keyPrefix == null) {
+                throw new IllegalArgumentException();
             }
+            DownloadFilter filter = s3Object -> s3Object.key().startsWith(keyPrefix);
+            transferManager.downloadDirectory(DownloadDirectoryRequest.builder().bucket(bucketName).destination(Paths.get(dirPath)).filter(filter).build());
+            out.println("Downloaded the bucket(" + bucketName + ") with the prefix(" + keyPrefix + ") to the local directory: " + dirPath);
         }
     }
 
     //-----------------------Shutdown-----------------------
     void shutDown() {
-        transferManager.shutdownNow();
+        transferManager.close();
     }
 }

toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/DownloaderIT.java

@@ -34,7 +34,7 @@ public void download() throws Exception {
         file.createNewFile();
         files.add(file);
 
-        s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files, false);
+        s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files);
 
         new Downloader(null).download(BUCKET, PREFIX, DIR, s3Communicator);
         DirCleaner.deleteDir(DIR);

toolbackup/src/test/java/io/dockstore/toolbackup/client/cli/S3CommunicatorIT.java

@@ -8,15 +8,16 @@
 import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 
-import com.amazonaws.services.s3.model.AmazonS3Exception;
 import io.dockstore.toolbackup.client.cli.common.AWSConfig;
 import io.dockstore.toolbackup.client.cli.common.DirCleaner;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import software.amazon.awssdk.services.s3.model.S3Exception;
 
 /**
  * Created by kcao on 24/01/17.
@@ -26,7 +27,7 @@ public class S3CommunicatorIT {
     private static S3Communicator s3Communicator;
 
     @BeforeClass
-    public static void setUp() {
+    public static void setUp() throws ExecutionException, InterruptedException {
         AWSConfig.generateCredentials();
 
         DirectoryGenerator.createDir(DIR);
@@ -44,21 +45,21 @@ public void uploadDirectory() throws Exception {
         file.createNewFile();
         files.add(file);
 
-        s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files, false);
+        s3Communicator.uploadDirectory(BUCKET, PREFIX, DIR, files);
     }
 
     @Test(expected = IllegalArgumentException.class)
-    public void uploadDirectoryNonexistentDirectory() {
-        s3Communicator.uploadDirectory(BUCKET, PREFIX, NON_EXISTING_DIR, null, false);
+    public void uploadDirectoryNonexistentDirectory() throws ExecutionException, InterruptedException {
+        s3Communicator.uploadDirectory(BUCKET, PREFIX, NON_EXISTING_DIR, null);
     }
 
     @Test(expected = RuntimeException.class)
     public void downloadDirectoryNotDir() {
         s3Communicator.downloadDirectory(BUCKET, PREFIX, NON_EXISTING_DIR);
     }
 
-    @Test(expected = AmazonS3Exception.class)
-    public void downloadDirectoryNoBucket() {
+    @Test(expected = S3Exception.class)
+    public void downloadDirectoryNoBucket() throws ExecutionException, InterruptedException {
         assumeFalse(s3Communicator.doesBucketExist(NON_EXISTING_BUCKET));
         s3Communicator.downloadDirectory(NON_EXISTING_BUCKET, "", DIR);
     }