add size field to manifest and list only manifests

src/main/java/com/instaclustr/esop/impl/Manifest.java

@@ -48,6 +48,8 @@ public class Manifest implements Cloneable {
 
     private String schemaVersion;
 
+    private long size;
+
     public static Manifest from(final Snapshot snapshot) {
         return new Manifest(snapshot);
     }
@@ -91,6 +93,14 @@ public List<String> getTokens() {
         return tokens;
     }
 
+    public void setSize(long size) {
+        this.size = size;
+    }
+
+    public long getSize() {
+        return this.size;
+    }
+
     @JsonIgnore
     public String getInitialTokensCassandraYamlFragment() {
         return "initial_token: " + String.join(",", getTokens());

src/main/java/com/instaclustr/esop/impl/StorageInteractor.java

@@ -26,6 +26,10 @@ public StorageInteractor(final StorageLocation storageLocation) {
         this.storageLocation = storageLocation;
     }
 
+    public String resolveNodeAwareRemoteRoot() {
+        return Paths.get(storageLocation.clusterId).resolve(storageLocation.datacenterId).resolve(storageLocation.nodeId).toString();
+    }
+
     public String resolveNodeAwareRemotePath(final Path objectKey) {
         return Paths.get(storageLocation.clusterId).resolve(storageLocation.datacenterId).resolve(storageLocation.nodeId).resolve(objectKey).toString();
     }

src/main/java/com/instaclustr/esop/impl/backup/coordination/BaseBackupOperationCoordinator.java

@@ -132,7 +132,7 @@ public void coordinate(final Operation<BackupOperationRequest> operation) {
             final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
             final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
 
-            if (!snapshot.isPresent()) {
+            if (snapshot.isEmpty()) {
                 throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
             }
 
@@ -150,7 +150,7 @@ public void coordinate(final Operation<BackupOperationRequest> operation) {
                 backuper.init(manifest.getManifestEntries(true));
                 performUpload(manifest.getManifestEntries(false), backuper, operation, request);
 
-                // upload manifest as the last, possibly with updated file sizes as they were encrypted
+                manifest.setSize(manifest.getManifestEntries(true).stream().map(m -> m.size).reduce(Long::sum).orElse(0L));
                 backuper.uploadText(objectMapper.writeValueAsString(manifest),
                                     backuper.objectKeyToNodeAwareRemoteReference(manifest.getManifest().objectKey));
 

src/main/java/com/instaclustr/esop/s3/v2/BaseS3Backuper.java

@@ -32,13 +32,16 @@
 import software.amazon.awssdk.services.s3.model.CompletedPart;
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectAttributesRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectAttributesResponse;
 import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest;
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
 import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
 import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse;
 import software.amazon.awssdk.services.s3.model.MultipartUpload;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
 import software.amazon.awssdk.services.s3.model.NoSuchUploadException;
+import software.amazon.awssdk.services.s3.model.ObjectAttributes;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.SdkPartType;
 import software.amazon.awssdk.services.s3.model.StorageClass;
@@ -265,10 +268,10 @@ private void uploadFile(S3Client s3Client,
             if (!completeResponse.sdkHttpResponse().isSuccessful()) {
                 throw new RuntimeException(String.format("Unsuccessful multipart upload of %s, upload id %s", objectReference.canonicalPath, uploadId));
             } else {
-                logger.info("Completed multipart upload of {}, upload id {}, etag {}", objectReference.canonicalPath, uploadId, completeResponse.eTag());
+                logger.debug("Completed multipart upload of {}, upload id {}, etag {}", objectReference.canonicalPath, uploadId, completeResponse.eTag());
             }
 
-            logger.info("Waiting for " + objectReference.canonicalPath + " to exist");
+            logger.debug("Waiting for " + objectReference.canonicalPath + " to exist");
 
             s3Clients.getNonEncryptingClient().waiter().waitUntilObjectExists(HeadObjectRequest.builder()
                                                                                                .bucket(request.storageLocation.bucket)
@@ -278,17 +281,25 @@ private void uploadFile(S3Client s3Client,
                                                                                                          .waitTimeout(Duration.of(1, ChronoUnit.MINUTES))
                                                                                                          .build());
 
-            logger.info("Object under key " + objectReference.canonicalPath + " exists");
-
-//            GetObjectAttributesResponse objectAttributes = s3Clients.getNonEncryptingClient()
-//                                                                    .getObjectAttributes(GetObjectAttributesRequest
-//                                                                                         .builder()
-//                                                                                         .bucket(request.storageLocation.bucket)
-//                                                                                         .key(objectReference.canonicalPath)
-//                                                                                         .objectAttributes(ObjectAttributes.OBJECT_SIZE)
-//                                                                                         .build());
-//
-//            manifestEntry.size = objectAttributes.objectSize();
+            logger.debug("Object under key " + objectReference.canonicalPath + " exists");
+
+            if (s3Clients.hasEncryptingClient()) {
+                try {
+                    GetObjectAttributesResponse objectAttributes = s3Clients.getNonEncryptingClient()
+                            .getObjectAttributes(GetObjectAttributesRequest
+                                                         .builder()
+                                                         .bucket(request.storageLocation.bucket)
+                                                         .key(objectReference.canonicalPath)
+                                                         .objectAttributes(ObjectAttributes.OBJECT_SIZE)
+                                                         .build());
+
+                    manifestEntry.size = objectAttributes.objectSize();
+                }
+                catch (Throwable t) {
+                    logger.warn("Unable to get attribute {} for key {} by GetObjectAttributes request. Please check your permissions.",
+                                ObjectAttributes.OBJECT_SIZE, objectReference.canonicalPath);
+                }
+            }
         } catch (Throwable t) {
             t.printStackTrace();
             multipartAbortionService.abortMultipartUpload(uploadId, request, objectReference);

src/main/java/com/instaclustr/esop/s3/v2/BaseS3Restorer.java

@@ -405,7 +405,8 @@ public static List<List<String>> splitList(List<String> list, int maxLength) {
     public void downloadManifestsToDirectory(Path downloadDir) throws Exception {
         FileUtils.createDirectory(downloadDir);
         FileUtils.cleanDirectory(downloadDir.toFile());
-        final List<S3Object> manifestSumms = listBucket("", s -> s.contains("manifests"));
+
+        final List<S3Object> manifestSumms = listBucket(resolveNodeAwareRemotePath(Paths.get("manifests")), filter -> true);
         for (S3Object o : manifestSumms) {
             Path manifestPath = Paths.get(o.key());
             Path destination = downloadDir.resolve(manifestPath);

src/test/java/com/instaclustr/esop/backup/embedded/azure/AzureListingAndBackupRemovalTest.java

@@ -24,7 +24,7 @@ public class AzureListingAndBackupRemovalTest extends BaseListingRemovalTest {
 
     @Override
     protected List<Module> getModules() {
-        return new ArrayList<Module>() {{
+        return new ArrayList<>() {{
             add(new AzureModule());
         }};
     }

src/test/java/com/instaclustr/esop/backup/embedded/gcp/GoogleListingAndBackupRemovalTest.java

@@ -24,7 +24,7 @@ public class GoogleListingAndBackupRemovalTest extends BaseListingRemovalTest {
 
     @Override
     protected List<Module> getModules() {
-        return new ArrayList<Module>() {{
+        return new ArrayList<>() {{
             add(new GCPModule());
         }};
     }

src/test/java/com/instaclustr/esop/backup/embedded/local/LocalListingAndBackupRemovalTest.java

@@ -19,7 +19,7 @@ public class LocalListingAndBackupRemovalTest extends BaseListingRemovalTest {
 
     @Override
     protected List<Module> getModules() {
-        return new ArrayList<Module>() {{
+        return new ArrayList<>() {{
             add(new LocalFileModule());
         }};
     }

src/test/java/com/instaclustr/esop/backup/embedded/s3/aws/v2/S3ListingAndBackupRemovalTest.java

@@ -22,7 +22,7 @@ public class S3ListingAndBackupRemovalTest extends BaseListingRemovalTest {
 
     @Override
     protected List<Module> getModules() {
-        return new ArrayList<Module>() {{
+        return new ArrayList<>() {{
             add(new S3Module());
         }};
     }