v3.3.1

.gitignore

@@ -13,3 +13,5 @@ out
 *.iml
 .gradle
 .asscache
+/ecs-sync.ipr
+/ecs-sync.iws

build.gradle

@@ -1,6 +1,6 @@
 allprojects {
     group = 'com.emc.ecs'
-    version = '3.3.0'
+    version = '3.3.1'
 }
 
 ext.mainClass = 'com.emc.ecs.sync.EcsSync'
@@ -34,13 +34,13 @@ dependencies {
             files('lib/security-1.1.jar'),
             "commons-cli:commons-cli:1.3.1",
             "com.zaxxer:HikariCP-java7:2.4.11",
-            "com.amazonaws:aws-java-sdk-s3:1.11.53",
+            "com.amazonaws:aws-java-sdk-s3:1.11.714",
             // required due to vulnerability in jackson-databind
             "com.fasterxml.jackson.core:jackson-databind:2.8.11.1",
             "com.google.code.gson:gson:2.0",
             "com.emc.ecs:atmos-client:3.1.0",
             "com.emc.ecs:nfs-client:1.0.3",
-            "com.emc.ecs:object-client:3.1.2",
+            "com.emc.ecs:object-client:3.1.3",
             "com.sun.jersey:jersey-server:1.19.4",
             "mysql:mysql-connector-java:5.1.37",
             "net.java.truevfs:truevfs-profile-default:0.11.0",
@@ -66,6 +66,8 @@ compileJava {
     }
 }
 
+compileTestJava.options.encoding = 'UTF-8'
+
 dependencyLicenseReport.dependsOn 'compileJava'
 
 javadoc {
@@ -125,5 +127,5 @@ distributions {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '2.14'
+    gradleVersion = '4.10.3'
 }

ecs-sync-model/src/main/java/com/emc/ecs/sync/config/storage/AwsS3Config.java

@@ -53,6 +53,7 @@ public class AwsS3Config extends AbstractConfig {
     private Protocol protocol;
     private String host;
     private int port = -1;
+    private String region;
     private String accessKey;
     private String secretKey;
     private boolean disableVHosts;
@@ -138,6 +139,15 @@ public void setPort(int port) {
         this.port = port;
     }
 
+    @Option(orderIndex = 35, advanced = true, description = "Overrides the AWS region that would be inferred from the endpoint")
+    public String getRegion() {
+        return region;
+    }
+
+    public void setRegion(String region) {
+        this.region = region;
+    }
+
     @Option(orderIndex = 40, locations = Option.Location.Form, required = true, description = "The S3 access key")
     public String getAccessKey() {
         return accessKey;

ecs-sync-model/src/main/java/com/emc/ecs/sync/rest/LogLevel.java

@@ -18,7 +18,7 @@
 
 @XmlEnum
 public enum LogLevel {
-    silent(false), quiet(false), verbose(true), debug(true);
+    silent(false), quiet(false), verbose(true), debug(true), trace(true);
 
     private boolean includeStackTrace = false;
 

ecs-sync-ui/grails-app/services/sync/ui/config/FileConfigService.groovy

@@ -83,7 +83,12 @@ class FileConfigService extends ConfigService {
 
     @Override
     URI configObjectQuickLink(String path) {
-        grailsLinkGenerator.link(uri: "/download/$path").toURI()
+        // toURI is picky about illegal characters (particularly spaces)
+        // there may be an easier way to do this, but I couldn't find it
+        def encodedPath = path.split('[/\\\\]')
+                .collect { s -> URLEncoder.encode(s, 'UTF-8').replace('+', '%20') }
+                .join('/')
+        grailsLinkGenerator.link(uri: "/download/$encodedPath").toURI()
     }
 
     @Override

ecs-sync-ui/src/main/groovy/sync/ui/ScheduleEntry.groovy

@@ -52,7 +52,7 @@ class ScheduleEntry implements Validateable {
     }
 
     def setXmlKey(String key) {
-        this.name = key.split('/').last().replaceFirst(/[.]xml$/, '')
+        this.name = key.split('[/\\\\]').last().replaceFirst(/[.]xml$/, '')
     }
 
     static constraints = {

ecs-sync-ui/src/main/groovy/sync/ui/SyncHistoryEntry.groovy

@@ -98,6 +98,6 @@ class SyncHistoryEntry {
     }
 
     private static String fileName(String key) {
-        return key.split('/').last()
+        return key.split('[/\\\\]').last()
     }
 }

ecs-sync-ui/src/main/groovy/sync/ui/SyncUtil.groovy

@@ -72,6 +72,7 @@ class SyncUtil {
         // if no dbTable or dbFile was specified, create a unique table name
         if (!dbTable && !syncConfig.options.dbFile) {
             dbTable = "sync_${new Date().format(SyncHistoryEntry.idFormat)}"
+            syncConfig.options.dbTable = dbTable
             syncConfig.properties.generatedTableName = dbTable
         }
         // if no dbFile or dbConnectString, use the dbConnectString in the UI app config

ecs-sync-ui/src/main/groovy/sync/ui/migration/MigrationHistoryEntry.groovy

@@ -15,9 +15,10 @@
  */
 package sync.ui.migration
 
-
+import groovy.util.logging.Slf4j
 import sync.ui.config.ConfigService
 
+@Slf4j
 class MigrationHistoryEntry {
     static String prefix = "archive/migration/"
     static String idFormat = "yyyyMMdd'T'HHmmss"
@@ -88,6 +89,6 @@ class MigrationHistoryEntry {
     }
 
     private static String fileName(String key) {
-        return key.split('/').last()
+        return key.split('[/\\\\]').last()
     }
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Tue Oct 31 12:58:58 CDT 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.3-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.14-bin.zip

gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

gradlew.bat

@@ -49,7 +49,6 @@ goto fail
 @rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

src/main/java/com/emc/ecs/sync/model/ObjectMetadata.java

@@ -229,6 +229,11 @@ public void setExpirationDate(Date expirationDate) {
         this.expirationDate = expirationDate;
     }
 
+    public ObjectMetadata withExpirationDate(Date expirationDate) {
+        setExpirationDate(expirationDate);
+        return this;
+    }
+
     public String getRetentionPolicy() {
         return retentionPolicy;
     }
@@ -237,6 +242,11 @@ public void setRetentionPolicy(String retentionPolicy) {
         this.retentionPolicy = retentionPolicy;
     }
 
+    public ObjectMetadata withRetentionPolicy(String retentionPolicy) {
+        setRetentionPolicy(retentionPolicy);
+        return this;
+    }
+
     public Date getRetentionEndDate() {
         return retentionEndDate;
     }
@@ -245,6 +255,10 @@ public void setRetentionEndDate(Date retentionEndDate) {
         this.retentionEndDate = retentionEndDate;
     }
 
+    public ObjectMetadata withRetentionEndDate(Date retentionEndDate) {
+        setRetentionEndDate(retentionEndDate);
+        return this;
+    }
     /**
      * For a given object path, returns the appropriate path that should contain that
      * object's Metadata container.  This is a path/file with the same name inside the

src/main/java/com/emc/ecs/sync/service/SyncJobService.java

@@ -65,7 +65,10 @@ public void setLogLevel(LogLevel logLevel) {
         // try to avoid a runtime dependency on log4j (untested)
         try {
             org.apache.log4j.Logger rootLogger = org.apache.log4j.LogManager.getRootLogger();
-            if (LogLevel.debug == logLevel) {
+            if (LogLevel.trace == logLevel) {
+                rootLogger.setLevel(org.apache.log4j.Level.TRACE);
+                java.util.logging.LogManager.getLogManager().getLogger("").setLevel(Level.FINEST);
+            } else if (LogLevel.debug == logLevel) {
                 rootLogger.setLevel(org.apache.log4j.Level.DEBUG);
                 java.util.logging.LogManager.getLogManager().getLogger("").setLevel(Level.FINE);
             } else if (LogLevel.verbose == logLevel) {
@@ -79,16 +82,6 @@ public void setLogLevel(LogLevel logLevel) {
                 java.util.logging.LogManager.getLogManager().getLogger("").setLevel(Level.SEVERE);
             }
 
-            org.apache.log4j.AppenderSkeleton mainAppender = (org.apache.log4j.AppenderSkeleton) rootLogger.getAppender("mainAppender");
-            org.apache.log4j.AppenderSkeleton stackAppender = (org.apache.log4j.AppenderSkeleton) rootLogger.getAppender("stacktraceAppender");
-            if (logLevel.isIncludeStackTrace()) {
-                if (mainAppender != null) mainAppender.setThreshold(org.apache.log4j.Level.OFF);
-                if (stackAppender != null) stackAppender.setThreshold(org.apache.log4j.Level.ALL);
-            } else {
-                if (mainAppender != null) mainAppender.setThreshold(org.apache.log4j.Level.ALL);
-                if (stackAppender != null) stackAppender.setThreshold(org.apache.log4j.Level.OFF);
-            }
-
             this.logLevel = logLevel;
         } catch (Throwable t) {
             log.warn("could not configure log4j (perhaps you're using a different logger, which is fine)", t);

src/main/java/com/emc/ecs/sync/storage/s3/AwsS3Storage.java

@@ -17,16 +17,17 @@
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.event.ProgressEventType;
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.internal.Constants;
 import com.amazonaws.services.s3.model.*;
 import com.amazonaws.services.s3.transfer.PersistableTransfer;
 import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
 import com.amazonaws.services.s3.transfer.Upload;
 import com.amazonaws.services.s3.transfer.internal.S3ProgressListener;
 import com.emc.ecs.sync.config.ConfigurationException;
@@ -93,20 +94,28 @@ public void configure(SyncStorage source, Iterator<SyncFilter> filters, SyncStor
 
         if (config.getSocketTimeoutMs() >= 0) cc.setSocketTimeout(config.getSocketTimeoutMs());
 
-        s3 = new AmazonS3Client(creds, cc);
+        AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
+                .withCredentials(new AWSStaticCredentialsProvider(creds))
+                .withClientConfiguration(cc);
 
         if (config.getHost() != null) {
-            String portStr = "";
-            if (config.getPort() > 0) portStr = ":" + config.getPort();
-            s3.setEndpoint(config.getHost() + portStr);
+            String endpoint = "";
+            if (config.getProtocol() != null) endpoint += config.getProtocol() + "://";
+            endpoint += config.getHost();
+            if (config.getPort() > 0) endpoint += ":" + config.getPort();
+            builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, config.getRegion()));
+        } else if (config.getRegion() != null) {
+            builder.withRegion(config.getRegion());
         }
 
         if (config.isDisableVHosts()) {
             log.info("The use of virtual hosted buckets has been DISABLED.  Path style buckets will be used.");
-            s3.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
+            builder.withPathStyleAccessEnabled(true);
         }
 
-        boolean bucketExists = s3.doesBucketExist(config.getBucketName());
+        s3 = builder.build();
+
+        boolean bucketExists = s3.doesBucketExistV2(config.getBucketName());
 
         boolean bucketHasVersions = false;
         if (bucketExists && config.isIncludeVersions()) {
@@ -418,23 +427,34 @@ void putObject(SyncObject obj, String targetKey) {
         if (options.isSyncAcl())
             req.setAccessControlList(s3AclFromSyncAcl(obj.getAcl(), options.isIgnoreInvalidAcls()));
 
-        // xfer manager will figure out if MPU is needed (based on threshold), do the MPU if necessary,
-        // and abort if it fails
-        TransferManagerConfiguration xferConfig = new TransferManagerConfiguration();
-        xferConfig.setMultipartUploadThreshold((long) config.getMpuThresholdMb() * 1024 * 1024);
-        xferConfig.setMinimumUploadPartSize((long) config.getMpuPartSizeMb() * 1024 * 1024);
-        TransferManager xferManager = new TransferManager(s3, Executors.newFixedThreadPool(config.getMpuThreadCount()));
-        xferManager.setConfiguration(xferConfig);
-
-        // directly update
-
-        final Upload upload = xferManager.upload(req, progressListener);
+        TransferManager xferManager = null;
         try {
-            String eTag = time((Callable<String>) () -> upload.waitForUploadResult().getETag(), OPERATION_MPU);
-            log.debug("Wrote {}, etag: {}", targetKey, eTag);
-        } catch (Exception e) {
-            if (e instanceof RuntimeException) throw (RuntimeException) e;
-            throw new RuntimeException("upload thread was interrupted", e);
+            // xfer manager will figure out if MPU is needed (based on threshold), do the MPU if necessary,
+            // and abort if it fails
+            xferManager = TransferManagerBuilder.standard()
+                    .withS3Client(s3)
+                    .withExecutorFactory(() -> Executors.newFixedThreadPool(config.getMpuThreadCount()))
+                    .withMultipartUploadThreshold((long) config.getMpuThresholdMb() * 1024 * 1024)
+                    .withMinimumUploadPartSize((long) config.getMpuPartSizeMb() * 1024 * 1024)
+                    .withShutDownThreadPools(true)
+                    .build();
+
+            // directly update
+
+            final Upload upload = xferManager.upload(req, progressListener);
+            try {
+                String eTag = time((Callable<String>) () -> upload.waitForUploadResult().getETag(), OPERATION_MPU);
+                log.debug("Wrote {}, etag: {}", targetKey, eTag);
+            } catch (Exception e) {
+                log.error("upload exception", e);
+                if (e instanceof RuntimeException) throw (RuntimeException) e;
+                throw new RuntimeException("upload thread was interrupted", e);
+            }
+        } finally {
+            // NOTE: apparently if we do not reference xferManager again after the upload() call (as in this finally
+            // block), the JVM will for some crazy reason determine it is eligible for GC and call finalize(), which
+            // shuts down the thread pool, fails the upload, and gives absolutely no indication of what's going on...
+            if (xferManager != null) xferManager.shutdownNow(false);
         }
     }