Merge remote-tracking branch 'origin/master'

DANS-KNAW · May 15, 2024 · 769ff3d · 769ff3d
2 parents 45c6d8f + 990df57
commit 769ff3d
Show file tree

Hide file tree

Showing 19 changed files with 653 additions and 142 deletions.
diff --git a/pom.xml b/pom.xml
@@ -87,6 +87,45 @@
             <groupId>nl.knaw.dans</groupId>
             <artifactId>dans-java-utils</artifactId>
         </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>RELEASE</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>5.8.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-testing</artifactId>
+            <version>3.0.5</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <version>5.4.0</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/src/main/assembly/dist/bin/dd-manage-deposit b/src/main/assembly/dist/bin/dd-manage-deposit
@@ -5,4 +5,4 @@ BINPATH=$(command readlink -f $0 2> /dev/null || command grealpath $0 2> /dev/nu
 APPHOME=$(dirname  $(dirname $BINPATH))
 CONFIG_PATH=/etc/opt/dans.knaw.nl/$SCRIPTNAME/config.yml
 
-java -Ddans.default.config=$CONFIG_PATH -jar $APPHOME/bin/$SCRIPTNAME.jar "$@"
+java $DANS_JAVA_OPTS -Ddans.default.config=$CONFIG_PATH -jar $APPHOME/bin/$SCRIPTNAME.jar "$@"
diff --git a/src/main/assembly/dist/cfg/config.yml b/src/main/assembly/dist/cfg/config.yml
@@ -18,7 +18,7 @@ depositBoxes:
   - /var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/failed
   - /var/opt/dans.knaw.nl/tmp/sword2-uploads
 
-pollingInterval: 5000
+pollingInterval: 500
 
 depositPropertiesDatabase:
   driverClass: org.postgresql.Driver

diff --git a/src/main/java/nl/knaw/dans/managedeposit/DdManageDepositApplication.java b/src/main/java/nl/knaw/dans/managedeposit/DdManageDepositApplication.java
@@ -17,11 +17,11 @@
 package nl.knaw.dans.managedeposit;
 
 import io.dropwizard.core.Application;
+import io.dropwizard.core.setup.Bootstrap;
+import io.dropwizard.core.setup.Environment;
 import io.dropwizard.db.DataSourceFactory;
 import io.dropwizard.hibernate.HibernateBundle;
 import io.dropwizard.hibernate.UnitOfWorkAwareProxyFactory;
-import io.dropwizard.core.setup.Bootstrap;
-import io.dropwizard.core.setup.Environment;
 import nl.knaw.dans.managedeposit.core.CsvMessageBodyWriter;
 import nl.knaw.dans.managedeposit.core.DepositProperties;
 import nl.knaw.dans.managedeposit.core.service.DepositStatusUpdater;
@@ -34,10 +34,6 @@
 
 public class DdManageDepositApplication extends Application<DdManageDepositConfiguration> {
 
-    public static void main(final String[] args) throws Exception {
-        new DdManageDepositApplication().run(args);
-    }
-
     private final HibernateBundle<DdManageDepositConfiguration> depositPropertiesHibernate =
         new HibernateBundle<>(DepositProperties.class) {
 
@@ -47,6 +43,10 @@ public DataSourceFactory getDataSourceFactory(DdManageDepositConfiguration confi
             }
         };
 
+    public static void main(final String[] args) throws Exception {
+        new DdManageDepositApplication().run(args);
+    }
+
     @Override
     public String getName() {
         return "Dd Manage Deposit";
@@ -70,13 +70,10 @@ public void run(final DdManageDepositConfiguration configuration, final Environm
 
         final UnitOfWorkAwareProxyFactory proxyFactory = new UnitOfWorkAwareProxyFactory(depositPropertiesHibernate);
         DepositStatusUpdater depositStatusUpdater = proxyFactory.create(
-            DepositStatusUpdater.class,
-            new Class[] { DepositPropertiesDAO.class },
-            new Object[] { depositPropertiesDAO });
+            DepositStatusUpdater.class, DepositPropertiesDAO.class, depositPropertiesDAO);
 
         final IngestPathMonitor ingestPathMonitor = new IngestPathMonitor(configuration.getDepositBoxes(), depositStatusUpdater, configuration.getPollingInterval());
         environment.lifecycle().manage(ingestPathMonitor);
-
     }
 
 }
diff --git a/src/main/java/nl/knaw/dans/managedeposit/DdManageDepositConfiguration.java b/src/main/java/nl/knaw/dans/managedeposit/DdManageDepositConfiguration.java
@@ -18,15 +18,16 @@
 
 import io.dropwizard.core.Configuration;
 import io.dropwizard.db.DataSourceFactory;
-import nl.knaw.dans.managedeposit.core.service.TextTruncation;
 
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 
+@SuppressWarnings("unused")
 public class DdManageDepositConfiguration extends Configuration {
+    private static final long DEFAULT_POLLING_INTERVAL = 500;
     @Valid
     @NotNull
     private DataSourceFactory database = new DataSourceFactory();
@@ -52,7 +53,7 @@ public void setDepositPropertiesDatabase(DataSourceFactory database) {
     }
 
     public long getPollingInterval() {
-        return pollingInterval > 0 ? pollingInterval : TextTruncation.pollingInterval;
+        return pollingInterval > 0 ? pollingInterval : DEFAULT_POLLING_INTERVAL;
     }
 
     public void setPollingInterval(long pollingInterval) {

diff --git a/src/main/java/nl/knaw/dans/managedeposit/core/CsvMessageBodyWriter.java b/src/main/java/nl/knaw/dans/managedeposit/core/CsvMessageBodyWriter.java
@@ -45,14 +45,16 @@ public boolean isWriteable(Class type, Type genericType, Annotation[] annotation
     @Override
     public void writeTo(List data, Class aClass, Type type, Annotation[] annotations, MediaType mediaType, MultivaluedMap multivaluedMap, OutputStream outputStream) throws
         IOException, WebApplicationException {
-        if (data != null && data.size() > 0) {
+        if (data != null && !data.isEmpty()) {
             // TODO: pass the mapper in at configuration time
             CsvMapper mapper = new CsvMapper();
             Object o = data.get(0);
             CsvSchema schema = mapper.schemaFor(o.getClass())
                 .withHeader()
                 .sortedBy("depositor", "depositId", "bagName", "depositState", "depositCreationTimestamp", "depositUpdateTimestamp", "description", "location", "storageInBytes", "deleted")
-                    .rebuild().build();
+                .rebuild()
+                .setNullValue("undefined")
+                .build();
 
             mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
             mapper.registerModule(new JavaTimeModule());

diff --git a/src/main/java/nl/knaw/dans/managedeposit/core/DepositProperties.java b/src/main/java/nl/knaw/dans/managedeposit/core/DepositProperties.java
@@ -30,6 +30,8 @@
     name = "showAll",
     query = "SELECT dp FROM DepositProperties dp"
 )
+
+@SuppressWarnings("unused")
 public class DepositProperties {
 
     @Column(name = "depositor", nullable = false)                          // depositor.userId
@@ -49,26 +51,23 @@ public class DepositProperties {
     @Column(name = "deposit_update_timestamp")                             // modified timestamp of deposit.properties
     private OffsetDateTime depositUpdateTimestamp;
 
-    @Column(name = "description", length = TextTruncation.maxDescriptionLength)   // state.description
+    @Column(name = "description", length = TextTruncation.MAX_DESCRIPTION_LENGTH)   // state.description
     private String description;
 
-    @Column(name = "location", length = TextTruncation.maxDirectoryLength)        // full parent-path on disk
+    @Column(name = "location", length = TextTruncation.MAX_DIRECTORY_LENGTH)        // full parent-path on disk
     private String location;
 
     @Column(name = "storage_in_bytes")                                     // Total storage of deposit directory
     private long storageInBytes;
 
     @Column(name = "deleted")                                              // deposit is deleted from inbox - archived
     private boolean deleted;
-    public String getDepositId() {
-        return depositId;
-    }
 
     public DepositProperties() {
     }
 
     public DepositProperties(String depositId, String depositor, String bagName, String depositState,
-        String description, OffsetDateTime depositCreationTimestamp, String location, long storageInBytes) {
+        String description, OffsetDateTime depositCreationTimestamp, String location, long storageInBytes, OffsetDateTime depositUpdateTimestamp) {
         this.depositId = depositId;
         this.depositor = depositor;
         this.bagName = bagName;
@@ -77,6 +76,11 @@ public DepositProperties(String depositId, String depositor, String bagName, Str
         this.depositCreationTimestamp = depositCreationTimestamp;
         this.location = location;
         this.storageInBytes = storageInBytes;
+        this.depositUpdateTimestamp = depositUpdateTimestamp;
+    }
+
+    public String getDepositId() {
+        return depositId;
     }
 
     public String getDepositor() {
@@ -169,4 +173,20 @@ public boolean equals(Object o) {
     public int hashCode() {
         return 31 * depositId.hashCode() + depositor.hashCode();
     }
-}
+
+    @Override
+    public String toString() {
+        return "DepositProperties{" +
+            "depositor='" + depositor + "'" +
+            ", depositId='" + depositId + '\'' +
+            ", bagName='" + bagName + '\'' +
+            ", depositState='" + depositState + '\'' +
+            ", depositCreationTimestamp=" + depositCreationTimestamp +
+            ", depositUpdateTimestamp=" + depositUpdateTimestamp +
+            ", description='" + description + '\'' +
+            ", location='" + location + '\'' +
+            ", storageInBytes=" + storageInBytes +
+            ", deleted=" + deleted +
+            '}';
+    }
+}
diff --git a/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositPropertiesAssembler.java b/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositPropertiesAssembler.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.DateTimeException;
 import java.time.OffsetDateTime;
 import java.util.Optional;
 
@@ -34,49 +35,46 @@ class DepositPropertiesAssembler {
     DepositPropertiesAssembler() {
     }
 
-    Optional<DepositProperties> assembleObject(File depositPropertiesFile, boolean  updateModificationDateTime) {
-
+    Optional<DepositProperties> assembleObject(File depositPropertiesFile, long CalculatedFolderSize) {
         Path depositPath = depositPropertiesFile.getParentFile().toPath();
         log.debug("assembleObject(depositPropertiesPath:Path): '{}'", depositPropertiesFile.getAbsolutePath());
-        DepositProperties dp; // = null
+        DepositProperties dp = null;
         Configuration configuration;
         try {
             configuration = DepositPropertiesFileReader.readDepositProperties(depositPropertiesFile);
+            String creationTime = configuration.getString("creation.timestamp");
 
             dp = new DepositProperties(depositPath.getFileName().toString(),
                 configuration.getString("depositor.userId", ""),
                 configuration.getString("bag-store.bag-name", ""),
                 configuration.getString("state.label", ""),
-                TextTruncation.stripEnd(configuration.getString("state.description", ""), TextTruncation.maxDescriptionLength),
-                OffsetDateTime.parse(configuration.getString("creation.timestamp", OffsetDateTime.now().toString())),
-                TextTruncation.stripBegin(depositPropertiesFile.getParentFile().getParentFile().getAbsolutePath(), TextTruncation.maxDirectoryLength),
-                calculateFolderSize(depositPath));
-
-            if (updateModificationDateTime) {
-                dp.setDepositUpdateTimestamp(OffsetDateTime.now());
-            }
-            else {
-                dp.setDepositUpdateTimestamp(dp.getDepositCreationTimestamp());
-            }
-
+                TextTruncation.stripEnd(configuration.getString("state.description", ""), TextTruncation.MAX_DESCRIPTION_LENGTH),
+                (creationTime == null || creationTime.isEmpty()) ? null : OffsetDateTime.parse(creationTime),
+                TextTruncation.stripBegin(depositPropertiesFile.getParentFile().getParentFile().getAbsolutePath(), TextTruncation.MAX_DIRECTORY_LENGTH),
+                CalculatedFolderSize == 0 ? calculateFolderSize(depositPath) : CalculatedFolderSize,
+                OffsetDateTime.now());
         }
-        catch (ConfigurationException e) {
-            log.error(e.getMessage(), e);
-            throw new RuntimeException(e);
+        catch (ConfigurationException | DateTimeException e) {
+            log.error("Error reading deposit.properties file: {}", e.getMessage());
         }
-        return Optional.of(dp);
+        catch (RuntimeException e) {
+            log.error("Error accessing deposit files : {}", e.getMessage());
+        }
+        return Optional.ofNullable(dp);
     }
 
     private long calculateFolderSize(Path path) {
-        long size;
-        try (var pathStream = Files.walk(path)) {
-            size = pathStream
-                .filter(p -> p.toFile().isFile())
-                .mapToLong(p -> p.toFile().length())
-                .sum();
-        }
-        catch (IOException e) {
-            throw new RuntimeException(e);
+        long size = 0;
+        if (Files.exists(path)) {
+            try (var pathStream = Files.walk(path)) {
+                size = pathStream
+                    .filter(p -> p.toFile().isFile())
+                    .mapToLong(p -> p.toFile().length())
+                    .sum();
+            }
+            catch (IOException e) {
+                throw new RuntimeException(e);
+            }
         }
 
         return size;

diff --git a/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositPropertiesFileReader.java b/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositPropertiesFileReader.java
@@ -26,7 +26,7 @@
 import java.io.File;
 
 class DepositPropertiesFileReader {
-    private static final Logger log = LoggerFactory.getLogger(DepositPropertiesAssembler.class);
+    private static final Logger log = LoggerFactory.getLogger(DepositPropertiesFileReader.class);
 
     static public Configuration readDepositProperties(File propertiesFile) throws ConfigurationException {
         log.debug("readDepositProperties: '{}'", propertiesFile.toString());