replaced ZipInputStream in CreateNewDataFilesCommand

src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java

@@ -100,71 +100,48 @@ public IngestServiceShapefileHelper(File zippedShapefile, File rezipFolder){
         //this.processFile(zippedShapefile, rezipFolder);
 
     }
-    
-   private FileInputStream getFileInputStream(File fileObject){
-       if (fileObject==null){
-           return null;
-       }
-       try {
+
+    private FileInputStream getFileInputStream(File fileObject){
+        if (fileObject==null){
+            return null;
+        }
+        try {
             return new FileInputStream(fileObject);
         } catch (FileNotFoundException ex) {
             logger.severe("Failed to create FileInputStream from File: " + fileObject.getAbsolutePath());
             return null;
         }
-   }
-   
-   private void closeFileInputStream(FileInputStream fis){
-       if (fis==null){
-           return;
-       }
+    }
+
+    private void closeFileInputStream(FileInputStream fis){
+        if (fis==null){
+            return;
+        }
         try {
-            fis.close();            
+            fis.close();
         } catch (IOException ex) {
             logger.info("Failed to close FileInputStream");
         }
-   }
-   
+    }
+
     public boolean processFile() {
 
        if ((!isValidFile(this.zippedShapefile))||(!isValidFolder(this.rezipFolder))){
             return false;
         }
-
-       // (1) Use the ShapefileHandler to the .zip for a shapefile
-       //
-        FileInputStream shpfileInputStream = this.getFileInputStream(zippedShapefile);
-        if (shpfileInputStream==null){
-            return false;
-        }
-
-        this.shpHandler = new ShapefileHandler(shpfileInputStream);
-        if (!shpHandler.containsShapefile()){
-            logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
-            return false;
-        }
-
-        this.closeFileInputStream(shpfileInputStream);
-
-        //  (2) Rezip the shapefile pieces
-        logger.info("rezipFolder: " + rezipFolder.getAbsolutePath());
-        shpfileInputStream = this.getFileInputStream(zippedShapefile);
-        if (shpfileInputStream==null){
-            return false;
-        }
-
-        boolean rezipSuccess;
         try {
-            rezipSuccess = shpHandler.rezipShapefileSets(shpfileInputStream, rezipFolder);
+            this.shpHandler = new ShapefileHandler(zippedShapefile);
+            if (!shpHandler.containsShapefile()){
+                logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
+                return false;
+            }
+            logger.info("rezipFolder: " + rezipFolder.getAbsolutePath());
+            return shpHandler.rezipShapefileSets(rezipFolder);
         } catch (IOException ex) {
             logger.severe("Shapefile was not correctly unpacked/repacked");
             logger.severe("shpHandler message: " + shpHandler.errorMessage);
             return false;
         }
-
-        this.closeFileInputStream(shpfileInputStream);
-
-        return rezipSuccess;
-
         //   return createDataFiles(rezipFolder);
 
     }

src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java

@@ -510,15 +510,18 @@ public static String determineFileType(File f, String fileName) throws IOExcepti
             // Check for shapefile extensions as described here: http://en.wikipedia.org/wiki/Shapefile
             //logger.info("Checking for shapefile");
 
-            ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(f));
+            ShapefileHandler shp_handler = new ShapefileHandler(f);
              if (shp_handler.containsShapefile()){
               //  logger.info("------- shapefile FOUND ----------");
                  fileType = ShapefileHandler.SHAPEFILE_FILE_TYPE; //"application/zipped-shapefile";
              }
-
-            Optional<BagItFileHandler> bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
-             if(bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
-                 fileType = BagItFileHandler.FILE_TYPE;
+             try {
+                 Optional<BagItFileHandler> bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
+                 if (bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
+                     fileType = BagItFileHandler.FILE_TYPE;
+                 }
+             } catch (Exception e) {
+                 logger.warning("Error checking for BagIt package: " + e.getMessage());
              }
         } 
 

src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java

@@ -1,22 +1,20 @@
 package edu.harvard.iq.dataverse.util;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.Date;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
 import java.util.HashMap;
 import java.util.*;
 
 import java.nio.file.Files;
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
-import java.util.logging.Level;
+
 import java.util.logging.Logger;
 import org.apache.commons.io.FileUtils;
 
@@ -42,11 +40,10 @@
  *                                  "shape1.pdf", "README.md", "shape_notes.txt"
  * 
  *  Code Example:  
- *          FileInputStream shp_file_input_stream = new FileInputStream(new File("zipped_shapefile.zip"))
- *          ShapefileHandler shp_handler = new ShapefileHandler(shp_file_input_stream);
+ *          ShapefileHandler shp_handler = new ShapefileHandler(new File("zipped_shapefile.zip"));
  *          if (shp_handler.containsShapefile()){
  *              File rezip_folder = new File("~/folder_for_rezipping");
- *              boolean rezip_success = shp_handler.rezipShapefileSets(shp_file_input_stream, rezip_folder );
+ *              boolean rezip_success = shp_handler.rezipShapefileSets(rezip_folder );
  *              if (!rezip_success){
  *                  // rezip failed, should be an error message (String) available
                     System.out.println(shp_handler.error_message);
@@ -73,7 +70,8 @@ public class ShapefileHandler{
     public final static String SHP_XML_EXTENSION = "shp.xml";
     public final static String BLANK_EXTENSION = "__PLACEHOLDER-FOR-BLANK-EXTENSION__";
     public final static List<String> SHAPEFILE_ALL_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj", "sbn", "sbx", "fbn", "fbx", "ain", "aih", "ixs", "mxs", "atx", "cpg", "qpj", "qmd", SHP_XML_EXTENSION);
-
+    private final File zipFile;
+
     public boolean DEBUG = false;
 
     private boolean zipFileProcessed = false;
@@ -97,9 +95,6 @@ public class ShapefileHandler{
 
     private List<File> finalRezippedFiles = new ArrayList<>();
 
-    private String outputFolder = "unzipped";
-    private String rezippedFolder = "rezipped";
-
     // Debug helper
     private void msg(String s){
         //logger.info(s);
@@ -115,40 +110,28 @@ private void msgt(String s){
     }
 
     /*
-        Constructor, start with filename
-    */
-    public ShapefileHandler(String filename){
-
-        if (filename==null){
-            this.addErrorMessage("The filename was null");
-            return;
-        }
-
-        FileInputStream zip_file_stream;
-        try {
-            zip_file_stream = new FileInputStream(new File(filename));
-        } catch (FileNotFoundException ex) {
-            this.addErrorMessage("The file was not found");
+         Constructor, start with File
+     */
+    public ShapefileHandler(File zip_file) throws IOException {
+        zipFile = zip_file;
+        if (zip_file == null) {
+            this.addErrorMessage("The file was null");
             return;
         }
-
-       this.examineZipfile(zip_file_stream);
-
-    }
-
-
-   /*
-        Constructor, start with FileInputStream
-    */
-   public ShapefileHandler(FileInputStream zip_file_stream){
 
-        if (zip_file_stream==null){
-            this.addErrorMessage("The zip_file_stream was null");
-            return;
+        try (var zip_file_object = new ZipFile(zip_file)) {
+            this.examineZipfile(zip_file_object);
+        }
+        catch (FileNotFoundException ex) {
+            // While this constructor had a FileInputStream as argument:
+            //  FileUtil.determineFileType threw this exception before calling the constructor with a FileInputStream
+            //  IngestServiceShapefileHelper.processFile won´t call this constructor if the file is not valid hence does not exist.
+            //  When the file would have disappeared in the meantime, it would have produced a slightly different error message.
+            logger.severe("File not found: " + zip_file.getAbsolutePath());
+            throw ex;
         }
-        this.examineZipfile(zip_file_stream);
     }
-    
+
     public List<File> getFinalRezippedFiles(){
         return this.finalRezippedFiles;
     }
@@ -290,26 +273,19 @@ inside the uploaded zip file (issue #6873). To achieve this, we recreate
         subfolders in the FileMetadata of the newly created DataFiles. 
         (-- L.A. 09/2020) 
     */
-    private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File target_directory){
+    private boolean unzipFilesToDirectory(ZipFile zipfileInput, File target_directory){
         logger.fine("unzipFilesToDirectory: " + target_directory.getAbsolutePath() );
 
-        if (zipfile_input_stream== null){
-            this.addErrorMessage("unzipFilesToDirectory. The zipfile_input_stream is null.");
-            return false;
-        }
         if (!target_directory.isDirectory()){
              this.addErrorMessage("This directory does not exist: " + target_directory.getAbsolutePath());
             return false;
         }
 
-       List<String> unzippedFileNames = new ArrayList<>(); 
+       List<String> unzippedFileNames = new ArrayList<>();
 
-       ZipInputStream zipStream = new ZipInputStream(zipfile_input_stream);
 
-       ZipEntry origEntry;
-        byte[] buffer = new byte[2048];
         try {
-            while((origEntry = zipStream.getNextEntry())!=null){
+            for(var origEntry : Collections.list(zipfileInput.entries())){
 
                 String zentryFileName = origEntry.getName();
                 logger.fine("\nOriginal entry name: " + origEntry);
@@ -359,15 +335,10 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File
                     unzippedFileNames.add(outpath);
                 }
                 logger.fine("Write zip file: " + outpath);
-                FileOutputStream fileOutputStream;
-                long fsize = 0;
-                fileOutputStream = new FileOutputStream(outpath);
-                int len;// = 0;
-                while ((len = zipStream.read(buffer)) > 0){
-                    fileOutputStream.write(buffer, 0, len);
-                    fsize+=len;
-                } // end while
-                fileOutputStream.close();
+                try(var inputStream = zipfileInput.getInputStream(origEntry)) {
+                    Files.createDirectories(new File(outpath).getParentFile().toPath());
+                    Files.copy(inputStream, Path.of(outpath), StandardCopyOption.REPLACE_EXISTING);
+                }
             } // end outer while
         } catch (IOException ex) {
             for (StackTraceElement el : ex.getStackTrace()){
@@ -376,19 +347,13 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File
             this.addErrorMessage("Failed to open ZipInputStream entry" +  ex.getMessage());
             return false;
         }
-
-        try {
-             zipStream.close();
-        } catch (IOException ex) {
-             Logger.getLogger(ShapefileHandler.class.getName()).log(Level.SEVERE, null, ex);
-        }
-    return true;
+        return true;
     }
     /*
         Rezip the shapefile(s) into a given directory
         Assumes that the zipfile_input_stream has already been checked!
     */
-    public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rezippedFolder) throws IOException{
+    public boolean rezipShapefileSets(File rezippedFolder) throws IOException{
         logger.fine("rezipShapefileSets");
         //msgt("rezipShapefileSets");
         if (!this.zipFileProcessed){
@@ -399,10 +364,6 @@ public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rez
              this.addErrorMessage("There are no shapefiles here!");
             return false;
         }
-        if (zipfile_input_stream== null){
-            this.addErrorMessage("The zipfile_input_stream is null.");
-            return false;
-        }
         if (rezippedFolder == null){
             this.addErrorMessage("The rezippedFolder is null.");
             return false;
@@ -432,9 +393,11 @@ public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rez
 
 
         // Unzip files!
-        if (!this.unzipFilesToDirectory(zipfile_input_stream, dir_for_unzipping)){
-            this.addErrorMessage("Failed to unzip files.");
-            return false;
+        try(var zipfileObject = new ZipFile(zipFile)) {
+            if (!this.unzipFilesToDirectory(zipfileObject, dir_for_unzipping)) {
+                this.addErrorMessage("Failed to unzip files.");
+                return false;
+            }
         }
         // Redistribute files!
         String target_dirname = rezippedFolder.getAbsolutePath();
@@ -680,27 +643,17 @@ private boolean isFileToSkip(String fname){
     /**************************************
      * Iterate through the zip file contents.
      * Does it contain any shapefiles?
-     *
-     * @param FileInputStream zip_file_stream
      */
-    private boolean examineZipfile(FileInputStream zip_file_stream){
+    private boolean examineZipfile(ZipFile zip_file){
        // msgt("examineZipfile");
 
-        if (zip_file_stream==null){
-               this.addErrorMessage("The zip file stream was null");
-               return false;
-           }
-
        // Clear out file lists
        this.filesListInDir.clear();
        this.filesizeHash.clear();
        this.fileGroups.clear();
 
        try{
-            ZipInputStream zipStream = new ZipInputStream(zip_file_stream);
-            ZipEntry entry;
-
-            while((entry = zipStream.getNextEntry())!=null){
+           for(var entry : Collections.list(zip_file.entries())){
 
                  String zentryFileName = entry.getName();
                  //msg("zip entry: " + entry.getName());
@@ -738,8 +691,6 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
                     this.filesizeHash.put(unzipFilePath, entry.getSize());
                 }
            } // end while
-
-           zipStream.close();
 
            if (this.filesListInDir.isEmpty()){
                errorMessage = "No files in zipStream";
@@ -749,23 +700,11 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
            this.zipFileProcessed = true;
            return true;
 
-       }catch(ZipException ex){
-               this.addErrorMessage("ZipException");
-               msgt("ZipException");
-               return false;
-
-       }catch(IOException ex){
-           //ex.printStackTrace(); 
-           this.addErrorMessage("IOException File name");
-           msgt("IOException");
-           return false;
        }catch(IllegalArgumentException ex){
            this.addErrorMessage("IllegalArgumentException when parsing zipfile");
            msgt("IllegalArgumentException when parsing zipfile");
            return false;
 
-       }finally{
-
        }
 
    } // end examineFile