Improvement/make brat reader more forgiving take2

dkpro-core-api-io-asl/src/main/java/org/dkpro/core/api/io/JCasFileWriter_ImplBase.java

@@ -96,7 +96,7 @@ public abstract class JCasFileWriter_ImplBase
      */
     public static final String PARAM_STRIP_EXTENSION = "stripExtension";
     @ConfigurationParameter(name = PARAM_STRIP_EXTENSION, mandatory = true, defaultValue = "false")
-    private boolean stripExtension;
+    protected boolean stripExtension;
 
     /**
      * Use the document ID as file name even if a relative path information is present.
@@ -175,11 +175,12 @@ else if (singularTarget) {
             return getOutputStream((String) null, aExtension);
         }
         else {
-            return getOutputStream(getRelativePath(aJCas), aExtension);
+            String relPath = getRelativePath(aJCas);
+            return getOutputStream(relPath, aExtension);
         }
     }
 
-    protected String getTargetLocation()
+    public String getTargetLocation()
     {
         return targetLocation;
     }

dkpro-core-api-io-asl/src/main/java/org/dkpro/core/api/io/ResourceCollectionReaderBase.java

@@ -114,7 +114,7 @@ public abstract class ResourceCollectionReaderBase
      */
     public static final String PARAM_PATTERNS = ComponentParameters.PARAM_PATTERNS;
     @ConfigurationParameter(name = PARAM_PATTERNS, mandatory = false)
-    private String[] patterns;
+    protected String[] patterns;
 
     /**
      * Use the default excludes.
@@ -171,7 +171,7 @@ public void initialize(UimaContext aContext)
             throw new IllegalArgumentException(
                     "Either a source location, pattern, or both must be specified.");
         }
-
+                
         // if an ExternalResourceLocator providing a custom ResourcePatternResolver
         // has been specified, use it, by default use PathMatchingResourcePatternresolver
 
@@ -358,7 +358,7 @@ protected String getSourceLocation()
         return sourceLocation;
     }
 
-    protected boolean isSingleLocation()
+    public boolean isSingleLocation()
     {
         return patterns == null;
     }

dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/FileCopy.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020
+ * National Research Council of Canada
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.dkpro.core.api.resources;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+
+import org.apache.commons.io.FileUtils;
+
+public class FileCopy {
+
+    public static void copyFolder(File srcFolder, File destFolder) throws NoSuchFileException {
+        copyFolder(srcFolder.toPath(), destFolder.toPath());
+    }
+
+
+    public static void copyFolder(Path srcFolder, Path destFolder) throws NoSuchFileException {
+        final Path srcFolderAbs = srcFolder.toAbsolutePath();
+        Path destFolderAbs = destFolder.toAbsolutePath();
+        if (!srcFolder.toFile().exists()) {
+            throw new NoSuchFileException(srcFolder.toString());
+        }
+
+        try {
+            Files.walk(srcFolderAbs).forEach(s -> {
+                try {
+                    Path d = destFolder.resolve(srcFolderAbs.relativize(s));
+                    if (Files.isDirectory(s)) {
+                        if (!Files.exists(d)) {
+                            Files.createDirectory(d);
+                        }
+                        return;
+                    }
+                    Files.copy(s, d);
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
+                }
+            });
+        }
+        catch (Exception ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    public static void copyFileToFolder(Path srcFile, Path destFolder) throws IOException
+    {
+        if (Files.isDirectory(srcFile)) {
+            throw new IllegalArgumentException("Source file path " + srcFile + " is a directory");
+        }
+        if (!Files.isDirectory(destFolder)) {
+            throw new IllegalArgumentException(
+                    "Destination directory path " + destFolder + " is not a directory");
+        }
+
+        FileUtils.copyFileToDirectory(srcFile.toFile(), destFolder.toFile());
+    }
+}

dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/FileGlob.java

@@ -0,0 +1,197 @@
+/*
+ * Copyright 2019
+ * National Research Council of Canada
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.dkpro.core.api.resources;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.commons.io.FilenameUtils;
+
+public class FileGlob {
+
+    public static class FileDeleter implements Consumer<File> {
+
+        @Override
+        public void accept(File file) {
+            file.delete();
+        }
+
+    }
+
+    public static class FileGlobVisitor extends SimpleFileVisitor<Path> {
+        private PathMatcher matcher = null;
+        private List<File> visitedFiles = null;
+        private Consumer<File> action = null;
+
+        public FileGlobVisitor(String pattern) {
+            initFileGlobVisitor(pattern, null);
+        }
+
+        public FileGlobVisitor(String pattern, Consumer<File> _action) {
+            initFileGlobVisitor(pattern, _action);
+        }
+
+        private void initFileGlobVisitor(String pattern, Consumer<File> _action) {
+            visitedFiles = new ArrayList<File>();    
+            FileSystem fs = FileSystems.getDefault();
+            //Have to escape windows file separators since \\ is a glob escape character
+            matcher = fs.getPathMatcher("glob:" + pattern.replace("\\", "\\\\"));
+            action = _action;
+        }
+
+        @Override
+        public FileVisitResult visitFile(Path file, BasicFileAttributes attribs) {
+            Path fPath = file.toAbsolutePath();
+            if (matcher.matches(fPath)) {
+                visitedFiles.add(new File(file.toString()));
+                if (action != null) {
+                    action.accept(file.toFile());
+                }
+            }
+            return FileVisitResult.CONTINUE;
+        }
+
+        @Override
+        public FileVisitResult visitFileFailed(Path file, IOException io)
+        {   
+            return FileVisitResult.SKIP_SUBTREE;
+        }        
+
+        public File[] getFiles() {
+            File[] files = (File[]) visitedFiles.toArray(new File[visitedFiles.size()]);
+            return files;
+        }
+
+    }
+
+    public static File[] listFiles(String pattern)  { 
+        pattern = new File(pattern).getAbsolutePath();
+        Path startDir = Paths.get(getStartingDir(pattern));
+
+        File[] files = new File[0];
+        FileGlobVisitor matcherVisitor = new FileGlobVisitor(pattern);
+        try {
+            Files.walkFileTree(startDir, matcherVisitor);
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+
+        files = matcherVisitor.getFiles();
+
+        return files;
+    }
+
+
+    public static File[] listFiles(File rootDir, String[] patterns)  {
+        if (!rootDir.isDirectory()) {
+            throw new IllegalArgumentException("Root path was not a directory (was " + rootDir.toString() + ")");
+        }
+
+        for (int ii = 0; ii < patterns.length; ii++) {
+            patterns[ii] = FilenameUtils.concat(rootDir.toString(), patterns[ii]);
+        }
+        File[] matchingFiles = listFiles(patterns);
+
+        return matchingFiles;
+    }
+
+
+    public static File[] listFiles(String[] patterns)  {
+        Set<File> matchingFilesLst = new HashSet<File>();
+        for (String aPattern: patterns) {
+            File[] filesThisPattern = listFiles(aPattern);
+            for (File aFile: filesThisPattern) {
+                matchingFilesLst.add(aFile);
+            }
+        }
+
+        File[] matchingFilesArr = matchingFilesLst.toArray(new File[matchingFilesLst.size()]);
+        return matchingFilesArr;
+    }
+
+    public static void deleteFiles(String pattern) {
+        pattern = new File(pattern).getAbsolutePath();
+
+        Path startDir = Paths.get(getStartingDir(pattern));
+
+        File[] files = new File[0];
+        FileGlobVisitor matcherVisitor = new FileGlobVisitor(pattern, new FileDeleter());
+        try {
+            Files.walkFileTree(startDir, matcherVisitor);
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+
+        files = matcherVisitor.getFiles();
+    }
+
+    public static void main(String[] args)  {
+        String pattern = args[0];
+        System.out.println("Files matching: " + pattern);
+        File[] files = FileGlob.listFiles(pattern);
+        if (files.length == 0) {
+            System.out.println("No match found");
+        }
+        for (File aFile: files) {
+            System.out.println(aFile.getAbsolutePath());
+        }
+    }
+
+    protected static String getStartingDir(String pattern) {
+        String startingDir = truncatePatternToFirstWildcard(pattern);
+        if (!endsWithFileSeparator(startingDir)) {
+            File parentDir = Paths.get(startingDir).toFile().getParentFile();            
+            if (parentDir != null) {
+                startingDir = parentDir.toString();
+            }
+        }
+
+        return startingDir;
+    }
+
+    private static boolean endsWithFileSeparator(String path) {
+        //Non-Windows OS
+        if (!System.getProperty("os.name").toLowerCase().startsWith("win")) {
+            return path.endsWith(File.separator);
+        } else {
+            return path.endsWith(File.separator) | path.endsWith("/");
+        }
+    }
+
+    private static String truncatePatternToFirstWildcard(String pattern) {
+        pattern = pattern.replaceFirst("[\\*\\?].*$", "");
+
+        return pattern;
+    }    
+}

dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/BinaryCasWriterReaderTest.java

@@ -93,8 +93,9 @@ public class BinaryCasWriterReaderTest
     private File testFolder;
 
     @Before
-    public void setup() 
+    public void setup() throws IOException 
     {
+        DkproTestContext.get().initializeTestWorkspace();
         testFolder = testContext.getTestOutputFolder();
     }
 
@@ -363,11 +364,11 @@ BinaryCasWriter.PARAM_TARGET_LOCATION, new File(testFolder, "out.bin"),
         assertEquals("out.bin", getFeature(dmd, "documentTitle", String.class));
         assertEquals("out.bin", getFeature(dmd, "documentId", String.class));
         assertTrue(separatorsToUnix(getFeature(dmd, "documentUri", String.class))
-                .endsWith("/target/test-output/BinaryCasWriterReaderTest-testReadingFileWithoutDocumentMetaData/out.bin"));
+                .endsWith("test-workspaces/BinaryCasWriterReaderTest-testReadingFileWithoutDocumentMetaData/output/out.bin"));
         assertTrue(separatorsToUnix(getFeature(dmd, "collectionId", String.class))
-                .endsWith("/target/test-output/BinaryCasWriterReaderTest-testReadingFileWithoutDocumentMetaData/"));
+                .endsWith("test-workspaces/BinaryCasWriterReaderTest-testReadingFileWithoutDocumentMetaData/output/"));
         assertTrue(separatorsToUnix(getFeature(dmd, "documentBaseUri", String.class))
-                .endsWith("/target/test-output/BinaryCasWriterReaderTest-testReadingFileWithoutDocumentMetaData/"));
+                .endsWith("test-workspaces/BinaryCasWriterReaderTest-testReadingFileWithoutDocumentMetaData/output/"));
         assertEquals(false, getFeature(dmd, "isLastSegment", Boolean.class));
     }
 
@@ -401,11 +402,11 @@ BinaryCasWriter.PARAM_TARGET_LOCATION, new File(testFolder, "out.bin"),
         assertEquals("out.bin", getFeature(dmd, "documentTitle", String.class));
         assertEquals("out.bin", getFeature(dmd, "documentId", String.class));
         assertTrue(separatorsToUnix(getFeature(dmd, "documentUri", String.class))
-                .endsWith("/target/test-output/BinaryCasWriterReaderTest-testReadingFileOverridingDocumentMetaData/out.bin"));
+                .endsWith("test-workspaces/BinaryCasWriterReaderTest-testReadingFileOverridingDocumentMetaData/output/out.bin"));
         assertTrue(separatorsToUnix(getFeature(dmd, "collectionId", String.class))
-                .endsWith("/target/test-output/BinaryCasWriterReaderTest-testReadingFileOverridingDocumentMetaData/"));
+                .endsWith("test-workspaces/BinaryCasWriterReaderTest-testReadingFileOverridingDocumentMetaData/output/"));
         assertTrue(separatorsToUnix(getFeature(dmd, "documentBaseUri", String.class))
-                .endsWith("/target/test-output/BinaryCasWriterReaderTest-testReadingFileOverridingDocumentMetaData/"));
+                .endsWith("test-workspaces/BinaryCasWriterReaderTest-testReadingFileOverridingDocumentMetaData/output/"));
         assertEquals(false, getFeature(dmd, "isLastSegment", Boolean.class));
     }
 

dkpro-core-io-bnc-asl/pom.xml

@@ -40,6 +40,11 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>    
     <dependency>
       <groupId>org.dkpro.core</groupId>
       <artifactId>dkpro-core-api-io-asl</artifactId>