Merge pull request #169 from aodn/features/6008-reduce-polygon-decimal

Add reducer support

indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java

@@ -26,9 +26,13 @@ public class IndexerConfig {
     @Value("${app.geometry.coastalPrecision:0.5}")
     protected double coastalPrecision;
 
+    @Value("${app.geometry.reducerPrecision:#{null}}")
+    protected Double reducerPrecision;
+
     @PostConstruct
     public void init() {
         GeometryUtils.setCoastalPrecision(coastalPrecision);
+        GeometryUtils.setReducerPrecision(reducerPrecision);
         GeometryUtils.init();
     }
 

indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java

@@ -0,0 +1,32 @@
+package au.org.aodn.esindexer.utils;
+
+import org.springframework.core.io.ClassPathResource;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class FileUtils {
+    public static File saveResourceToTemp(String resourceName, String filename) {
+        String tempDir = System.getProperty("java.io.tmpdir");
+        ClassPathResource resource = new ClassPathResource(resourceName);
+
+        File tempFile = new File(tempDir, filename);
+        try(InputStream input = resource.getInputStream()) {
+            tempFile.deleteOnExit();  // Ensure the file is deleted when the JVM exits
+
+            // Write the InputStream to the temporary file
+            try (FileOutputStream outputStream = new FileOutputStream(tempFile)) {
+                byte[] buffer = new byte[1024];
+                int bytesRead;
+                while ((bytesRead = input.read(buffer)) != -1) {
+                    outputStream.write(buffer, 0, bytesRead);
+                }
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return tempFile;
+    }
+}

indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java

@@ -8,6 +8,7 @@
 import org.apache.logging.log4j.Logger;
 import org.geotools.data.shapefile.ShapefileDataStore;
 import org.geotools.data.shapefile.ShapefileDataStoreFactory;
+import org.locationtech.jts.precision.GeometryPrecisionReducer;
 import org.geotools.data.simple.SimpleFeatureCollection;
 import org.geotools.data.simple.SimpleFeatureIterator;
 import org.geotools.data.simple.SimpleFeatureSource;
@@ -17,7 +18,6 @@
 import org.locationtech.jts.operation.union.UnaryUnionOp;
 import org.locationtech.jts.simplify.DouglasPeuckerSimplifier;
 import org.opengis.feature.simple.SimpleFeature;
-import org.springframework.core.io.ClassPathResource;
 
 import java.io.*;
 import java.net.URL;
@@ -45,13 +45,24 @@ public enum PointOrientation {
     @Getter
     @Setter
     protected static double coastalPrecision = 0.1;
+    // A value based on trial and error, to simplify the polygon to avoid complex geojson that takes time to transfer
+    // By default is set to null to disable it so test case do not need to change, but each env have this value set
+    // to reduce processing time.
+    @Getter
+    @Setter
+    protected static Double reducerPrecision = null;
+
+    protected static GeometryPrecisionReducer reducer = null;
 
     // Load a coastline shape file so that we can get a spatial extents that cover sea only
-    public static void init() {
+    public static synchronized void init() {
         try {
+            // Reset reducer
+            reducer = null;
+
             // shp file depends on shx, so need to have shx appear in temp folder.
-            saveResourceToTemp("land/ne_10m_land.shx", "shapefile.shx");
-            File tempFile = saveResourceToTemp("land/ne_10m_land.shp", "shapefile.shp");
+            FileUtils.saveResourceToTemp("land/ne_10m_land.shx", "shapefile.shx");
+            File tempFile = FileUtils.saveResourceToTemp("land/ne_10m_land.shp", "shapefile.shp");
 
             // Load the shapefile from the temporary file using ShapefileDataStore
             URL tempFileUrl = tempFile.toURI().toURL();
@@ -63,6 +74,11 @@ public static void init() {
             SimpleFeatureCollection featureCollection = featureSource.getFeatures();
             List<Geometry> geometries = new ArrayList<>();
 
+            if(getReducerPrecision() != null) {
+                PrecisionModel pm = new PrecisionModel(getReducerPrecision()); // 1 / 1000 meters ~= 1km
+                reducer = new GeometryPrecisionReducer(pm);
+            }
+
             try (SimpleFeatureIterator iterator = featureCollection.features()) {
                 while (iterator.hasNext()) {
                     SimpleFeature feature = iterator.next();
@@ -75,7 +91,7 @@ public static void init() {
                     Geometry simplifiedGeometry = DouglasPeuckerSimplifier
                             .simplify(landFeatureGeometry, getCoastalPrecision()); // Adjust tolerance
 
-                    geometries.add(simplifiedGeometry);
+                    geometries.add(reducer != null ? reducer.reduce(simplifiedGeometry) : simplifiedGeometry);
                 }
             }
             // Faster to use union list rather than union by geometry one by one.
@@ -85,28 +101,6 @@ public static void init() {
             throw new RuntimeException(ioe);
         }
     }
-
-    protected static File saveResourceToTemp(String resourceName, String filename) {
-        String tempDir = System.getProperty("java.io.tmpdir");
-        ClassPathResource resource = new ClassPathResource(resourceName);
-
-        File tempFile = new File(tempDir, filename);
-        try(InputStream input = resource.getInputStream()) {
-            tempFile.deleteOnExit();  // Ensure the file is deleted when the JVM exits
-
-            // Write the InputStream to the temporary file
-            try (FileOutputStream outputStream = new FileOutputStream(tempFile)) {
-                byte[] buffer = new byte[1024];
-                int bytesRead;
-                while ((bytesRead = input.read(buffer)) != -1) {
-                    outputStream.write(buffer, 0, bytesRead);
-                }
-            }
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return tempFile;
-    }
     /**
      * @param polygons - Assume to be EPSG:4326, as GeoJson always use this encoding.
      * @return
@@ -252,6 +246,7 @@ protected static List<List<Geometry>> removeLandAreaFromGeometry(List<List<Geome
                             // it fixed the non-noded intersection issue
                             .map(geometry -> geometry.isValid() ? geometry : geometry.buffer(0))
                             .map(geometry -> geometry.difference(landGeometry))
+                            .map(geometry -> reducer != null ? reducer.reduce(geometry) : geometry)
                             .map(GeometryUtils::convertToListGeometry)
                             .flatMap(Collection::stream)
                             .toList()
@@ -347,9 +342,10 @@ protected static List<List<Geometry>> createGeometryWithoutLand(List<List<Abstra
     public static Map<?, ?> createGeometryFrom(List<List<AbstractEXGeographicExtentType>> rawInput, Integer gridSize) {
         // The return polygon is in EPSG:4326, so we can call createGeoJson directly
 
-        // This line will cause the spatial extents to break into grid, it may help to debug but will make production
-        // slow and sometimes cause polygon break.
-        // List<List<Geometry>> polygonNoLand = splitAreaToGrid(createGeometryWithoutLand(rawInput));
+        // Un-remark this line and remark the line below if you want to visualize the polygon on map, change this
+        // line will cause the spatial extends draw on map with land removed.
+        // List<List<Geometry>> polygon = createGeometryWithoutLand(rawInput);
+
         List<List<Geometry>> polygon = GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput);
         return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null;
     }

indexer/src/main/resources/application-dev.yaml

@@ -4,6 +4,7 @@ server:
 app:
   geometry:
     enableGridSpatialExtents: true
+    reducerPrecision: 4.0
 
 elasticsearch:
   index:

indexer/src/main/resources/application-edge.yaml

@@ -7,6 +7,7 @@ spring:
 app:
   geometry:
     enableGridSpatialExtents: true
+    reducerPrecision: 4.0
 
 management:
   endpoints:

indexer/src/main/resources/application-production.yaml

@@ -1,3 +1,7 @@
+app:
+  geometry:
+    reducerPrecision: 4.0
+
 management:
   endpoints:
     web:

indexer/src/main/resources/application-staging.yaml

@@ -1,3 +1,7 @@
+app:
+  geometry:
+    reducerPrecision: 4.0
+
 management:
   endpoints:
     web:

indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java

@@ -26,11 +26,13 @@ public GeometryUtilsTest() throws JAXBException {
     @BeforeEach
     public void init() {
         GeometryUtils.setCoastalPrecision(0.03);
-        GeometryUtils.init();
     }
 
     @Test
     public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBException {
+        GeometryUtils.setReducerPrecision(null);
+        GeometryUtils.init();
+
         String xml = readResourceFile("classpath:canned/sample_complex_area.xml");
         MDMetadataType source = jaxb.unmarshal(xml);
         // Whole spatial extends
@@ -87,4 +89,51 @@ public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBExcep
         assertEquals(126.0, ncoors[3].getX(), 0.01);
         assertEquals(-35.9999, ncoors[3].getY(), 0.01);
     }
+    /**
+     * This test turn on the reducer to further reduce the complexity of the land area after
+     * DouglasPeuckerSimplifier simplifier, this further reduce the number of digit to get a smaller geojson
+     * which should improve transfer speed.
+     *
+     * @throws IOException - Not expect to throw
+     * @throws JAXBException - Not expect to throw
+     */
+    @Test
+    public void verifyLandStrippedFromSpatialExtentsWithReducerOn() throws IOException, JAXBException {
+        GeometryUtils.setReducerPrecision(4.0);
+        GeometryUtils.init();
+
+        String xml = readResourceFile("classpath:canned/sample_complex_area.xml");
+        MDMetadataType source = jaxb.unmarshal(xml);
+
+        // Strip the land away.
+        List<List<Geometry>> noLand = GeometryUtils.createGeometryItems(
+                source,
+                (rawInput, s) -> GeometryUtils.createGeometryWithoutLand(rawInput),
+                null
+        );
+
+        List<List<Geometry>> nl = Objects.requireNonNull(noLand);
+
+        assertEquals(nl.size(),1, "No Land have 1 polygon array");
+        assertEquals(16, nl.get(0).size(), "Size 16 with land");
+
+        Geometry nle = nl.get(0).get(0).getEnvelope();
+        Coordinate[] ncoors = nle.getCoordinates();
+
+        // The envelope of the two polygon should match given one is the original and the other just strip the land
+        assertEquals(118.0, ncoors[0].getX(), 0.00);
+        assertEquals(-36.0, ncoors[0].getY(), 0.00);
+
+        assertEquals(118.0, ncoors[1].getX(), 0.00);
+        assertEquals(-32.25, ncoors[1].getY(), 0.00);
+
+        assertEquals(126, ncoors[2].getX(), 0.00);
+        assertEquals(-32.25, ncoors[2].getY(), 0.00);
+
+        assertEquals(126.0, ncoors[3].getX(), 0.00);
+        assertEquals(-36.0, ncoors[3].getY(), 0.00);
+
+        assertEquals(118.0, ncoors[4].getX(), 0.00);
+        assertEquals(-36.0, ncoors[4].getY(), 0.00);
+    }
 }