Merge pull request #382 from nicolas-f/sql_compression

Sql compression
Universite-Gustave-Eiffel · Sep 20, 2021 · b2f9dcb · b2f9dcb
2 parents e0f0f6d + 043b78a
commit b2f9dcb
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 16 deletions.
diff --git a/noisemodelling-jdbc/src/main/java/org/noise_planet/noisemodelling/jdbc/BezierContouring.java b/noisemodelling-jdbc/src/main/java/org/noise_planet/noisemodelling/jdbc/BezierContouring.java
@@ -30,6 +30,8 @@
 import org.locationtech.jts.operation.union.CascadedPolygonUnion;
 import org.locationtech.jts.simplify.TopologyPreservingSimplifier;
 import org.noise_planet.noisemodelling.pathfinder.ComputeRays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.sql.*;
 import java.text.DecimalFormat;
@@ -40,6 +42,7 @@
  * @author  Nicolas Fortin, Université Gustave Eiffel
  */
 public class BezierContouring {
+    Logger log = LoggerFactory.getLogger(BezierContouring.class);
     static final int BATCH_MAX_SIZE = 500;
     String pointTable = "LDEN_RESULT";
     String triangleTable = "TRIANGLES";
@@ -406,9 +409,14 @@ void processCell(Connection connection, int cellId, Map<Short, ArrayList<Geometr
                 ArrayList<Polygon> polygons = new ArrayList<>();
                 if(!smooth) {
                     // Merge triangles
-                    CascadedPolygonUnion union = new CascadedPolygonUnion(entry.getValue());
-                    Geometry mergeTriangles = union.union();
-                    explode(mergeTriangles, polygons);
+                    try {
+                        CascadedPolygonUnion union = new CascadedPolygonUnion(entry.getValue());
+                        Geometry mergeTriangles = union.union();
+                        explode(mergeTriangles, polygons);
+                    } catch (TopologyException t) {
+                        log.warn(t.getLocalizedMessage(), t);
+                        explode(factory.createGeometryCollection(entry.getValue().toArray(new Geometry[0])), polygons);
+                    }
                 } else {
                     explode(factory.createGeometryCollection(entry.getValue().toArray(new Geometry[0])), polygons);
                 }

diff --git a/noisemodelling-jdbc/src/main/java/org/noise_planet/noisemodelling/jdbc/LDENConfig.java b/noisemodelling-jdbc/src/main/java/org/noise_planet/noisemodelling/jdbc/LDENConfig.java
@@ -74,6 +74,7 @@ public void setComputeLAEQOnly(boolean computeLAEQOnly) {
     String lwFrequencyPrepend = "LW";
 
     File sqlOutputFile;
+    Boolean sqlOutputFileCompression = true;
     Boolean dropResultsTable = true;
 
     public LDENConfig(INPUT_MODE input_mode) {
@@ -106,6 +107,14 @@ public Boolean getDropResultsTable() {
         return dropResultsTable;
     }
 
+    public Boolean getSqlOutputFileCompression() {
+        return sqlOutputFileCompression;
+    }
+
+    public void setSqlOutputFileCompression(Boolean sqlOutputFileCompression) {
+        this.sqlOutputFileCompression = sqlOutputFileCompression;
+    }
+
     /**
      * @param dropResultsTable Drop previous results tables before inserting results
      */

diff --git a/...ing-jdbc/src/main/java/org/noise_planet/noisemodelling/jdbc/LDENPointNoiseMapFactory.java b/...ing-jdbc/src/main/java/org/noise_planet/noisemodelling/jdbc/LDENPointNoiseMapFactory.java
@@ -33,16 +33,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
+import java.io.*;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.*;
 import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.zip.GZIPOutputStream;
 
 /**
  *
@@ -53,6 +51,7 @@ public class LDENPointNoiseMapFactory implements PointNoiseMap.PropagationProces
     Thread tableWriterThread;
     Connection connection;
     static final int BATCH_MAX_SIZE = 500;
+    static final int WRITER_CACHE = 65536;
     LDENComputeRaysOut.LdenData ldenData = new LDENComputeRaysOut.LdenData();
     /**
      * Attenuation and other attributes relative to direction on sphere
@@ -234,7 +233,7 @@ private static class TableWriter implements Runnable {
         LDENComputeRaysOut.LdenData ldenData;
         double[] a_weighting;
         boolean started = false;
-        BufferedWriter o;
+        Writer o;
 
         public TableWriter(Connection connection, LDENConfig ldenConfig, LDENComputeRaysOut.LdenData ldenData) {
             this.connection = connection;
@@ -484,6 +483,14 @@ void createKeys()  throws SQLException, IOException {
             }
         }
 
+        OutputStreamWriter getStream() throws IOException {
+            if(ldenConfig.sqlOutputFileCompression) {
+                return new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(sqlFilePath), WRITER_CACHE));
+            } else {
+                return new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(sqlFilePath), WRITER_CACHE));
+            }
+        }
+
         @Override
         public void run() {
             // Drop and create tables
@@ -501,7 +508,7 @@ public void run() {
                     ldenConfig.aborted = true;
                 }
             } else {
-                try(BufferedWriter bw = new BufferedWriter(new FileWriter(sqlFilePath))) {
+                try(OutputStreamWriter bw = getStream()) {
                     o = bw;
                     init();
                     mainLoop();

diff --git a/...bc/src/main/java/org/noise_planet/noisemodelling/jdbc/utils/StringPreparedStatements.java b/...bc/src/main/java/org/noise_planet/noisemodelling/jdbc/utils/StringPreparedStatements.java
@@ -3,10 +3,7 @@
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.io.WKTWriter;
 
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Reader;
+import java.io.*;
 import java.math.BigDecimal;
 import java.net.URL;
 import java.sql.*;
@@ -18,12 +15,12 @@
  * This class implements PreparedStatement in order to output to a File instead of a connection
  */
 public class StringPreparedStatements implements PreparedStatement {
-    BufferedWriter w;
+    Writer w;
     String query;
     Map<Integer, Object> parameters = new HashMap<>();
     WKTWriter wktWriter = new WKTWriter(3);
 
-    public StringPreparedStatements(BufferedWriter w, String query) {
+    public StringPreparedStatements(Writer w, String query) {
         this.w = w;
         this.query = query;
     }

diff --git a/...ling-pathfinder/src/main/java/org/noise_planet/noisemodelling/pathfinder/ComputeRays.java b/...ling-pathfinder/src/main/java/org/noise_planet/noisemodelling/pathfinder/ComputeRays.java
@@ -49,7 +49,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;