From cc1ea159f6ac58b671d5f06f91014afbfab77e13 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Thu, 21 Nov 2024 17:22:28 +1100 Subject: [PATCH 1/4] Fix bug in centroid point calcuation --- pom.xml | 5 + server/pom.xml | 4 + .../ogcapi/server/core/mapper/Converter.java | 1 + .../server/core/mapper/StacToCollections.java | 2 +- .../core/model/enumeration/CQLCrsType.java | 2 - .../core/parser/stac/GeometryVisitor.java | 10 +- .../core/parser/stac/IntersectsImpl.java | 9 +- .../server/core/util/GeometryUtils.java | 14 +- .../server/core/parser/stac/ParserTest.java | 42 +- .../0015db7e-e684-7548-e053-08114f8cd4ad.json | 665 ++++++++++++++++++ 10 files changed, 737 insertions(+), 17 deletions(-) create mode 100644 server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json diff --git a/pom.xml b/pom.xml index bfab632f..492769ee 100644 --- a/pom.xml +++ b/pom.xml @@ -132,6 +132,11 @@ mockserver-client-java 5.15.0 + + org.locationtech.spatial4j + spatial4j + 0.8 + diff --git a/server/pom.xml b/server/pom.xml index 6fd645ad..ad67d9e5 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -101,6 +101,10 @@ org.geotools gt-epsg-hsql + + org.locationtech.spatial4j + spatial4j + org.projectlombok lombok diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java index e551592c..0771ad74 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java @@ -136,6 +136,7 @@ default Collection getCollection(D m, Filter fil if(m.getSummaries() != null ) { Map noLand = m.getSummaries().getGeometryNoLand(); if (noLand != null) { + // Geometry from elastic search always store in EPSG4326 GeometryUtils.readGeometry(noLand) .ifPresent(input -> { Geometry g = filter != null ? (Geometry)filter.accept(visitor, input) : input; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java index acf3fb3f..b49996cf 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java @@ -22,7 +22,7 @@ public abstract class StacToCollections implements Converter collections = model.getCollections().parallelStream() + List collections = model.getCollections().stream() .map(m -> getCollection(m, filter, hostname)) .collect(Collectors.toList()); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLCrsType.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLCrsType.java index 86ea41a4..eb5f7ed6 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLCrsType.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLCrsType.java @@ -39,8 +39,6 @@ public static CQLCrsType convertFromUrl(String url) { public static Geometry transformGeometry(Geometry geometry, CQLCrsType source, CQLCrsType target) throws FactoryException, TransformException { - GeometryFactory factory = JTSFactoryFinder.getGeometryFactory(); - CoordinateReferenceSystem sourceCRS = CRS.decode(source.code); CoordinateReferenceSystem targetCRS = CRS.decode(target.code); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/GeometryVisitor.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/GeometryVisitor.java index 31f7de6b..d5026a17 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/GeometryVisitor.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/GeometryVisitor.java @@ -3,9 +3,7 @@ import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.geotools.filter.visitor.DefaultFilterVisitor; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.GeometryCollection; -import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.geom.*; import org.opengis.filter.spatial.Intersects; @Slf4j @@ -15,13 +13,11 @@ public class GeometryVisitor extends DefaultFilterVisitor { @Override public Object visit(Intersects filter, Object data) { if(filter instanceof IntersectsImpl impl) { - if(impl.getPreparedGeometry().isPresent()) { + if(impl.getGeometry().isPresent()) { if (data instanceof Polygon || data instanceof GeometryCollection) { // To handle minor precision issues, try applying a small buffer (like 0.0) to clean up // minor topology errors. This is a trick commonly used with JTS - return impl.getPreparedGeometry().get() - .getGeometry() - .intersection(((Geometry) data).buffer(0.0)); + return impl.getGeometry().get().intersection(((Geometry) data).buffer(0.0)); } else { return data; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/IntersectsImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/IntersectsImpl.java index b09e6af0..03385e03 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/IntersectsImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/stac/IntersectsImpl.java @@ -8,9 +8,6 @@ import org.geotools.filter.AttributeExpressionImpl; import org.geotools.filter.LiteralExpressionImpl; import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.GeometryCollection; -import org.locationtech.jts.geom.prep.PreparedGeometry; -import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.opengis.filter.FilterVisitor; import org.opengis.filter.expression.Expression; import org.opengis.filter.spatial.Intersects; @@ -28,7 +25,7 @@ public class IntersectsImpl & CQLFieldsInterface> implements I protected Expression expression2; @Getter - protected Optional preparedGeometry = Optional.empty(); + protected Optional geometry = Optional.empty(); public IntersectsImpl(Expression expression1, Expression expression2, CQLCrsType cqlCrsType) { if(expression1 instanceof AttributeExpressionImpl attribute && expression2 instanceof LiteralExpressionImpl literal) { @@ -37,7 +34,9 @@ public IntersectsImpl(Expression expression1, Expression expression2, CQLCrsType try { String geojson = GeometryUtils.convertToGeoJson(literal, cqlCrsType); - preparedGeometry = GeometryUtils.readGeometry(geojson).map(g -> PreparedGeometryFactory.prepare(g)); + geometry = GeometryUtils + .readGeometry(geojson) + .map(g -> GeometryUtils.normalizePolygon(g)); } catch(Exception ex) { logger.warn("Exception in parsing, query result will be wrong", ex); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeometryUtils.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeometryUtils.java index 236d7eb3..152f635f 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeometryUtils.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeometryUtils.java @@ -10,6 +10,8 @@ import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; +import org.locationtech.spatial4j.context.jts.JtsSpatialContext; +import org.locationtech.spatial4j.shape.jts.JtsGeometry; import org.opengis.referencing.FactoryException; import org.opengis.referencing.operation.TransformException; import org.slf4j.Logger; @@ -25,7 +27,6 @@ public class GeometryUtils { protected static final int PRECISION = 15; - protected static GeometryFactory factory = new GeometryFactory(new PrecisionModel(), 4326); protected static ObjectMapper mapper = new ObjectMapper(); @@ -208,4 +209,15 @@ public static Optional readGeometry(Object input) { return Optional.empty(); } } + /** + * Normalize a polygon by adjusting longitudes to the range [-180, 180], and return both parts as a GeometryCollection. + * + * @param polygon The input polygon. + * @return A polygon / multi-polygon unwrap at dateline. + */ + public static Geometry normalizePolygon(Geometry polygon) { + // Set dateline 180 check to true to unwrap a polygon across -180 line + JtsGeometry jtsGeometry = new JtsGeometry(polygon, JtsSpatialContext.GEO, true, false); + return jtsGeometry.getGeom(); + } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/stac/ParserTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/stac/ParserTest.java index 102521ea..0b2441c6 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/stac/ParserTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/stac/ParserTest.java @@ -17,6 +17,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.io.ParseException; import org.opengis.filter.Filter; import org.opengis.referencing.FactoryException; @@ -54,7 +55,7 @@ public static void init() { * @throws ParseException - Will not throw */ @Test - public void verifyIntersectionWorks() throws CQLException, IOException, FactoryException, TransformException, ParseException { + public void verifyIntersectionWorks1() throws CQLException, IOException, FactoryException, TransformException, ParseException { // Assume we have the following CQL Converter.Param param = Converter.Param.builder() .coordinationSystem(CQLCrsType.EPSG4326) @@ -85,4 +86,43 @@ public void verifyIntersectionWorks() throws CQLException, IOException, FactoryE Assertions.assertTrue(expected.isPresent(), "Expected parse correct"); Assertions.assertEquals(g, expected.get(), "They are equals"); } + + /** + * Test case where POLYGON cross the -180 line, we should be able to handle it correctly. + * the parser will split the polygon into two and then apply the intersection with the noloand in json sample + * it will result in a single polygon and therefore we can calculate the centroid + * + * @throws CQLException - Will not throw + * @throws IOException - Will not throw + * @throws FactoryException - Will not throw + * @throws TransformException - Will not throw + * @throws ParseException - Will not throw + */ + @Test + public void verifyIntersectionWorks2() throws CQLException, IOException, FactoryException, TransformException, ParseException { + + // Parse the json and get the noland section + String json = BaseTestClass.readResourceFile("classpath:databag/0015db7e-e684-7548-e053-08114f8cd4ad.json"); + StacCollectionModel model = mapper.readValue(json, StacCollectionModel.class); + + Filter filter = CompilerUtil.parseFilter( + Language.CQL, + "score>=1.5 AND INTERSECTS(geometry,POLYGON ((-203.16603491348164 -60.248194404495756, -86.85117538227594 -60.248194404495756, -86.85117538227594 15.902738674628525, -203.16603491348164 15.902738674628525, -203.16603491348164 -60.248194404495756)))", + factory); + + Optional geo = GeometryUtils.readGeometry(model.getSummaries().getGeometryNoLand()); + + Assertions.assertTrue(geo.isPresent(), "Parse no land correct"); + GeometryVisitor visitor = GeometryVisitor.builder() + .build(); + + // return value are geo applied the CQL, and in this case only INTERSECTS + Geometry g = (Geometry)filter.accept(visitor, geo.get()); + + Assertions.assertFalse(g.isEmpty()); + Assertions.assertTrue(g instanceof Polygon); + + Assertions.assertEquals(g.getCentroid().getX(), 168.30090846621448, "getX()"); + Assertions.assertEquals(g.getCentroid().getY(), -33.95984804960966, "getY()"); + } } diff --git a/server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json b/server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json new file mode 100644 index 00000000..29f11517 --- /dev/null +++ b/server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json @@ -0,0 +1,665 @@ +{ + "summaries": { + "proj:geometry_noland": { + "geometries": [ + { + "type": "Polygon", + "coordinates": [ + [ + [ + 54, + -66 + ], + [ + 102.5, + -66 + ], + [ + 113, + -66 + ], + [ + 133.5, + -66 + ], + [ + 180, + -66 + ], + [ + 180, + -17 + ], + [ + 178.5, + -17 + ], + [ + 180, + -16 + ], + [ + 180, + -8 + ], + [ + 147.75, + -8 + ], + [ + 148.5, + -9 + ], + [ + 151, + -10 + ], + [ + 147.5, + -10 + ], + [ + 146, + -8 + ], + [ + 142, + -8 + ], + [ + 143.5, + -9 + ], + [ + 141, + -9 + ], + [ + 140, + -8 + ], + [ + 139, + -8.5 + ], + [ + 138.83333333333334, + -8 + ], + [ + 114.5, + -8 + ], + [ + 114.5, + -8.5 + ], + [ + 110, + -8 + ], + [ + 44, + -8 + ], + [ + 44, + -21.166666666666668 + ], + [ + 44.5, + -20.5 + ], + [ + 44.5, + -16 + ], + [ + 46.5, + -16 + ], + [ + 48, + -15 + ], + [ + 48, + -13.5 + ], + [ + 49, + -13.5 + ], + [ + 49.5, + -12 + ], + [ + 50.5, + -15.5 + ], + [ + 49.5, + -15.5 + ], + [ + 50, + -17 + ], + [ + 47, + -25 + ], + [ + 45, + -25.5 + ], + [ + 44, + -25 + ], + [ + 44, + -66 + ], + [ + 54, + -66 + ] + ], + [ + [ + 114.5, + -8 + ], + [ + 115.5, + -8.5 + ], + [ + 115, + -9 + ], + [ + 114.5, + -8 + ] + ], + [ + [ + 138, + -34 + ], + [ + 137.5, + -35 + ], + [ + 137, + -35.5 + ], + [ + 138, + -34 + ] + ], + [ + [ + 138, + -34 + ], + [ + 138, + -33.5 + ], + [ + 138, + -32.5 + ], + [ + 136, + -35 + ], + [ + 134, + -32.5 + ], + [ + 131, + -31.5 + ], + [ + 129, + -31.5 + ], + [ + 126, + -32.5 + ], + [ + 123.5, + -34 + ], + [ + 120, + -34 + ], + [ + 118, + -35 + ], + [ + 116, + -35 + ], + [ + 115, + -34 + ], + [ + 115.5, + -32 + ], + [ + 113, + -26 + ], + [ + 114, + -26.5 + ], + [ + 113.5, + -24.5 + ], + [ + 113.5, + -22.5 + ], + [ + 117, + -20.5 + ], + [ + 121, + -19.5 + ], + [ + 123, + -16.5 + ], + [ + 123.5, + -17.5 + ], + [ + 123.5, + -16 + ], + [ + 125, + -16.5 + ], + [ + 124.5, + -15.5 + ], + [ + 125, + -15.5 + ], + [ + 126, + -14 + ], + [ + 127.5, + -14 + ], + [ + 128, + -15.5 + ], + [ + 128.5, + -15 + ], + [ + 129.5, + -15 + ], + [ + 129.5, + -14.5 + ], + [ + 130.5, + -12.5 + ], + [ + 133, + -12 + ], + [ + 132, + -11 + ], + [ + 135, + -12.5 + ], + [ + 136, + -12 + ], + [ + 136.5, + -12 + ], + [ + 137, + -12.5 + ], + [ + 135.5, + -15 + ], + [ + 140.5, + -17.5 + ], + [ + 142.5, + -10.5 + ], + [ + 144, + -14.5 + ], + [ + 145.5, + -15 + ], + [ + 146.5, + -19 + ], + [ + 149, + -20 + ], + [ + 149.5, + -22.5 + ], + [ + 150.5, + -22.5 + ], + [ + 151, + -23.5 + ], + [ + 153, + -25.5 + ], + [ + 153.5, + -28.5 + ], + [ + 153, + -31.5 + ], + [ + 151, + -33.5 + ], + [ + 150, + -37.5 + ], + [ + 146.5, + -39 + ], + [ + 144.5, + -38.5 + ], + [ + 143.5, + -39 + ], + [ + 140.5, + -38 + ], + [ + 139.5, + -35.5 + ], + [ + 138, + -35.5 + ], + [ + 138, + -34 + ] + ], + [ + [ + 69, + -49.5 + ], + [ + 69, + -48.5 + ], + [ + 70.5, + -49 + ], + [ + 69, + -49.5 + ] + ], + [ + [ + 147.5, + -43 + ], + [ + 147, + -43.5 + ], + [ + 146, + -43.5 + ], + [ + 144.5, + -40.5 + ], + [ + 148, + -41 + ], + [ + 148, + -43 + ], + [ + 147.5, + -43 + ] + ], + [ + [ + 175.5, + -37.5 + ], + [ + 175.5, + -36.5 + ], + [ + 177, + -38 + ], + [ + 178.5, + -37.5 + ], + [ + 178, + -39.5 + ], + [ + 177, + -39.5 + ], + [ + 176, + -41 + ], + [ + 174.5, + -41.5 + ], + [ + 172.5, + -43.5 + ], + [ + 173, + -44 + ], + [ + 171.5, + -44.5 + ], + [ + 169.5, + -46.5 + ], + [ + 166.5, + -46 + ], + [ + 173, + -40.5 + ], + [ + 173, + -41.5 + ], + [ + 174.5, + -41 + ], + [ + 175, + -40 + ], + [ + 174, + -39.5 + ], + [ + 174.5, + -39 + ], + [ + 175, + -37 + ], + [ + 175.5, + -37.5 + ] + ], + [ + [ + 172.5, + -34.5 + ], + [ + 174.5, + -35 + ], + [ + 174.5, + -36.5 + ], + [ + 172.5, + -34.5 + ] + ], + [ + [ + 177.5, + -18 + ], + [ + 178, + -17.5 + ], + [ + 178.5, + -18 + ], + [ + 177.5, + -18 + ] + ], + [ + [ + 131, + -12 + ], + [ + 130.5, + -11 + ], + [ + 131.5, + -11.5 + ], + [ + 131, + -12 + ] + ], + [ + [ + 127.5, + -8.5 + ], + [ + 123.5, + -10.5 + ], + [ + 124.5, + -9 + ], + [ + 127.5, + -8.5 + ] + ] + ] + } + ], + "type": "GeometryCollection" + } + }, + "id": "0015db7e-e684-7548-e053-08114f8cd4ad" +} \ No newline at end of file From de2bdf5ad7feb4562ffc19525d381f8c7f2c2a6d Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Thu, 21 Nov 2024 17:26:22 +1100 Subject: [PATCH 2/4] pre-commit fix --- .../resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json b/server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json index 29f11517..745e0394 100644 --- a/server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json +++ b/server/src/test/resources/databag/0015db7e-e684-7548-e053-08114f8cd4ad.json @@ -662,4 +662,4 @@ } }, "id": "0015db7e-e684-7548-e053-08114f8cd4ad" -} \ No newline at end of file +} From 1c0b38d47be10c1b1cc98cdf46f31bcbe9c76d06 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Thu, 21 Nov 2024 17:30:06 +1100 Subject: [PATCH 3/4] Undo not necessary change --- .../org/aodn/ogcapi/server/core/mapper/StacToCollections.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java index b49996cf..acf3fb3f 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java @@ -22,7 +22,7 @@ public abstract class StacToCollections implements Converter collections = model.getCollections().stream() + List collections = model.getCollections().parallelStream() .map(m -> getCollection(m, filter, hostname)) .collect(Collectors.toList()); From 5b06c141c80ac5b397b44f3cf772bc7423ebdf4c Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 22 Nov 2024 10:45:53 +1100 Subject: [PATCH 4/4] Add comment --- .../core/configuration/ElasticSearchConfig.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java index ee42f196..85bbc85a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java @@ -48,12 +48,20 @@ public RestClientTransport restClientTransport() { public ElasticsearchClient geoNetworkElasticsearchClient(RestClientTransport transport) { return new ElasticsearchClient(transport); } - + /** + * The elastic search client to do the query + * @param client - The elastic search client + * @param mapper - Object mapper for string to object transformation + * @param indexName - The elastic index name that store the STAC from es-indexer + * @param pageSize - Do not set this value too high, say 5000 will crash elastic search + * @param searchAsYouTypeSize - The number of search result return for search as you type + * @return + */ @Bean public Search createElasticSearch(ElasticsearchClient client, ObjectMapper mapper, @Value("${elasticsearch.index.name}") String indexName, - @Value("${elasticsearch.index.pageSize:5000}") Integer pageSize, + @Value("${elasticsearch.index.pageSize:2500}") Integer pageSize, @Value("${elasticsearch.search_as_you_type.size:10}") Integer searchAsYouTypeSize) { return new ElasticSearch(client, mapper, indexName, pageSize, searchAsYouTypeSize);