⛓️‍💥 HTM-1091: Add feature geometry as a strong-type ⛓️‍💥 (#1025)

build/ci/docker-compose.yml

@@ -107,7 +107,7 @@ services:
       - postgis
       - oracle
       - sqlserver
-    image: solr:latest
+    image: ghcr.io/tailormap/solr:9.7.0
     environment:
       TZ: Europe/Amsterdam
     volumes:

src/main/java/org/tailormap/api/controller/SearchController.java

@@ -67,7 +67,10 @@ public ResponseEntity<Serializable> search(
       @ModelAttribute GeoService service,
       @ModelAttribute Application application,
       @RequestParam(required = false, name = "q") final String solrQuery,
-      @RequestParam(required = false, defaultValue = "0") Integer start) {
+      @RequestParam(required = false, defaultValue = "0") Integer start,
+      @RequestParam(required = false, name = "fq") final String solrFilterQuery,
+      @RequestParam(required = false, name = "pt") final String solrPoint,
+      @RequestParam(required = false, name = "d") final Double solrDistance) {
 
     AppLayerSettings appLayerSettings = application.getAppLayerSettings(appTreeLayerNode);
 
@@ -90,7 +93,14 @@ public ResponseEntity<Serializable> search(
     try (SolrClient solrClient = solrService.getSolrClientForSearching();
         SolrHelper solrHelper = new SolrHelper(solrClient)) {
       final SearchResponse searchResponse =
-          solrHelper.findInIndex(searchIndex, solrQuery, start, numResultsToReturn);
+          solrHelper.findInIndex(
+              searchIndex,
+              solrQuery,
+              solrFilterQuery,
+              solrPoint,
+              solrDistance,
+              start,
+              numResultsToReturn);
       return (null == searchResponse.getDocuments() || searchResponse.getDocuments().isEmpty())
           ? ResponseEntity.noContent().build()
           : ResponseEntity.ok().body(searchResponse);

src/main/java/org/tailormap/api/solr/SolrHelper.java

@@ -12,13 +12,15 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
 import org.apache.solr.client.solrj.request.schema.SchemaRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.client.solrj.response.UpdateResponse;
@@ -49,13 +51,23 @@
  * in a try-with-resources.
  */
 public class SolrHelper implements AutoCloseable, Constants {
-  public static final int SOLR_BATCH_SIZE = 1000;
   private static final Logger logger =
       LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-  // milliseconds
+
+  /**
+   * the number of documents that are submitted per batch to the Solr service: {@value
+   * #SOLR_BATCH_SIZE}.
+   */
+  public static final int SOLR_BATCH_SIZE = 1000;
+
+  /** {@value #SOLR_TIMEOUT} milliseconds. */
   private static final int SOLR_TIMEOUT = 7000;
+
   private final SolrClient solrClient;
 
+  /** the Solr field type name geometry fields: {@value #SOLR_SPATIAL_FIELDNAME}. */
+  private static final String SOLR_SPATIAL_FIELDNAME = "tm_geometry_rpt";
+
   /**
    * Constructor
    *
@@ -170,6 +182,8 @@ public SearchIndex addFeatureTypeIndex(
               if (value != null) {
                 if (value instanceof Geometry
                     && propertyName.equals(tmFeatureType.getDefaultGeometryAttribute())) {
+                  // We could use GeoJSON, but WKT is more compact and that would also incur a
+                  // change to the API
                   doc.setGeometry(GeometryProcessor.processGeometry(value, true, true, null));
                 } else {
                   if (searchFields.contains(propertyName)) {
@@ -280,15 +294,24 @@ public void clearIndexForLayer(@NotNull Long searchLayerId)
    * @param searchIndex the search index
    * @param solrQuery the query, when {@code null} or empty, the query is set to {@code *} (match
    *     all)
+   * @param solrPoint the point to search around, in (x y) format
+   * @param solrDistance the distance to search around the point in Solr distance units (kilometers)
    * @param start the start index, starting at 0
    * @param numResultsToReturn the number of results to return
    * @return the documents
    * @throws IOException if an I/O error occurs
    * @throws SolrServerException if a Solr error occurs
    */
   public SearchResponse findInIndex(
-      @NotNull SearchIndex searchIndex, String solrQuery, int start, int numResultsToReturn)
+      @NotNull SearchIndex searchIndex,
+      String solrQuery,
+      String solrFilterQuery,
+      String solrPoint,
+      Double solrDistance,
+      int start,
+      int numResultsToReturn)
       throws IOException, SolrServerException, SolrException {
+
     logger.info("Find in index for {}", searchIndex.getId());
     if (null == solrQuery || solrQuery.isBlank()) {
       solrQuery = "*";
@@ -308,6 +331,17 @@ public SearchResponse findInIndex(
             .addSort(SEARCH_ID_FIELD, SolrQuery.ORDER.asc)
             .setRows(numResultsToReturn)
             .setStart(start);
+
+    if (null != solrFilterQuery && !solrFilterQuery.isBlank()) {
+      query.addFilterQuery(solrFilterQuery);
+    }
+    if (null != solrPoint && null != solrDistance) {
+      if (null == solrFilterQuery || !solrFilterQuery.startsWith("{!geofilt")) {
+        query.addFilterQuery("{!geofilt sfield=" + INDEX_GEOM_FIELD + "}");
+      }
+      query.add("pt", solrPoint);
+      query.add("d", solrDistance.toString());
+    }
     query.set("q.op", "AND");
     logger.debug("Solr query: {}", query);
 
@@ -375,18 +409,47 @@ private void createSchemaIfNotExists() throws SolrServerException, IOException {
                 "uninvertible", false));
     searchLayerSchemaRequest.process(solrClient);
 
-    logger.info("Creating Solr field type {}", INDEX_GEOM_FIELD);
-    // TODO https://b3partners.atlassian.net/browse/HTM-1091
-    //  this should be a spatial field type using ("type", "location_rpt")
-    //  but that requires some more work
+    logger.info("Creating Solr field type for {}", INDEX_GEOM_FIELD);
+    // see
+    // https://solr.apache.org/guide/solr/latest/query-guide/spatial-search.html#schema-configuration-for-rpt
+    FieldTypeDefinition spatialFieldTypeDef = new FieldTypeDefinition();
+    Map<String, Object> spatialFieldAttributes =
+        new HashMap<>(
+            Map.of(
+                "name", SOLR_SPATIAL_FIELDNAME,
+                "class", "solr.SpatialRecursivePrefixTreeFieldType",
+                "spatialContextFactory", "JTS",
+                "geo", false,
+                "distanceUnits", "kilometers",
+                "distCalculator", "cartesian",
+                "format", "WKT",
+                "autoIndex", true,
+                "distErrPct", "0.025",
+                "maxDistErr", "0.001"));
+    spatialFieldAttributes.putAll(
+        Map.of(
+            "prefixTree", "packedQuad",
+            // see
+            // https://locationtech.github.io/spatial4j/apidocs/org/locationtech/spatial4j/context/jts/ValidationRule.html
+            "validationRule", "repairBuffer0",
+            // NOTE THE ODDITY in coordinate order of "worldBounds",
+            // "ENVELOPE(minX, maxX, maxY, minY)"
+            "worldBounds",
+                // webmercator / EPSG:3857 projected bounds
+                "ENVELOPE(-20037508.34, 20037508.34, 20048966.1, -20048966.1)"
+            // Amersfoort/RD new / EPSG:28992 projected bounds
+            // "ENVELOPE(482.06, 284182.97, 637049.52, 306602.42)"
+            ));
+    spatialFieldTypeDef.setAttributes(spatialFieldAttributes);
+    SchemaRequest.AddFieldType spatialFieldType =
+        new SchemaRequest.AddFieldType(spatialFieldTypeDef);
+    spatialFieldType.process(solrClient);
+    solrClient.commit();
+
+    logger.info("Creating Solr field {}", INDEX_GEOM_FIELD);
     SchemaRequest.AddField schemaRequestGeom =
         new SchemaRequest.AddField(
-            Map.of(
-                "name", INDEX_GEOM_FIELD,
-                "type", "string",
-                "indexed", false,
-                "stored", true,
-                "multiValued", false));
+            Map.of("name", INDEX_GEOM_FIELD, "type", "tm_geometry_rpt", "stored", true));
     schemaRequestGeom.process(solrClient);
 
     logger.info("Creating Solr field type {}", INDEX_DISPLAY_FIELD);

src/main/resources/openapi/viewer-api.yaml

@@ -317,7 +317,7 @@ components:
           type: string
           nullable: false
         geometry:
-          description: 'The geometry of the feature (WKT'
+          description: 'The geometry of the feature (WKT)'
           type: string
           nullable: true
         displayValues:
@@ -1309,6 +1309,25 @@ paths:
           type: integer
           minimum: 0
           default: 0
+      - description: '(Solr) distance search term (kilometers)'
+        in: query
+        name: d
+        required: true
+        schema:
+          type: number
+          format: double
+      - description: '(Solr) search point term'
+        in: query
+        name: pt
+        required: false
+        schema:
+          type: string
+      - description: '(Solr) filter query term'
+        in: query
+        name: fq
+        required: false
+        schema:
+          type: string
     get:
       operationId: 'search'
       description: 'retrieve a limited list of search responses that fulfill the requested conditions (parameters).'

src/test/java/org/tailormap/api/controller/SearchControllerIntegrationTest.java

@@ -11,6 +11,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWithIgnoringCase;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
@@ -183,4 +184,31 @@ void testBadRequestQuery() throws Exception {
         .andExpect(content().contentType(MediaType.APPLICATION_JSON))
         .andExpect(jsonPath("$.message").value("Error while searching with given query"));
   }
+
+  @Test
+  void testSpatialQueryDistance() throws Exception {
+    final String url = apiBasePath + layerWegdeelSqlServer + "/search";
+
+    mockMvc
+        .perform(
+            get(url)
+                .accept(MediaType.APPLICATION_JSON)
+                .with(setServletPath(url))
+                .param("q", "open")
+                .param("start", "0")
+                // added in backend
+                // .param("fq", "{!geofilt sfield=geometry}")
+                .param("pt", "133809 458811")
+                .param("d", "0.005"))
+        .andExpect(status().isOk())
+        .andExpect(content().contentType(MediaType.APPLICATION_JSON))
+        .andExpect(jsonPath("$.start").value(0))
+        .andExpect(jsonPath("$.total").value(2))
+        .andExpect(jsonPath("$.documents").isArray())
+        .andExpect(jsonPath("$.documents.length()").value(2))
+        .andExpect(jsonPath("$.documents[0].fid").isString())
+        .andExpect(jsonPath("$.documents[0].fid").value(startsWithIgnoringCase("wegdeel")))
+        .andExpect(jsonPath("$.documents[0].displayValues").isArray())
+        .andExpect(jsonPath("$.documents[0]." + INDEX_GEOM_FIELD).isString());
+  }
 }