basic JSON fetch from database with no binding

html select boxes to choose region/bundle/feed
update map style JSON to match GTFS vector tile layer names
try serving up vector tiles UI html/js without CORS

src/main/java/com/conveyal/analysis/components/BackendComponents.java

@@ -11,6 +11,7 @@
 import com.conveyal.analysis.controllers.HttpController;
 import com.conveyal.analysis.controllers.OpportunityDatasetController;
 import com.conveyal.analysis.controllers.RegionalAnalysisController;
+import com.conveyal.analysis.controllers.DatabaseController;
 import com.conveyal.analysis.controllers.UserActivityController;
 import com.conveyal.analysis.controllers.WorkerProxyController;
 import com.conveyal.analysis.grids.SeamlessCensusGridExtractor;
@@ -96,6 +97,7 @@ public List<HttpController> standardHttpControllers () {
                 new BrokerController(broker, eventBus),
                 new UserActivityController(taskScheduler),
                 new DataSourceController(fileStorage, database, taskScheduler, censusExtractor),
+                new DatabaseController(database),
                 new WorkerProxyController(broker)
         );
     }

src/main/java/com/conveyal/analysis/components/HttpApi.java

@@ -78,6 +78,9 @@ private spark.Service configureSparkService () {
         LOG.info("Analysis server will listen for HTTP connections on port {}.", config.serverPort());
         spark.Service sparkService = spark.Service.ignite();
         sparkService.port(config.serverPort());
+        // Serve up UI files. staticFileLocation("vector-client") inside classpath will not see changes to files.
+        // Note that this eliminates the need for CORS.
+        sparkService.externalStaticFileLocation("src/main/resources/vector-client");
 
         // Specify actions to take before the main logic of handling each HTTP request.
         sparkService.before((req, res) -> {
@@ -87,10 +90,10 @@ private spark.Service configureSparkService () {
             // Set CORS headers to allow requests to this API server from a frontend hosted on a different domain.
             // This used to be hardwired to Access-Control-Allow-Origin: * but that leaves the server open to XSRF
             // attacks when authentication is disabled (e.g. when running locally).
-            res.header("Access-Control-Allow-Origin", config.allowOrigin());
-            // For caching, signal to the browser that responses may be different based on origin.
-            // TODO clarify why this is important, considering that normally all requests come from the same origin.
-            res.header("Vary", "Origin");
+//            res.header("Access-Control-Allow-Origin", config.allowOrigin());
+//            // For caching, signal to the browser that responses may be different based on origin.
+//            // TODO clarify why this is important, considering that normally all requests come from the same origin.
+//            res.header("Vary", "Origin");
 
             // The default MIME type is JSON. This will be overridden by the few controllers that do not return JSON.
             res.type("application/json");
@@ -121,17 +124,17 @@ private spark.Service configureSparkService () {
 
         // Handle CORS preflight requests (which are OPTIONS requests).
         // See comment above about Access-Control-Allow-Origin
-        sparkService.options("/*", (req, res) -> {
-            // Cache the preflight response for up to one day (the maximum allowed by browsers)
-            res.header("Access-Control-Max-Age", "86400");
-            res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS");
-            // Allowing credentials is necessary to send an Authorization header
-            res.header("Access-Control-Allow-Credentials", "true");
-            res.header("Access-Control-Allow-Headers", "Accept,Authorization,Content-Type,Origin," +
-                    "X-Requested-With,Content-Length,X-Conveyal-Access-Group"
-            );
-            return "OK";
-        });
+//        sparkService.options("/*", (req, res) -> {
+//            // Cache the preflight response for up to one day (the maximum allowed by browsers)
+//            res.header("Access-Control-Max-Age", "86400");
+//            res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS");
+//            // Allowing credentials is necessary to send an Authorization header
+//            res.header("Access-Control-Allow-Credentials", "true");
+//            res.header("Access-Control-Allow-Headers", "Accept,Authorization,Content-Type,Origin," +
+//                    "X-Requested-With,Content-Length,X-Conveyal-Access-Group"
+//            );
+//            return "OK";
+//        });
 
         // Allow client to fetch information about the backend build version.
         sparkService.get(

src/main/java/com/conveyal/analysis/controllers/DatabaseController.java

@@ -0,0 +1,121 @@
+package com.conveyal.analysis.controllers;
+
+import com.conveyal.analysis.UserPermissions;
+import com.conveyal.analysis.persistence.AnalysisDB;
+import com.google.common.collect.Lists;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+import org.bson.BsonArray;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.bson.json.JsonWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import spark.Request;
+import spark.Response;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.conveyal.analysis.util.JsonUtil.toJson;
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.eq;
+
+/**
+ * Serve up arbitrary records from the database without binding to Java objects.
+ * This converts BSON to JSON. Similar things could be done converting relational rows to JSON.
+ * This allows authenticated retrieval of anything in the database by the UI, even across schema migrations.
+ */
+public class DatabaseController implements HttpController {
+
+    private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private final AnalysisDB database;
+
+    private final MongoCollection<Document> regions;
+    private final MongoCollection<Document> bundles;
+
+    public DatabaseController(AnalysisDB database) {
+        this.database = database;
+        // TODO verify if it is threadsafe to reuse this collection in all threads
+        // Also verify whether it's any slower to just get the collection on every GET operation.
+        // Testing with Apache bench, retaining and reusing the collection seems much smoother.
+        this.regions = database.getBsonCollection("regions");
+        this.bundles = database.getBsonCollection("bundles");
+    }
+
+    /**
+     * Fetch anything from database. Buffers in memory so not suitable for huge responses.
+     * register serialization with sparkService.get("/api/db/:collection", this::getDocuments, toJson);
+     */
+    private Iterable<Document> getDocuments (Request req, Response res) {
+        String accessGroup = UserPermissions.from(req).accessGroup;
+        final String collectionName = req.params("collection");
+        MongoCollection<Document> collection = collectionName.equals("bundles") ? bundles :
+                database.getBsonCollection(collectionName);
+        List<Bson> filters = Lists.newArrayList(eq("accessGroup", accessGroup));
+        req.queryMap().toMap().forEach((key, values) -> {
+            for (String value : values) {
+                filters.add(eq(key, value));
+            }
+        });
+        List<Document> documents = new ArrayList<>();
+        collection.find(and(filters)).into(documents);
+        return documents;
+    }
+
+    /**
+     * Fetch anything from database. Streaming processing, no in-memory buffering of the BsonDocuments.
+     * The output stream does buffer to some extent but should stream chunks instead of serializing into memory.
+     */
+    private Object getDocumentsStreaming (Request req, Response res) {
+        String accessGroup = UserPermissions.from(req).accessGroup;
+        final String collectionName = req.params("collection");
+        MongoCollection<Document> collection = collectionName.equals("bundles") ? bundles :
+                database.getBsonCollection(collectionName);
+        List<Bson> filters = Lists.newArrayList(eq("accessGroup", accessGroup));
+        req.queryMap().toMap().forEach((key, values) -> {
+            for (String value : values) {
+                filters.add(eq(key, value));
+            }
+        });
+        // getOutputStream returns a ServletOutputStream, usually Jetty implementation HttpOutputStream which
+        // buffers the output. doc.toJson() creates a lot of short-lived objects which could be factored out.
+        // The Mongo driver says to use JsonWriter or toJson() rather than utility methods:
+        // https://github.com/mongodb/mongo-java-driver/commit/63409f9cb3bbd0779dd5139355113d9b227dfa05
+        try (OutputStream out = res.raw().getOutputStream()) {
+            out.write('['); // Begin JSON array.
+            boolean firstElement = true;
+            for (Document doc : collection.find(and(filters))) {
+                if (firstElement) {
+                    firstElement = false;
+                } else {
+                    out.write(',');
+                }
+                out.write(doc.toJson().getBytes(StandardCharsets.UTF_8));
+            }
+            out.write(']'); // Close JSON array.
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to write database records as JSON.", e);
+        }
+        // Since we're directly writing to the OutputStream, no need to return anything.
+        // But do not return null or Spark will complain cryptically.
+        return "";
+    }
+
+    // Testing with Apache bench shows some stalling
+    // -k keepalive connections fails immediately
+
+    @Override
+    public void registerEndpoints (spark.Service sparkService) {
+        sparkService.get("/api/db/:collection", this::getDocuments, toJson);
+        //sparkService.get("/api/db/:collection", this::getDocumentsStreaming);
+    }
+
+}

src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java

@@ -6,6 +6,7 @@
 import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
 import org.bson.codecs.configuration.CodecProvider;
 import org.bson.codecs.configuration.CodecRegistry;
 import org.bson.codecs.pojo.Conventions;
@@ -88,6 +89,14 @@ public MongoCollection getMongoCollection (String name, Class clazz) {
         return database.getCollection(name, clazz);
     }
 
+    /**
+     * Lowest-level access to Mongo collections, viewed as BSON rather than mapped to Java classes.
+     */
+    public MongoCollection<Document> getBsonCollection (String name) {
+        // If MongoCollections are threadsafe
+        return database.getCollection(name);
+    }
+
     /** Interface to supply configuration to this component. */
     public interface Config {
         default String databaseUri() { return "mongodb://127.0.0.1:27017"; }

src/main/resources/vector-client/index.html

@@ -4,28 +4,135 @@
     <meta charset="utf-8" />
     <title>Conveyal GTFS Vector Map</title>
     <meta name="viewport" content="initial-scale=1,maximum-scale=1,user-scalable=no" />
-    <script src="https://api.mapbox.com/mapbox-gl-js/v2.0.0/mapbox-gl.js"></script>
-    <link href="https://api.mapbox.com/mapbox-gl-js/v2.0.0/mapbox-gl.css" rel="stylesheet" />
+    <script src="https://api.mapbox.com/mapbox-gl-js/v2.8.2/mapbox-gl.js"></script>
+    <link href="https://api.mapbox.com/mapbox-gl-js/v2.8.2/mapbox-gl.css" rel="stylesheet" />
     <style>
-	    body { margin: 0; padding: 0; }
-	    #map { position: absolute; top: 0; bottom: 0; width: 100%; }
+	    body { margin: 0; padding: 0; font-family: sans-serif}
+        #container {
+            display: flex; flex-direction: row; flex-wrap: nowrap; justify-content: space-between;
+            position: absolute; top: 0; bottom: 0; width: 100%
+        }
+        #panel {
+            float: left; display: flex; flex-direction: column; width: 20%; padding: 10px;
+        }
+	    #map { float: right; width: 80%; }
     </style>
 </head>
 <body>
-<div id="map"></div>
+<div id="container">
+    <div id="panel">
+        <label for="region">Region:</label>
+        <select name="region" id="regions"></select>
+        <label for="bundle">Bundle:</label>
+        <select name="bundle" id="bundles"></select>
+        <label for="feed">Feed:</label>
+        <select name="feed" id="feeds"></select>
+    </div>
+    <div id="map"></div>
+</div>
 <script>
-mapboxgl.accessToken = 'fill in';
+
+mapboxgl.accessToken = 'TOKEN_HERE';
+
+const regionSelectElement = document.getElementById("regions")
+function updateRegionSelector () {
+    regionSelectElement.add(new Option("None"));
+    // Returns an array of regions. Add them to the region selector DOM element.
+    fetch('http://localhost:7070/api/db/regions')
+        .then(response => response.json())
+        .then(regions => {
+            for (const region of regions) {
+                regionSelectElement.add(new Option(region.name, value = region._id));
+            }
+        });
+}
+updateRegionSelector();
+
+const bundleSelectElement = document.getElementById("bundles");
+function updateBundleSelector (regionId) {
+    // Returns an array of bundles. Add them to the bundle selector DOM element.
+    document.querySelectorAll('#bundles option').forEach(option => option.remove())
+    bundleSelectElement.add(new Option("None"));
+    fetch(`http://localhost:7070/api/db/bundles?regionId=${regionId}`)
+        .then(response => response.json())
+        .then(bundles => {
+            for (const bundle of bundles) {
+                bundleSelectElement.add(new Option(bundle.name, value = bundle._id));
+            }
+        });
+}
+
+const feedSelectElement = document.getElementById("feeds");
+function updateFeedSelector (bundle) {
+    document.querySelectorAll('#feeds option').forEach(option => option.remove())
+    feedSelectElement.add(new Option("None"));
+    for (const feed of bundle.feeds) {
+        feedSelectElement.add(new Option(feed.name, value = feed.feedId));
+    }
+}
+
+// TODO better encapsulation of state/model with let state = { ... }
+// Also load URL query parameters into state.
+regionId = null;
+region = null;
+bundleId = null;
+bundle = null;
+feedId = null;
+
+function updateRegion (regionId) {
+    fetch(`http://localhost:7070/api/db/regions?_id=${regionId}`)
+        .then(response => response.json())
+        .then(r => {
+            region = r[0];
+            let centerLon = region.bounds.west + (region.bounds.east - region.bounds.west) / 2;
+            let centerLat = region.bounds.south + (region.bounds.north - region.bounds.south) / 2;
+            map.setCenter([centerLon, centerLat]);
+        });
+}
+
+function updateBundle (bundleId) {
+    fetch(`http://localhost:7070/api/db/bundles?_id=${bundleId}`)
+        .then(response => response.json())
+        .then(b => {
+            bundle = b[0];
+            updateFeedSelector(bundle);
+        });
+}
+
+regionSelectElement.onchange = function (event) {
+    regionId = event.target.value;
+    updateRegion(regionId);
+    updateBundleSelector(regionId);
+}
+
+bundleSelectElement.onchange = function (event) {
+    bundleId = event.target.value;
+    updateBundle(bundleId);
+}
+
+feedSelectElement.onchange = function (event) {
+    feedId = event.target.value;
+    // setUrl expects a URL to TileJSON, not a URL to the tiles themselves.
+    map.getSource('r5').setTiles([`http://localhost:7070/api/gtfs/${bundle.feedGroupId}/${feedId}/tiles/{z}/{x}/{y}`]);
+}
 
 let map = new mapboxgl.Map({
     container: 'map',
     center: [7.4475, 46.948056],
-    // center: [-122.68, 45.52],
     zoom: 9,
     style: 'vectorstyle.json'
-    // style: 'mapbox://styles/mapbox/outdoors-v10'
 });
 
-</script>
+map.on('click', (e) => {
+    const bbox = [
+        [e.point.x - 5, e.point.y - 5],
+        [e.point.x + 5, e.point.y + 5]
+    ];
+    const selectedFeatures = map.queryRenderedFeatures(bbox, { layers: ['patterns'] });
+    const names = selectedFeatures.map(feature => feature.properties.name);
+    console.log(names);
+});
 
+</script>
 </body>
 </html>