Merge pull request #725 from conveyal/dev

Prepare Release v6.3

.github/workflows/cypress-integration.yml

@@ -16,20 +16,21 @@ jobs:
       - uses: actions/checkout@v2
         with:
           repository: conveyal/analysis-ui
-          ref: dev
+          ref: a869cd11919343a163e110e812b5d27f3a4ad4c8
           path: ui
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
           path: r5
 
-      # Build .jar and copy to ./ui directory
+      # Build .jar and copy to ./ui directory (along with config file)
       - uses: actions/setup-java@v1
         with:
           java-version: 11
       - run: gradle shadowJar -x test
         working-directory: r5
       - run: cp $(ls ./r5/build/libs/*-all.jar | head -n1) ./ui/latest.jar
+      - run: cp ./r5/analysis.properties.template ./ui/analysis.properties
 
       # Install / cache dependencies with Cypress to handle caching Cypress binary.
       - uses: actions/setup-node@v2
@@ -38,6 +39,7 @@ jobs:
       - uses: cypress-io/github-action@v2
         env:
           NEXT_PUBLIC_BASEMAP_DISABLED: true
+          NEXT_PUBLIC_CYPRESS: true
           NEXT_PUBLIC_MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }}
         with:
           build: yarn build

.github/workflows/gradle.yml

@@ -26,17 +26,12 @@ jobs:
     #   image: mherwig/docker-alpine-java-mongo:latest
     #   env:
     #     BUILD_TARGET:staging
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      GPR_USERNAME: ${{ secrets.GPR_USERNAME }}
-      GPR_TOKEN: ${{ secrets.GPR_TOKEN }}
     steps:
     # Starting in v2.2 checkout action fetches all tags when fetch-depth=0, for auto-versioning.
     - uses: actions/[email protected]
       with:
         fetch-depth: 0
-    # Java setup step completes very fast, no need to run in a preconfigured docker container
+    # Java setup step completes very fast, no need to run in a preconfigured docker container.
     - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
@@ -46,29 +41,16 @@ jobs:
       with:
         path: ~/.gradle/caches
         key: gradle-caches
+    - name: Show version string
+      run: gradle -q printVersion | head -n1
     - name: Build and Test
       run: gradle build
-    - name: Ensure shadow JAR is runnable as backend
+    - name: Ensure shadow JAR is runnable as local backend
       run: |
         cp analysis.properties.template analysis.properties
         gradle testShadowJarRunnable
     - name: Publish to GH Packages
+      # Supply access token to build.gradle (used in publishing.repositories.maven.credentials)
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: gradle publish
-    - name: Upload to S3
-      # Use git describe to get a similar string to the Gradle project version (possibly missing .dirty).
-      run: |
-        VERSION=$(gradle -q printVersion | head -n1)
-        LOCAL_FILE=$(ls build/libs/*-all.jar | head -n1)
-        aws s3 cp --no-progress --region eu-west-1 $LOCAL_FILE s3://r5-builds/${VERSION}.jar
-    - name: Publish Docker image to GH Container Registry
-      run: |
-        echo $GPR_TOKEN | docker login ghcr.io -u $GPR_USERNAME --password-stdin
-        gradle jib
-    # If we are on the head of dev or master, also copy to branch-latest.jar. 'aws s3 cp' will overwrite by default.
-    - name: Copy to branch-latest.jar on S3
-      if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/master'
-      run: |
-        VERSION=$(gradle -q printVersion | head -n1)
-        BRANCH=${GITHUB_REF#refs/heads/}
-        echo VERSION is $VERSION, BRANCH is $BRANCH
-        aws s3 cp --no-progress --region eu-west-1 s3://r5-builds/${VERSION}.jar s3://r5-builds/${BRANCH}-latest.jar

README.md

@@ -8,7 +8,7 @@ We refer to the routing method as "realistic" because it works by planning many
 
 We say "Real-world and Reimagined" networks because R5's networks are built from widely available open OSM and GTFS data describing baseline transportation systems, but R5 includes a system for applying light-weight patches to those networks for immediate, interactive scenario comparison.
 
-R5 is a core component of [Conveyal Analysis](https://www.conveyal.com/analysis), which allows users to create transportation scenarios and evaluate them in terms of cumulative opportunities accessibility indicators. 
+R5 is a core component of [Conveyal Analysis](https://www.conveyal.com/learn), which allows users to create transportation scenarios and evaluate them in terms of cumulative opportunities accessibility indicators. See the [methodology section](https://docs.conveyal.com/analysis/methodology) of the [Conveyal user manual](https://docs.conveyal.com/) for more information.
 
 **Please note** that the Conveyal team does not provide technical support for third-party deployments of its analysis platform. We provide paid subscriptions to a cloud-based deployment of this system, which performs these complex calculations hundreds of times faster using a compute cluster. This project is open source primarily to ensure transparency and reproducibility in public planning and decision making processes, and in hopes that it may help researchers, students, and potential collaborators to understand and build upon our methodology.
 

analysis.properties.template

@@ -1,7 +1,10 @@
 # This file contains the configuration options for Conveyal Analysis Backend
 
-# The host and port of the remote Mongo server (if any). Comment out for local Mongo instance.
+immediate-shutdown=false
+
+# The host and port of the Mongo server.
 # database-uri=mongodb://127.0.0.1:27017
+database-uri=mongodb://localhost
 
 # The name of the database in the Mongo instance.
 database-name=analysis
@@ -11,12 +14,6 @@ database-name=analysis
 # In staging this should be the underlying S3 URL so files are not cached and you see the most recent deployment.
 frontend-url=https://localhost
 
-# S3 buckets where Analysis inputs and results are stored.
-bundle-bucket=analysis-staging-bundles
-grid-bucket=analysis-staging-grids
-results-bucket=analysis-staging-results
-resources-bucket=analysis-staging-resources
-
 # The S3 bucket where we can find tiles of the entire US census, built with Conveyal seamless-census.
 seamless-census-bucket=lodes-data-2014
 seamless-census-region=us-east-1

build.gradle

@@ -3,15 +3,12 @@ plugins {
     id 'com.github.johnrengelman.shadow' version '6.0.0'
     id 'maven-publish'
     id 'com.palantir.git-version' version '0.12.3'
-    id 'com.google.cloud.tools.jib' version '2.6.0'
 }
 
 group = 'com.conveyal'
 // set version to `git describe --tags --always --first-parent`, plus '.dirty' if local changes are present.
 version gitVersion()
 
-jib.to.image = 'ghcr.io/conveyal/r5:' + version
-
 java {
     sourceCompatibility = JavaVersion.VERSION_11
     targetCompatibility = JavaVersion.VERSION_11
@@ -20,12 +17,12 @@ java {
 jar {
     // For Java 11 Modules, specify a module name.
     // Do not create module-info.java until all our dependencies specify a module name.
-    // Main-Class R5Main will start a worker, BackendMain must be specified on JVM command line to start backend.
+    // Main-Class BackendMain will start a local backend.
     // Build-Jdk-Spec mimics a Maven manifest entry that helps us automatically install the right JVM.
     // Implementation-X attributes are needed for ImageIO (used by Geotools) to initialize in some environments.
     manifest {
         attributes 'Automatic-Module-Name': 'com.conveyal.analysis',
-            'Main-Class': 'com.conveyal.r5.R5Main',
+            'Main-Class': 'com.conveyal.analysis.BackendMain',
             'Build-Jdk-Spec': targetCompatibility.getMajorVersion(),
             'Implementation-Title': 'Conveyal Analysis Backend',
             'Implementation-Vendor': 'Conveyal LLC',
@@ -42,14 +39,15 @@ test {
 }
 
 // `gradle publish` will upload both shadow and simple JAR to Github Packages
+// On GH Actions, GITHUB_ACTOR env variable is supplied without specifying it in action yml.
 publishing {
     repositories {
         maven {
             name = "GitHubPackages"
             url = uri("https://maven.pkg.github.com/conveyal/r5")
             credentials {
-                username = System.getenv("GPR_USERNAME")
-                password = System.getenv("GPR_TOKEN")
+                username = System.getenv("GITHUB_ACTOR")
+                password = System.getenv("GITHUB_TOKEN")
             }
         }
     }
@@ -74,21 +72,14 @@ task copyDependencies(type: Copy) {
     into 'dependencies'
 }
 
-// Run R5 as an analysis backend with all dependencies on the classpath, without building a shadowJar.
+// Run R5 as a local analysis backend with all dependencies on the classpath, without building a shadowJar.
 task runBackend (type: JavaExec) {
    dependsOn(build)
    classpath(sourceSets.main.runtimeClasspath)
    main("com.conveyal.analysis.BackendMain")
 }
 
-// Run R5 as an analysis worker with all dependencies on the classpath, without building a shadowJar.
-task runWorker (type: JavaExec) {
-   dependsOn(build)
-   classpath(sourceSets.main.runtimeClasspath)
-   main("com.conveyal.r5.R5Main")
-}
-
-// Start up the analysis backend from a shaded JAR and ask it to shut down immediately.
+// Start up an analysis local backend from a shaded JAR and ask it to shut down immediately.
 // This is used to check in the automated build that the JAR is usable before we keep it.
 // Create a configuration properties file (by copying the template) before running this task.
 task testShadowJarRunnable(type: JavaExec) {
@@ -103,15 +94,17 @@ task testShadowJarRunnable(type: JavaExec) {
 task createVersionProperties(dependsOn: processResources) {
     doLast {
         def details = versionDetails()
-        new File(buildDir, "resources/main/version.properties").withWriter { w ->
+        def dir = new File(buildDir, "resources/main/com/conveyal/r5/")
+        mkdir(dir)
+        new File(dir, "version.properties").withWriter { w ->
             Properties p = new Properties()
             p['version'] = project.version.toString()
             p['commit'] = details.gitHashFull
             p['branch'] = details.branchName
             p.store w, null
         }
         // Also make a simple one-line version.txt for scripts to use
-        new File(buildDir, "version.txt").text = "$version"
+        new File(dir, "version.txt").text = "$version"
     }
 }
 
@@ -172,7 +165,7 @@ dependencies {
     // Provides the EPSG coordinate reference system catalog as an HSQL database.
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-epsg-hsql'
 
-    compile 'com.wdtinc:mapbox-vector-tile:3.1.0'
+    implementation 'com.wdtinc:mapbox-vector-tile:3.1.0'
 
     // Legacy JTS with com.vividsolutions package name. Newer Geotools compatible with Java 11 uses a newer version of
     // JTS with the org.locationtech package name. But our MapDB format includes serialized JTS geometries with the
@@ -196,7 +189,7 @@ dependencies {
     implementation 'com.google.guava:guava:28.2-jre'
 
     // Java 8 rewrite of the Guava cache with asynchronous LoadingCaches. We don't currently use the async
-    //capabilities, but Caffeine's LoadingCache syntax is more modern idiomatic Java than Guava's.
+    // capabilities, but Caffeine's LoadingCache syntax is more modern idiomatic Java than Guava's.
     implementation 'com.github.ben-manes.caffeine:caffeine:2.8.1'
 
     implementation ('org.apache.httpcomponents:httpclient:4.5.6') {
@@ -205,6 +198,7 @@ dependencies {
     }
 
     // Persistent storage of files / objects on Amazon S3.
+    // Now used only for Seamless Census TODO eliminate this final AWS dependency
     implementation 'com.amazonaws:aws-java-sdk-s3:1.11.341'
 
     // Old version of GraphQL-Java used by legacy gtfs-api embedded in analysis-backend.

src/main/java/com/conveyal/analysis/AnalysisServerException.java

@@ -11,10 +11,10 @@ public class AnalysisServerException extends RuntimeException {
     private static final Logger LOG = LoggerFactory.getLogger(AnalysisServerException.class);
 
     public int httpCode;
-    public TYPE type;
+    public Type type;
     public String message;
 
-    public enum TYPE {
+    public enum Type {
         BAD_REQUEST,
         BROKER,
         FILE_UPLOAD,
@@ -23,25 +23,26 @@ public enum TYPE {
         JSON_PARSING,
         NONCE,
         NOT_FOUND,
+        RUNTIME,
         UNAUTHORIZED,
         UNKNOWN;
     }
 
     public static AnalysisServerException badRequest(String message) {
-        return new AnalysisServerException(TYPE.BAD_REQUEST, message, 400);
+        return new AnalysisServerException(Type.BAD_REQUEST, message, 400);
     }
 
     public static AnalysisServerException fileUpload(String message) {
-        return new AnalysisServerException(TYPE.FILE_UPLOAD, message, 400);
+        return new AnalysisServerException(Type.FILE_UPLOAD, message, 400);
     }
 
     public static AnalysisServerException forbidden(String message) {
-        return new AnalysisServerException(TYPE.FORBIDDEN, message, 403);
+        return new AnalysisServerException(Type.FORBIDDEN, message, 403);
     }
 
     public static AnalysisServerException graphQL(List<GraphQLError> errors) {
         return new AnalysisServerException(
-                TYPE.GRAPHQL,
+                Type.GRAPHQL,
                 errors
                     .stream()
                     .map(e -> e.getMessage())
@@ -51,36 +52,36 @@ public static AnalysisServerException graphQL(List<GraphQLError> errors) {
     }
 
     public static AnalysisServerException nonce() {
-        return new AnalysisServerException(TYPE.NONCE, "The data you attempted to change is out of date and could not be " +
+        return new AnalysisServerException(Type.NONCE, "The data you attempted to change is out of date and could not be " +
                 "updated. This project may be open by another user or in another browser tab.", 400);
     }
 
     public static AnalysisServerException notFound(String message) {
-        return new AnalysisServerException(TYPE.NOT_FOUND, message, 404);
+        return new AnalysisServerException(Type.NOT_FOUND, message, 404);
     }
 
     public static AnalysisServerException unauthorized(String message) {
-        return new AnalysisServerException(TYPE.UNAUTHORIZED, message, 401);
+        return new AnalysisServerException(Type.UNAUTHORIZED, message, 401);
     }
 
     public static AnalysisServerException unknown(Exception e) {
-        return new AnalysisServerException(TYPE.UNKNOWN, ExceptionUtils.asString(e), 400);
+        return new AnalysisServerException(Type.UNKNOWN, ExceptionUtils.stackTraceString(e), 400);
     }
 
     public static AnalysisServerException unknown(String message) {
-        return new AnalysisServerException(TYPE.UNKNOWN, message, 400);
+        return new AnalysisServerException(Type.UNKNOWN, message, 400);
     }
 
     public AnalysisServerException(Exception e, String message) {
         this(message);
-        LOG.error(ExceptionUtils.asString(e));
+        LOG.error(ExceptionUtils.stackTraceString(e));
     }
 
     public AnalysisServerException(String message) {
-        this(TYPE.UNKNOWN, message, 400);
+        this(Type.UNKNOWN, message, 400);
     }
 
-    public AnalysisServerException(AnalysisServerException.TYPE t, String m, int c) {
+    public AnalysisServerException(Type t, String m, int c) {
         httpCode = c;
         type = t;
         message = m;