Merge branch 'dev' into lts-plus-locked-schedules

.github/workflows/codeql-analysis.yml

@@ -43,6 +43,15 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    # Java setup step completes very fast, no need to run in a preconfigured docker container.
+    # CodeQL is intended to detect any Java toolchains added to the execution environment.
+    - name: Set up JDK 21
+      uses: actions/setup-java@v3
+      with:
+        distribution: corretto
+        java-version: 21
+        cache: gradle
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/gradle.yml

@@ -28,24 +28,25 @@ jobs:
     #     BUILD_TARGET:staging
     steps:
     # Starting in v2.2 checkout action fetches all tags when fetch-depth=0, for auto-versioning.
-    - uses: actions/checkout@v2.3.2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     # Java setup step completes very fast, no need to run in a preconfigured docker container.
-    - name: Set up JDK 11
+    - name: Set up JDK 21
       uses: actions/setup-java@v3
       with:
-        java-version: 11
-        distribution: temurin
-        cache: 'gradle'
+        distribution: corretto
+        java-version: 21
+        cache: gradle
     - name: Show version string
       run: gradle -q printVersion | head -n1
-    - name: Build and Test
+    - name: Build and test
       run: gradle build
-    - name: Ensure shadow JAR is runnable as local backend
+    # Check that build product is able to start up as a backend server before handing it to end-to-end testing
+    - name: Ensure backend runnable
       run: |
         cp analysis.properties.template analysis.properties
-        gradle testShadowJarRunnable
+        gradle testRunnable -x test
     - name: Publish to GH Packages
       # Supply access token to build.gradle (used in publishing.repositories.maven.credentials)
       env:

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Conveyal
+Copyright (c) 2020-2023 Conveyal
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

build.gradle

@@ -1,27 +1,29 @@
 plugins {
     id 'java'
-    id 'com.github.johnrengelman.shadow' version '8.1.0'
+    id 'application'
     id 'maven-publish'
     id 'com.palantir.git-version' version '2.0.0'
+    id 'com.github.johnrengelman.shadow' version '8.1.1'
 }
 
 group = 'com.conveyal'
 // set version to `git describe --tags --always --first-parent`, plus '.dirty' if local changes are present.
 version gitVersion()
 
 java {
-    sourceCompatibility = JavaVersion.VERSION_11
-    targetCompatibility = JavaVersion.VERSION_11
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
 }
 
 jar {
-    // For Java 11 Modules, specify a module name.
+    // For Java 11+ Modules, specify a module name.
     // Do not create module-info.java until all our dependencies specify a module name.
     // Main-Class BackendMain will start a local backend.
-    // Build-Jdk-Spec mimics a Maven manifest entry that helps us automatically install the right JVM.
+    // Build-Jdk-Spec mimics a Maven manifest entry that helps us automatically install or select the right JVM.
     // Implementation-X attributes are needed for ImageIO (used by Geotools) to initialize in some environments.
     manifest {
-        attributes 'Automatic-Module-Name': 'com.conveyal.analysis',
+        attributes 'Automatic-Module-Name': 'com.conveyal.r5',
             'Main-Class': 'com.conveyal.analysis.BackendMain',
             'Build-Jdk-Spec': targetCompatibility.getMajorVersion(),
             'Implementation-Title': 'Conveyal Analysis Backend',
@@ -34,18 +36,19 @@ shadowJar {
     mergeServiceFiles()
 }
 
-// Allow reflective access by Kryo to normally closed Java internals.
-// This is used for testing equality, but also for building automatic Kryo (de)serializers.
+// Allow reflective access by ObjectDiffer to normally closed Java internals. Used for round-trip testing serialization.
+// IntelliJ seems not to pass these JVM arguments when running tests from within the IDE, so the Kryo serialization
+// tests may only succeed under command line Gradle.
 test {
+    useJUnitPlatform()
     jvmArgs = ['--add-opens=java.base/java.io=ALL-UNNAMED',
                '--add-opens=java.base/java.time=ALL-UNNAMED',
                '--add-opens=java.base/java.time.zone=ALL-UNNAMED',
                '--add-opens=java.base/java.lang=ALL-UNNAMED']
-    useJUnitPlatform()
 }
 
-// `gradle publish` will upload both shadow and simple JAR to Github Packages
-// On GH Actions, GITHUB_ACTOR env variable is supplied without specifying it in action yml.
+// Set up publication of jar files to GitHub Packages Maven repository.
+// On GitHub Actions, GITHUB_ACTOR env variable is supplied without specifying it in action yml.
 publishing {
     repositories {
         maven {
@@ -58,10 +61,12 @@ publishing {
         }
     }
     publications {
-        // The presence of the shadow plugin somehow causes the shadow-jar to also be automatically included in this
-        // publication. Ideally we want to produce the shadow jar and upload it to S3 as a worker, but only publish the
-        // much smaller plain JAR without dependencies to Github Packages. On the other hand, we may want to publish
-        // shadow jars for tagged releases.
+        // The shadow plugin automatically creates and registers a component called "shadow" for integration with this
+        // Maven publish plugin. `gradle publish` will then upload both shadow jar and simple jar to Github Packages.
+        // See https://imperceptiblethoughts.com/shadow/getting-started/#default-java-groovy-tasks
+        // To run R5 with dependencies, Conveyal does not use shadow jars anymore, only the zip distribution or runBackend.
+        // For development builds and tests we don't need to produce a shadow jar, only publish the much smaller plain
+        // jar without dependencies to Github Packages. For now, we continue to attach shadow jars to tagged releases.
         gpr(MavenPublication) {
             from(components.java)
         }
@@ -80,19 +85,25 @@ task copyDependencies(type: Copy) {
     into 'dependencies'
 }
 
+application {
+    applicationDefaultJvmArgs = ['-Xmx6G']
+    mainClass = 'com.conveyal.analysis.BackendMain'
+}
+
 // Run R5 as a local analysis backend with all dependencies on the classpath, without building a shadowJar.
 task runBackend (type: JavaExec) {
-   dependsOn(build)
-   classpath(sourceSets.main.runtimeClasspath)
-   mainClass = 'com.conveyal.analysis.BackendMain'
+    dependsOn(build)
+    maxHeapSize('7G')
+    classpath(sourceSets.main.runtimeClasspath)
+    mainClass = 'com.conveyal.analysis.BackendMain'
 }
 
-// Start up an analysis local backend from a shaded JAR and ask it to shut down immediately.
-// This is used to check in the automated build that the JAR is usable before we keep it.
+// Start up an analysis local backend and ask it to shut down immediately.
+// This is used to check in the automated build that the JAR is usable in end-to-end tests before we keep it.
 // Create a configuration properties file (by copying the template) before running this task.
-task testShadowJarRunnable(type: JavaExec) {
-    dependsOn(shadowJar)
-    classpath(shadowJar.archiveFile.get())
+task testRunnable(type: JavaExec) {
+    dependsOn(build)
+    classpath(sourceSets.main.runtimeClasspath)
     mainClass = 'com.conveyal.analysis.BackendMain'
     jvmArgs("-Dconveyal.immediate.shutdown=true")
 }
@@ -116,6 +127,12 @@ task createVersionProperties(dependsOn: processResources) {
     }
 }
 
+// Fix inconsistent Gradle behavior (see https://github.com/gradle/gradle/issues/16791)
+// By default JavaExec tasks use the JVM Gradle was launched with, ignoring the project-level toolchain.
+tasks.withType(JavaExec).configureEach {
+    javaLauncher.set(javaToolchains.launcherFor(java.toolchain))
+}
+
 classes {
     dependsOn createVersionProperties
 }
@@ -136,10 +153,10 @@ configurations.all {
 
 dependencies {
     // Provides our logging API
-    implementation 'org.slf4j:slf4j-api:1.7.30'
+    implementation 'org.slf4j:slf4j-api:2.0.7'
 
     // Implementation of the logging API
-    implementation 'ch.qos.logback:logback-classic:1.2.3'
+    implementation 'ch.qos.logback:logback-classic:1.4.11'
 
     // Spark is an HTTP framework built on Jetty. Its name is the same as several other projects.
     implementation (group: 'com.sparkjava', name: 'spark-core', version: '2.7.2') {
@@ -195,9 +212,6 @@ dependencies {
     // Commons IO gives us BOMInputStream for handling UTF-8 Byte Order Marks.
     implementation 'commons-io:commons-io:2.6'
 
-    // Guava provides a lot of functionality, collections, and tools "missing" from the Java standard library.
-    implementation 'com.google.guava:guava:28.2-jre'
-
     // Java 8 rewrite of the Guava cache with asynchronous LoadingCaches. We don't currently use the async
     // capabilities, but Caffeine's LoadingCache syntax is more modern idiomatic Java than Guava's.
     implementation 'com.github.ben-manes.caffeine:caffeine:2.8.1'
@@ -214,15 +228,19 @@ dependencies {
     // Commons Math gives us FastMath, MersenneTwister, and low-discrepancy vector generators.
     implementation 'org.apache.commons:commons-math3:3.0'
 
-    // Provides some shared serializers for Kryo. Introduces transitive dependencies on Guava, Trove, and Kryo.
+    // Provides some Kryo serializers for Guava and Trove collecitons.
     // Also provides classes for testing that a round trip through serialization reproduces the same network.
     // This is an external dependency (not merged into backend) because it's also used by OTP2.
-    // TODO arguably we should declare non-transitive dependencies on Guava, Trove, and Kryo since we use them directly
-    implementation 'com.conveyal:kryo-tools:1.3.0'
+    implementation 'com.conveyal:kryo-tools:1.6.0'
 
+    // Ensure the versions of the next three dependencies match the transitive dependencies of kryo-tools.
+    implementation 'com.esotericsoftware:kryo:5.5.0'
+    // Guava provides a lot of functionality, collections, and tools "missing" from the Java standard library.
+    implementation 'com.google.guava:guava:32.1.2-jre'
     // Trove supplies very efficient collections of primitive data types for Java.
     implementation 'net.sf.trove4j:trove4j:3.0.3'
 
+
     // TODO eliminate custom Conveyal geojson library, use Geotools?
     implementation 'com.conveyal:jackson2-geojson:0.9'
 
@@ -245,8 +263,7 @@ dependencies {
     ////// Test-only dependencies //////
 
     // Java unit testing framework.
-    testImplementation(platform('org.junit:junit-bom:5.7.0'))
-    testImplementation('org.junit.jupiter:junit-jupiter')
+    testImplementation('org.junit.jupiter:junit-jupiter:5.9.2')
 
     // Chart drawing library for examining travel time distributions when crafting tests.
     // Although rarely used it should be low-impact: it is a test-only dependency with no transitive dependenices.

src/main/java/com/conveyal/analysis/components/HttpApi.java

@@ -179,7 +179,7 @@ private void respondToException(Exception e, Request request, Response response,
         // Include a stack trace except when the error is known to be about unauthenticated or unauthorized access,
         // in which case we don't want to leak information about the server to people scanning it for weaknesses.
         if (type != UNAUTHORIZED && type != FORBIDDEN) {
-            body.put("stackTrace", errorEvent.stackTrace);
+            body.put("stackTrace", errorEvent.filteredStackTrace);
         }
         response.status(code);
         response.type("application/json");