diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
new file mode 100644
index 000000000..e0702e703
--- /dev/null
+++ b/.github/workflows/build-docker.yml
@@ -0,0 +1,42 @@
+name: Create and publish a Docker image
+# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+
+on:
+ push:
+ branches: ['dev', 'master', 'dev-flex']
+
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+ build-and-push-image:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+ - name: Build and push Docker image
+ uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+ with:
+ context: .
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
\ No newline at end of file
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 000000000..4da74de30
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+ push:
+ branches: [ "dev", master ]
+ pull_request:
+ # The branches below must be a subset of the branches above
+ branches: [ "dev" ]
+ schedule:
+ - cron: '22 22 * * 2'
+
+jobs:
+ analyze:
+ name: Analyze
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+ strategy:
+ fail-fast: false
+ matrix:
+ language: [ 'java', 'javascript', 'python' ]
+ # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+ # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v2
+ with:
+ languages: ${{ matrix.language }}
+ # If you wish to specify custom queries, you can do so here or in a config file.
+ # By default, queries listed here will override any specified in a config file.
+ # Prefix the list here with "+" to use these queries and those in the config file.
+
+ # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+ # queries: security-extended,security-and-quality
+
+
+ # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+ # If this step fails, then you should remove it and run the build manually (see below)
+ - name: Autobuild
+ uses: github/codeql-action/autobuild@v2
+
+ # âšī¸ Command-line programs to run using the OS shell.
+ # đ See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+ # If the Autobuild fails above, remove it and uncomment the following three lines.
+ # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+ # - run: |
+ # echo "Run, Build Application using script"
+ # ./location_of_script_within_repo/buildscript.sh
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index 3054ca7d5..1e68803fd 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -4,7 +4,10 @@ on: [push, pull_request]
jobs:
build:
-
+ env:
+ # Only deploy JARs to S3 (and run semantic release) on push to dev or master,
+ # or if the commit message includes the specified "label".
+ SAVE_JAR_TO_S3: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'master' || contains(github.event.head_commit.message, '[save-jar]')) }}
runs-on: ubuntu-latest
services:
postgres:
@@ -24,15 +27,16 @@ jobs:
--health-retries 5
steps:
- uses: actions/checkout@v2
- - name: Set up JDK 1.8
- uses: actions/setup-java@v1
+ - name: Set up JDK 19
+ uses: actions/setup-java@v3
with:
- java-version: 1.8
- # Install node 14 for running e2e tests (and for maven-semantic-release).
- - name: Use Node.js 18.x
+ java-version: 19
+ distribution: 'temurin'
+ # Install node 20 for running e2e tests (and for maven-semantic-release).
+ - name: Use Node.js 20.x
uses: actions/setup-node@v1
with:
- node-version: 18.x
+ node-version: 20.x
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.3.0
with:
@@ -64,10 +68,7 @@ jobs:
- name: Setup GTFS+ directory (used during testing)
run: mkdir /tmp/gtfsplus
- name: Build with Maven (run unit tests)
- run: mvn --no-transfer-progress package
- - name: Restart MongoDB with fresh database (for e2e tests)
- if: env.SHOULD_RUN_E2E == 'true'
- run: ./scripts/restart-mongo-with-fresh-db.sh
+ run: mvn --no-transfer-progress -X package
- name: Run e2e tests
if: env.SHOULD_RUN_E2E == 'true'
run: mvn test
@@ -97,15 +98,20 @@ jobs:
# Run maven-semantic-release to potentially create a new release of datatools-server. The flag --skip-maven-deploy is
# used to avoid deploying to maven central. So essentially, this just creates a release with a changelog on github.
+ - name: Use Node.js 20.x
+ uses: actions/setup-node@v1
+ with:
+ node-version: 20.x
- name: Run maven-semantic-release
+ if: env.SAVE_JAR_TO_S3 == 'true'
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy
# The git commands get the commit hash of the HEAD commit and the commit just before HEAD.
- name: Prepare deploy artifacts
- # Only deploy on push (pull_request will deploy a temp. merge commit. See #400.)
- if: github.event_name == 'push'
+ if: env.SAVE_JAR_TO_S3 == 'true'
+ # (pull_request will deploy a temp. merge commit. See #400.)
run: |
# get branch name of current branch for use in jar name
export BRANCH=$GITHUB_REF_SLUG
@@ -125,6 +131,6 @@ jobs:
FIRST_JAR="${ALL_JARS[0]}"
cp "$FIRST_JAR" "deploy/dt-latest-$BRANCH_CLEAN.jar"
- name: Deploy to S3
- if: github.event_name == 'push'
+ if: env.SAVE_JAR_TO_S3 == 'true'
run: |
aws s3 cp ./deploy s3://datatools-builds --recursive --acl public-read
diff --git a/.gitignore b/.gitignore
index b04d422d8..c4fe5d5ac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,7 +14,7 @@ tmp/
deploy/
# Configurations
-configurations/*
+configurations/*.yml
!configurations/default
!configurations/test
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000..a64eddcd4
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1
+FROM maven:3.8.7-openjdk-18-slim
+
+COPY . /datatools
+
+# Build jar
+WORKDIR /datatools
+RUN mvn package -DskipTests
+RUN cp target/dt*.jar /datatools/
+RUN mv dt*.jar datatools-server.jar
+
+RUN mkdir -p /var/datatools_gtfs/gtfsplus
+
+# Launch server
+# This relies on a configuration volume and aws volume being present. See `docker-compose.yml`, or the example below
+# Try: docker run --publish 4000:4000 -v ~/config/:/config datatools-latest
+CMD ["java", "-XX:MaxRAMPercentage=95", "-jar", "datatools-server.jar", "/config/env.yml", "/config/server.yml"]
+EXPOSE 4000
\ No newline at end of file
diff --git a/README.md b/README.md
index 53620da17..3be63c313 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,18 @@
# Transit Data Manager
-The core application for IBI Group's transit data tools suite.
+[![Join the chat at https://matrix.to/#/#transit-data-tools:gitter.im](https://badges.gitter.im/repo.png)](https://matrix.to/#/#transit-data-tools:gitter.im)
+
+The core application for IBI Group's TRANSIT-Data-Tools suite.
## Documentation
View the [latest documentation](http://conveyal-data-tools.readthedocs.org/en/latest/) at ReadTheDocs.
-Note: `dev` branch docs can be found [here](http://conveyal-data-tools.readthedocs.org/en/dev/).
\ No newline at end of file
+Note: `dev` branch docs can be found [here](http://conveyal-data-tools.readthedocs.org/en/dev/).
+
+## Docker Image
+The easiest way to get `datatools-server` running is to use the provided `Dockerfile` and `docker-compose.yml`. The `docker-compose.yml` includes both database servers that are needed. Edit the supplied configurations in the `configurations` directory to ensure the server starts correctly. Once this is done running `docker-compose up` will start Datatools and all required database servers.
+
+## Getting in touch
+
+We have a Gitter [space](https://matrix.to/#/#transit-data-tools:gitter.im) for the full TRANSIT-Data-Tools project where you can post questions and comments.
diff --git a/configurations/default/env.yml.tmp b/configurations/default/env.yml.tmp
index 119b399a6..91e53f92b 100644
--- a/configurations/default/env.yml.tmp
+++ b/configurations/default/env.yml.tmp
@@ -12,8 +12,8 @@ DISABLE_AUTH: false
OSM_VEX: http://localhost:1000
SPARKPOST_KEY: your-sparkpost-key
SPARKPOST_EMAIL: email@example.com
-GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue
+GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue # If running via docker, this is jdbc:postgresql://postgres/dmtest
# GTFS_DATABASE_USER:
# GTFS_DATABASE_PASSWORD:
-#MONGO_HOST: mongo-host:27017
+#MONGO_HOST: mongo-host:27017 # If running via docker, this is mongo:27017
MONGO_DB_NAME: catalogue
diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp
index 20edb3008..74b0b2338 100644
--- a/configurations/default/server.yml.tmp
+++ b/configurations/default/server.yml.tmp
@@ -8,6 +8,7 @@ application:
notifications_enabled: false
docs_url: http://conveyal-data-tools.readthedocs.org
support_email: support@ibigroup.com
+ public_gtfs_contact_email: public-gtfs-support-email@yourdomain.com
port: 4000
data:
gtfs: /tmp
@@ -17,6 +18,8 @@ application:
modules:
enterprise:
enabled: false
+ # Setting this to true will upload all feeds to S3 instead of linking to their URL
+ prefer_s3_links: false
editor:
enabled: true
deployment:
@@ -24,6 +27,8 @@ modules:
ec2:
enabled: false
default_ami: ami-your-ami-id
+ tag_key: a-tag-key-to-add-to-all-instances
+ tag_value: a-tag-value-to-add-to-all-instances
# Note: using a cloudfront URL for these download URLs will greatly
# increase download/deploy speed.
otp_download_url: https://optional-otp-repo.com
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 000000000..cc847068d
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,26 @@
+version: '3.8'
+services:
+ datatools-server:
+ build: ./
+ ports:
+ - "4000:4000"
+ volumes:
+ - type: bind
+ source: ./configurations/default/
+ target: /config
+ - type: bind
+ source: ~/.aws
+ target: /root/.aws
+ depends_on:
+ - mongo
+ - postgres
+ mongo:
+ image: mongo
+ restart: always
+ postgres:
+ environment:
+ POSTGRES_HOST_AUTH_METHOD: trust
+ POSTGRES_USER: root
+ POSTGRES_DB: dmtest
+ image: postgres
+ restart: always
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index f9b95a751..a5ef5d82d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -43,8 +43,8 @@
UTF-8
- 17.5
- 1.11.625
+ 20.1
+ 1.12.720
@@ -96,8 +96,8 @@
maven-compiler-plugin
3.7.0
-
- 1.8
+
+ 11
@@ -163,10 +163,21 @@
maven-surefire-plugin
2.22.2
+
+
+
+ --illegal-access=permit
+
+
+
+
+ jitpack.io
+ https://jitpack.io
+
-
- jitpack.io
- https://jitpack.io
-
@@ -222,7 +228,7 @@
ch.qos.logback
logback-classic
- 1.2.3
+ 1.2.13
@@ -253,16 +259,20 @@
5.5.2
test
-
+
+ org.mobilitydata.gtfs-validator
+ gtfs-validator-main
+ 6.0.0
+
- com.github.conveyal
+ com.github.ibi-group
gtfs-lib
- bdb76ee
+ a8a376c82b
@@ -278,15 +288,21 @@
org.mongodb
mongodb-driver-sync
- 4.0.5
+ 4.0.6
com.google.guava
guava
- 30.0-jre
+ 32.0.0-jre
+
+ javax.xml.bind
+ jaxb-api
+ 2.3.1
+
+