diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 000000000..e0702e703 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,42 @@ +name: Create and publish a Docker image +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images + +on: + push: + branches: ['dev', 'master', 'dev-flex'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..4da74de30 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "dev", master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "dev" ] + schedule: + - cron: '22 22 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'java', 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3054ca7d5..1e68803fd 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -4,7 +4,10 @@ on: [push, pull_request] jobs: build: - + env: + # Only deploy JARs to S3 (and run semantic release) on push to dev or master, + # or if the commit message includes the specified "label". + SAVE_JAR_TO_S3: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'master' || contains(github.event.head_commit.message, '[save-jar]')) }} runs-on: ubuntu-latest services: postgres: @@ -24,15 +27,16 @@ jobs: --health-retries 5 steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - name: Set up JDK 19 + uses: actions/setup-java@v3 with: - java-version: 1.8 - # Install node 14 for running e2e tests (and for maven-semantic-release). - - name: Use Node.js 18.x + java-version: 19 + distribution: 'temurin' + # Install node 20 for running e2e tests (and for maven-semantic-release). + - name: Use Node.js 20.x uses: actions/setup-node@v1 with: - node-version: 18.x + node-version: 20.x - name: Start MongoDB uses: supercharge/mongodb-github-action@1.3.0 with: @@ -64,10 +68,7 @@ jobs: - name: Setup GTFS+ directory (used during testing) run: mkdir /tmp/gtfsplus - name: Build with Maven (run unit tests) - run: mvn --no-transfer-progress package - - name: Restart MongoDB with fresh database (for e2e tests) - if: env.SHOULD_RUN_E2E == 'true' - run: ./scripts/restart-mongo-with-fresh-db.sh + run: mvn --no-transfer-progress -X package - name: Run e2e tests if: env.SHOULD_RUN_E2E == 'true' run: mvn test @@ -97,15 +98,20 @@ jobs: # Run maven-semantic-release to potentially create a new release of datatools-server. The flag --skip-maven-deploy is # used to avoid deploying to maven central. So essentially, this just creates a release with a changelog on github. + - name: Use Node.js 20.x + uses: actions/setup-node@v1 + with: + node-version: 20.x - name: Run maven-semantic-release + if: env.SAVE_JAR_TO_S3 == 'true' env: GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy # The git commands get the commit hash of the HEAD commit and the commit just before HEAD. - name: Prepare deploy artifacts - # Only deploy on push (pull_request will deploy a temp. merge commit. See #400.) - if: github.event_name == 'push' + if: env.SAVE_JAR_TO_S3 == 'true' + # (pull_request will deploy a temp. merge commit. See #400.) run: | # get branch name of current branch for use in jar name export BRANCH=$GITHUB_REF_SLUG @@ -125,6 +131,6 @@ jobs: FIRST_JAR="${ALL_JARS[0]}" cp "$FIRST_JAR" "deploy/dt-latest-$BRANCH_CLEAN.jar" - name: Deploy to S3 - if: github.event_name == 'push' + if: env.SAVE_JAR_TO_S3 == 'true' run: | aws s3 cp ./deploy s3://datatools-builds --recursive --acl public-read diff --git a/.gitignore b/.gitignore index b04d422d8..c4fe5d5ac 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ tmp/ deploy/ # Configurations -configurations/* +configurations/*.yml !configurations/default !configurations/test diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..a64eddcd4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 +FROM maven:3.8.7-openjdk-18-slim + +COPY . /datatools + +# Build jar +WORKDIR /datatools +RUN mvn package -DskipTests +RUN cp target/dt*.jar /datatools/ +RUN mv dt*.jar datatools-server.jar + +RUN mkdir -p /var/datatools_gtfs/gtfsplus + +# Launch server +# This relies on a configuration volume and aws volume being present. See `docker-compose.yml`, or the example below +# Try: docker run --publish 4000:4000 -v ~/config/:/config datatools-latest +CMD ["java", "-XX:MaxRAMPercentage=95", "-jar", "datatools-server.jar", "/config/env.yml", "/config/server.yml"] +EXPOSE 4000 \ No newline at end of file diff --git a/README.md b/README.md index 53620da17..3be63c313 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,18 @@ # Transit Data Manager -The core application for IBI Group's transit data tools suite. +[![Join the chat at https://matrix.to/#/#transit-data-tools:gitter.im](https://badges.gitter.im/repo.png)](https://matrix.to/#/#transit-data-tools:gitter.im) + +The core application for IBI Group's TRANSIT-Data-Tools suite. ## Documentation View the [latest documentation](http://conveyal-data-tools.readthedocs.org/en/latest/) at ReadTheDocs. -Note: `dev` branch docs can be found [here](http://conveyal-data-tools.readthedocs.org/en/dev/). \ No newline at end of file +Note: `dev` branch docs can be found [here](http://conveyal-data-tools.readthedocs.org/en/dev/). + +## Docker Image +The easiest way to get `datatools-server` running is to use the provided `Dockerfile` and `docker-compose.yml`. The `docker-compose.yml` includes both database servers that are needed. Edit the supplied configurations in the `configurations` directory to ensure the server starts correctly. Once this is done running `docker-compose up` will start Datatools and all required database servers. + +## Getting in touch + +We have a Gitter [space](https://matrix.to/#/#transit-data-tools:gitter.im) for the full TRANSIT-Data-Tools project where you can post questions and comments. diff --git a/configurations/default/env.yml.tmp b/configurations/default/env.yml.tmp index 119b399a6..91e53f92b 100644 --- a/configurations/default/env.yml.tmp +++ b/configurations/default/env.yml.tmp @@ -12,8 +12,8 @@ DISABLE_AUTH: false OSM_VEX: http://localhost:1000 SPARKPOST_KEY: your-sparkpost-key SPARKPOST_EMAIL: email@example.com -GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue +GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue # If running via docker, this is jdbc:postgresql://postgres/dmtest # GTFS_DATABASE_USER: # GTFS_DATABASE_PASSWORD: -#MONGO_HOST: mongo-host:27017 +#MONGO_HOST: mongo-host:27017 # If running via docker, this is mongo:27017 MONGO_DB_NAME: catalogue diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index 20edb3008..74b0b2338 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -8,6 +8,7 @@ application: notifications_enabled: false docs_url: http://conveyal-data-tools.readthedocs.org support_email: support@ibigroup.com + public_gtfs_contact_email: public-gtfs-support-email@yourdomain.com port: 4000 data: gtfs: /tmp @@ -17,6 +18,8 @@ application: modules: enterprise: enabled: false + # Setting this to true will upload all feeds to S3 instead of linking to their URL + prefer_s3_links: false editor: enabled: true deployment: @@ -24,6 +27,8 @@ modules: ec2: enabled: false default_ami: ami-your-ami-id + tag_key: a-tag-key-to-add-to-all-instances + tag_value: a-tag-value-to-add-to-all-instances # Note: using a cloudfront URL for these download URLs will greatly # increase download/deploy speed. otp_download_url: https://optional-otp-repo.com diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..cc847068d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3.8' +services: + datatools-server: + build: ./ + ports: + - "4000:4000" + volumes: + - type: bind + source: ./configurations/default/ + target: /config + - type: bind + source: ~/.aws + target: /root/.aws + depends_on: + - mongo + - postgres + mongo: + image: mongo + restart: always + postgres: + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: root + POSTGRES_DB: dmtest + image: postgres + restart: always \ No newline at end of file diff --git a/pom.xml b/pom.xml index f9b95a751..a5ef5d82d 100644 --- a/pom.xml +++ b/pom.xml @@ -43,8 +43,8 @@ UTF-8 - 17.5 - 1.11.625 + 20.1 + 1.12.720 @@ -96,8 +96,8 @@ maven-compiler-plugin 3.7.0 - 1.8 - 1.8 + 11 + 11 @@ -163,10 +163,21 @@ maven-surefire-plugin 2.22.2 + + + + --illegal-access=permit + + + + + jitpack.io + https://jitpack.io + - - jitpack.io - https://jitpack.io - @@ -222,7 +228,7 @@ ch.qos.logback logback-classic - 1.2.3 + 1.2.13 @@ -253,16 +259,20 @@ 5.5.2 test - + + org.mobilitydata.gtfs-validator + gtfs-validator-main + 6.0.0 + - com.github.conveyal + com.github.ibi-group gtfs-lib - bdb76ee + a8a376c82b @@ -278,15 +288,21 @@ org.mongodb mongodb-driver-sync - 4.0.5 + 4.0.6 com.google.guava guava - 30.0-jre + 32.0.0-jre + + javax.xml.bind + jaxb-api + 2.3.1 + +