interop-test-suite.yml

name: SOP interoperability test suite

on:
  pull_request:
    branches: [ main ]

jobs:
  build-gosop-v1:
    name: Build gosop from branch v1-api
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Build gosop from branch
        uses: ./.github/actions/build-gosop
        with: 
          binary-location: ./gosop-${{ github.sha }}-v1
      # Upload as artifact
      - name: Upload gosop artifact
        uses: actions/upload-artifact@v3
        with:
          name: gosop-${{ github.sha }}-v1
          path: ./gosop-${{ github.sha }}-v1
        
  build-gosop-v2:
    name: Build gosop from branch v2-api
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Build gosop from branch
        uses: ./.github/actions/build-gosop
        with: 
          binary-location: ./gosop-${{ github.sha }}-v2
          branch-gosop: gosop-gopenpgp-v3
          gosop-build-path: build_gosop.sh
      # Upload as artifact
      - name: Upload gosop artifact
        uses: actions/upload-artifact@v3
        with:
          name: gosop-${{ github.sha }}-v2
          path: ./gosop-${{ github.sha }}-v2

  build-gosop-main:
    name: Build gosop from main
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Build gosop from branch
        uses: ./.github/actions/build-gosop
        with: 
          go-crypto-ref: main
          binary-location: ./gosop-main
      # Upload as artifact
      - name: Upload gosop-main artifact
        uses: actions/upload-artifact@v3
        with:
          name: gosop-main
          path: ./gosop-main

  test-suite:
    name: Run interoperability test suite
    runs-on: ubuntu-latest
    container: 
      image: ghcr.io/protonmail/openpgp-interop-test-docker:v.1.1.3
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.github_token }}
    needs: 
      - build-gosop-v1
      - build-gosop-v2
      - build-gosop-main
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      # Fetch gosop from main
      - name: Download gosop-main
        uses: actions/download-artifact@v3
        with:
          name: gosop-main
      # Test gosop-main
      - name: Make gosop-main executable
        run: chmod +x gosop-main
      - name: Print gosop-main version
        run: ./gosop-main version --extended
      # Fetch gosop from branch v1
      - name: Download gosop-branch-v1
        uses: actions/download-artifact@v3
        with:
          name: gosop-${{ github.sha }}-v1
      - name: Rename gosop-branch-v1
        run: mv gosop-${{ github.sha }}-v1 gosop-branch-v1
      # Test gosop-branch v1
      - name: Make gosop-branch-v1 executable
        run: chmod +x gosop-branch-v1
      - name: Print gosop-branch-v1 version
        run: ./gosop-branch-v1 version --extended
      # Fetch gosop from branch v2
      - name: Download gosop-branch-v2
        uses: actions/download-artifact@v3
        with:
          name: gosop-${{ github.sha }}-v2
      - name: Rename gosop-branch-v2
        run: mv gosop-${{ github.sha }}-v2 gosop-branch-v2
      # Test gosop-branch v2
      - name: Make gosop-branch-v2 executable
        run: chmod +x gosop-branch-v2
      - name: Print gosop-branch-v2 version
        run: ./gosop-branch-v2 version --extended
      # Run test suite
      - name: Prepare test configuration
        run: ./.github/test-suite/prepare_config.sh $CONFIG_TEMPLATE $CONFIG_OUTPUT $GITHUB_WORKSPACE/gosop-branch-v1 $GITHUB_WORKSPACE/gosop-branch-v2 $GITHUB_WORKSPACE/gosop-main
        env:
         CONFIG_TEMPLATE: .github/test-suite/config.json.template
         CONFIG_OUTPUT: .github/test-suite/config.json
      - name: Display configuration
        run: cat .github/test-suite/config.json
      - name: Run interoperability test suite
        run: cd $TEST_SUITE_DIR && $TEST_SUITE --config $GITHUB_WORKSPACE/$CONFIG --json-out $GITHUB_WORKSPACE/$RESULTS_JSON --html-out $GITHUB_WORKSPACE/$RESULTS_HTML
        env:
         CONFIG: .github/test-suite/config.json
         RESULTS_JSON: .github/test-suite/test-suite-results.json
         RESULTS_HTML: .github/test-suite/test-suite-results.html
      # Upload results
      - name: Upload test results json artifact
        uses: actions/upload-artifact@v3
        with:
          name: test-suite-results.json
          path: .github/test-suite/test-suite-results.json
      - name: Upload test results html artifact
        uses: actions/upload-artifact@v3
        with:
          name: test-suite-results.html
          path: .github/test-suite/test-suite-results.html

  compare-with-main:
    name: Compare with main
    runs-on: ubuntu-latest
    needs: test-suite
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Download test results json artifact
        id: download-test-results
        uses: actions/download-artifact@v3
        with:
          name: test-suite-results.json
      - name: Compare with baseline v1
        uses: ProtonMail/openpgp-interop-test-analyzer@5d7f4b6868ebe3bfc909302828342c461f5f4940
        with: 
          results: ${{ steps.download-test-results.outputs.download-path }}/test-suite-results.json
          output: baseline-comparison-v1.json
          baseline: gosop-main
          target: gosop-branch-v1
      - name: Compare with baseline v2
        uses: ProtonMail/openpgp-interop-test-analyzer@5d7f4b6868ebe3bfc909302828342c461f5f4940
        with: 
          results: ${{ steps.download-test-results.outputs.download-path }}/test-suite-results.json
          output: baseline-comparison-v2.json
          baseline: gosop-main
          target: gosop-branch-v2