nightly-build.yaml

name: Nightly Build
on:
  schedule:
    - cron: '0 16 * * *'
  workflow_dispatch:
jobs:
  build-and-test:
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        modules:
          - dl-on-flink-framework
          - dl-on-flink-operator
          - dl-on-flink-tensorflow-common
          - dl-on-flink-tensorflow
          - dl-on-flink-tensorflow-2.x
          - dl-on-flink-pytorch
          - dl-on-flink-lib
          - dl-on-flink-examples
          - dl-on-flink-examples/dl-on-flink-examples-pytorch
          - dl-on-flink-examples/dl-on-flink-examples-tensorflow
          - dl-on-flink-examples/dl-on-flink-examples-tensorflow-2.x
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Set up JDK 8
        uses: actions/setup-java@v2
        with:
          java-version: '8'
          distribution: 'adopt'
      - name: Set up python 3.7
        uses: actions/setup-python@v2
        with:
          # Semantic version range syntax or exact version of a Python version
          python-version: '3.7'
          # Optional - x64 or x86 architecture, defaults to x64
          architecture: 'x64'
      - name: Install Python dependencies
        run: |
          pip install -U --user wheel
      - name: Cache Maven packages
        uses: actions/cache@v2
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - name: build
        run: |
          mvn -B install -pl ${{ matrix.modules }} -am -DskipTests -Dfast -Pinstall-py-dep
      - name: test
        env:
          TF_ON_FLINK_IP: 127.0.0.1
        run: |
          mvn -B test -pl ${{ matrix.modules }}
  build-and-upload-bin:
    needs: build-and-test
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Maven packages
        uses: actions/cache@v2
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - name: Set up python 3.7
        uses: actions/setup-python@v2
        with:
          # Semantic version range syntax or exact version of a Python version
          python-version: '3.7'
          # Optional - x64 or x86 architecture, defaults to x64
          architecture: 'x64'
      - name: Set up JDK 8
        uses: actions/setup-java@v2
        with:
          java-version: '8'
          distribution: 'adopt'
      - name: Build bin
        run: mvn -B clean package -DskipTests
      - uses: actions/upload-artifact@v2
        with:
          name: dl-on-flink-dist
          path: dl-on-flink-dist/target/dl-on-flink-dist-*-bin.tgz
  build-and-test-python:
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        modules:
          - dl-on-flink-framework
          - dl-on-flink-tensorflow
          - dl-on-flink-tensorflow-2.x
          - dl-on-flink-pytorch
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Set up JDK 8
        uses: actions/setup-java@v2
        with:
          java-version: '8'
          distribution: 'adopt'
      - name: Set up python 3.7
        uses: actions/setup-python@v2
        with:
          # Semantic version range syntax or exact version of a Python version
          python-version: '3.7'
          # Optional - x64 or x86 architecture, defaults to x64
          architecture: 'x64'
      - name: Install Python dependencies
        run: |
          pip install -U --user wheel
      - name: Cache Maven packages
        uses: actions/cache@v2
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - name: build
        run: |
          mvn -B install -pl ${{ matrix.modules }} -am -DskipTests -Dfast -Pinstall-py-dep
      - name: test
        run: |
          cd ${{ matrix.modules }}/python
          pip install -e .
          python -m unittest discover -v .
  build-pytorch-nightly-wheel:
    needs: build-and-test-python
    name: Build ${{ matrix.package }} wheels on ubuntu-20.04
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        package:
          - dl-on-flink-pytorch
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Set up python 3.7
        uses: actions/setup-python@v2
        with:
          # Semantic version range syntax or exact version of a Python version
          python-version: '3.7'
          # Optional - x64 or x86 architecture, defaults to x64
          architecture: 'x64'
      - name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels
        env:
          NIGHTLY_WHEEL: true
        run: |
          pip install wheel
          cd ${{ matrix.package }}/python
          python setup.py bdist_wheel
      - uses: actions/upload-artifact@v2
        with:
          name: dl-on-flink-wheels
          path: ./dl-on-flink-pytorch/python/dist/*.whl
  build-linux-nightly-wheel:
    needs: build-and-test-python
    name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels on ubuntu-20.04
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.7, 3.8 ]
        package:
          - dl-on-flink-framework
          - dl-on-flink-tensorflow
          - dl-on-flink-tensorflow-2.x
        exclude:
          - package: dl-on-flink-tensorflow
            python-version: 3.8
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels
        env:
          NIGHTLY_WHEEL: true
        run: |
          docker run -i --rm -v $PWD:/v -w /v -e 'NIGHTLY_WHEEL=true' --net=host quay.io/pypa/manylinux2010_x86_64 \
            bash -c 'cd /v/${{ matrix.package }}/python \
              && python${{ matrix.python-version }} setup.py bdist_wheel \
              && for f in dist/*.whl; do echo repairing $f && bash -x -e /v/tools/auditwheel repair --plat manylinux2010_x86_64 -w /v/wheelhouse/ $f; done'
      - uses: actions/upload-artifact@v2
        with:
          name: dl-on-flink-wheels
          path: ./wheelhouse/*.whl
  build-macos-nightly-wheel:
    needs: build-and-test-python
    name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels on macOS-10.15
    runs-on: macos-10.15
    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.7, 3.8 ]
        package:
          - dl-on-flink-framework
          - dl-on-flink-tensorflow
          - dl-on-flink-tensorflow-2.x
        exclude:
          - package: dl-on-flink-tensorflow
            python-version: 3.8
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels on macOS-10.15
        env:
          NIGHTLY_WHEEL: true
        run: |
          pip install wheel auditwheel
          export ROOT_DIR=$PWD
          mkdir $ROOT_DIR/wheelhouse
          cd ${{ matrix.package }}/python
          python setup.py bdist_wheel
          cp dist/* $ROOT_DIR/wheelhouse
      - uses: actions/upload-artifact@v2
        with:
          name: dl-on-flink-wheels
          path: ./wheelhouse/*.whl
  test-wheel:
    name: Test python${{ matrix.python-version }} ${{ matrix.package }} wheels on ${{ matrix.os }}
    needs: [ build-linux-nightly-wheel, build-macos-nightly-wheel, build-pytorch-nightly-wheel ]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ macOS-10.15, ubuntu-20.04 ]
        python-version: [ 3.7, 3.8 ]
        package:
          - dl-on-flink-framework
          - dl-on-flink-tensorflow
          - dl-on-flink-tensorflow-2.x
          - dl-on-flink-pytorch
        exclude:
          - package: dl-on-flink-tensorflow
            python-version: 3.8
          - package: dl-on-flink-tensorflow-2.x
            python-version: 3.6
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Set up JDK 8
        uses: actions/setup-java@v2
        with:
          java-version: '8'
          distribution: 'adopt'
      - uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/download-artifact@v2
        with:
          name: dl-on-flink-wheels
          path: ./wheelhouse
      - name: build
        run: |
          mvn -B install -pl ${{ matrix.package }} -am -DskipTests
      - name: test
        env:
          TF_ON_FLINK_IP: 127.0.0.1
        run: |
          ls $PWD/wheelhouse
          pip install --pre ${{ matrix.package }} -f file://${PWD}/wheelhouse
          mvn -B test -pl ${{ matrix.package }}
  run-example:
    needs: [ build-and-upload-bin, build-linux-nightly-wheel, build-macos-nightly-wheel, build-pytorch-nightly-wheel ]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ macos-10.15, ubuntu-18.04, ubuntu-20.04 ]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Set up JDK 8
        uses: actions/setup-java@v2
        with:
          java-version: '8'
          distribution: 'adopt'
      - uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - uses: actions/download-artifact@v2
        with:
          name: dl-on-flink-wheels
          path: ./wheelhouse
      - uses: actions/download-artifact@v2
        with:
          name: dl-on-flink-dist
      - name: Install coreutil on Mac
        if: ${{ matrix.os == 'macos-10.15' }}
        run: brew install coreutils
      - name: Run example on ${{ matrix.os }}
        run: |
          bash tools/run_example.sh dl-on-flink-*-bin.tgz wheelhouse
      - uses: actions/upload-artifact@v2
        if: always()
        with:
          name: e2e-example-logs
          path: ./flink-*/log
  deploy-snapshot:
    if: github.repository == 'flink-extended/dl-on-flink'
    needs: [ run-example, test-wheel ]
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Maven packages
        uses: actions/cache@v2
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - name: Set up python 3.7
        uses: actions/setup-python@v2
        with:
          # Semantic version range syntax or exact version of a Python version
          python-version: '3.7'
          # Optional - x64 or x86 architecture, defaults to x64
          architecture: 'x64'
      - name: Install Python dependencies
        run: |
          pip install -U --user wheel
      - name: Set up JDK 8
        uses: actions/setup-java@v2
        with:
          java-version: '8'
          distribution: 'adopt'
          server-id: oss_id
          server-username: MAVEN_USERNAME
          server-password: MAVEN_PASSWORD
          gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
          gpg-passphrase: MAVEN_GPG_PASSPHRASE
      - name: Publish snapshot
        run: mvn -B clean deploy -DskipTests -Prelease
        env:
          MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }}
          MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
          MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
      - uses: actions/upload-artifact@v2
        with:
          name: dl-on-flink-dist
          path: dl-on-flink-dist/target/dl-on-flink-dist-*-bin.tgz