push-github-action.yml

# Useful documentation for GitHub Actions workflows:
# https://docs.github.com/en/actions/using-workflows/about-workflows

name: push-github-action

# Useful documentation for `on`:
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#on
on:
  # Run CI whenever a push is made to any branch
  push: null
  # Run CI on the tip of any open pull request
  pull_request: null
  # Allow CI to be run manually in GitHub Actions UI
  workflow_dispatch: null

# 1. Test Crystal binary on each supported OS
# 2. Test Crystal with Address Sanitizer on macOS specially
#    because segfaults more common on macOS
jobs:
  
  build-macos-asan:
    strategy:
      matrix:
        # NOTE: Test lots of Python versions to look for segfaults
        python-version:
          - "3.8.18"
          - "3.9.19"
          # NOTE: Disable runs on some Python versions so that total number
          #       of macOS CI runners is low enough to not need to wait for
          #       available runners for very long
          #- "3.10.14"
          - "3.11.9"
      fail-fast: false
    runs-on: macos-12
    env:
      # Suppress warning "malloc: nano zone abandoned due to inability to preallocate reserved vm space"
      # from macOS being confused by Address Sanitizer's changes to malloc.
      # https://stackoverflow.com/questions/64126942/malloc-nano-zone-abandoned-due-to-inability-to-preallocate-reserved-vm-space
      MallocNanoZone: 0
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      
      - name: Restore cached Python and virtual environment
        id: restore-cache
        uses: actions/cache@v4
        with:
          path: |
            cpython
            venv_asan
          key: build-macos-asan-python-${{ matrix.python-version }}
      
      - name: Compile Python ${{ matrix.python-version }} with Address Sanitizer
        if: steps.restore-cache.outputs.cache-hit != 'true'
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
        run: |
          wget --no-verbose https://github.com/python/cpython/archive/refs/tags/v$PYTHON_VERSION.zip
          unzip -q v3.*.zip
          mv cpython-3.* cpython
          cd cpython
          ./configure --with-pydebug --with-address-sanitizer
          make -s -j3
          ./python.exe -c 'print("OK")'
      
      - name: Create virtual environment
        if: steps.restore-cache.outputs.cache-hit != 'true'
        run: |
          cpython/python.exe -m venv venv_asan
      
      - name: Install Poetry
        # NOTE: Keeping pip and setuptools updated avoids error: Backend 'setuptools.build_meta:__legacy__' is not available
        #       https://github.com/pypa/pip/issues/6264
        # NOTE: Poetry >=1.5.0 on macOS cannot use "poetry run" to run make-mac.sh
        run: |
          python -m pip install -U pip setuptools
          python -m pip install -U "poetry>=1.4.0,<1.5.0"
      
      - name: Activate virtual environment
        run: |
          source venv_asan/bin/activate
          echo PATH=$PATH >> $GITHUB_ENV
          echo VIRTUAL_ENV=$VIRTUAL_ENV >> $GITHUB_ENV
      
      - name: Install dependencies with Poetry
        # If build takes a very long time, then it's likely that the version
        # of wxPython installed does not offer a precompiled wheel for this
        # version of Python. Check the wxPython PyPI page to confirm.
        timeout-minutes: 2  # normally takes 6s, as of 2023-03-03
        run: poetry install
      
      - name: Run non-UI tests
        run: poetry run python -m pytest
      
      - name: Run UI tests
        id: run_ui_tests
        env:
          CRYSTAL_FAULTHANDLER: 'True'
          CRYSTAL_ADDRESS_SANITIZER: 'True'
          # Multiply timeouts because ASan builds of Python are slower than regular builds
          CRYSTAL_GLOBAL_TIMEOUT_MULTIPLIER: '2.0'
        run: |
          crystal --test

  build-macos:
    strategy:
      matrix:
        os:
          # NOTE: Earliest macOS supported by Crystal is macOS 10.14
          #- macos-10.14  # no longer supported by GitHub Actions
          #- macos-10.15  # no longer supported by GitHub Actions
          #- macos-11  # no longer supported by GitHub Actions after 6/28/2024
          - macos-12   # aligns with OS of Cathode, enabling core dump debugging
          #- macos-13
          #- macos-14  # ARM-based; forces earlier incompatible Python version
        # Test the earliest supported Python version only
        python-version:
          - "3.8.18"
          #- "3.9.19"
      fail-fast: false
    runs-on: ${{ matrix.os }}
    timeout-minutes: 23  # 150% of normal time: 15 min, as of 2024-02-16
    env:
      # Enables capturing and uploading core dumps on macOS
      ENABLE_CORE_DUMPS: 'false'
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Poetry
        # NOTE: Keeping pip and setuptools updated avoids error: Backend 'setuptools.build_meta:__legacy__' is not available
        #       https://github.com/pypa/pip/issues/6264
        # NOTE: Poetry >=1.5.0 on macOS cannot use "poetry run" to run make-mac.sh
        run: |
          python -m pip install -U pip setuptools
          python -m pip install -U "poetry>=1.4.0,<1.5.0"

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: poetry

      - name: Install dependencies with Poetry
        # If build takes a very long time, then it's likely that the version
        # of wxPython installed does not offer a precompiled wheel for this
        # version of Python. Check the wxPython PyPI page to confirm.
        timeout-minutes: 2  # normally takes 6s, as of 2023-03-03
        run: poetry install

      - name: Display SQLite version and JSON support
        run: |
          python3 -c "import sqlite3; print('SQLite %s' % sqlite3.sqlite_version)"
          poetry run python -c "from crystal.util.xsqlite3 import sqlite_has_json_support; print('JSON Support: ' + ('yes' if sqlite_has_json_support else 'NO'))"

      - name: Run non-UI tests
        run: poetry run python -m pytest

      - name: Build .app and disk image
        working-directory: "./setup"
        # --app-only: Don't build disk image because it intermittently fails
        #             with "hdiutil create failed - Resource busy" in CI
        run: "CRYSTAL_SUPPORT_SCREENSHOTS=True poetry run ./make-mac.sh --app-only"
      
      - name: Enable core dumps
        if: env.ENABLE_CORE_DUMPS == 'true'
        run: |
          sudo chmod 1777 /cores
      
      - name: Run UI tests
        id: run_ui_tests
        env:
          CRYSTAL_FAULTHANDLER: 'True'
          # Force Crystal to print stdout and stderr rather than sending them to log files
          TERM: __interactive__
        run: |
          ulimit -c unlimited  # allow core dumps of unlimited size
          CRYSTAL_SCREENSHOTS_DIRPATH=$GITHUB_WORKSPACE/screenshots "setup/dist/Crystal Web Archiver.app/Contents/MacOS/Crystal Web Archiver" --test

      - name: Upload screenshot if test failure
        if: failure() && steps.run_ui_tests.outcome == 'failure'
        uses: actions/upload-artifact@v4
        with:
          name: screenshots-${{ matrix.os }}-${{ matrix.python-version }}
          path: ${{ github.workspace }}/screenshots/**/*
          if-no-files-found: ignore
      
      # If test failure then upload a core dump and the related Python binaries
      # inside GitHub Action's "hosted tool cache"
      - name: "Core dump: Upload if test failure"
        id: upload_core_dump
        if: env.ENABLE_CORE_DUMPS == 'true' && failure() && steps.run_ui_tests.outcome == 'failure'
        uses: actions/upload-artifact@v4
        with:
          name: coredump-${{ matrix.os }}-${{ matrix.python-version }}
          path: /cores
          if-no-files-found: error
        continue-on-error: true
      - name: "Core dump: Archive tool cache"
        if: env.ENABLE_CORE_DUMPS == 'true' && failure() && steps.upload_core_dump.outcome == 'success'
        run: |
          cd "${{ runner.tool_cache }}/Python"
          tar -czvf "${{ runner.temp }}/tool_cache_python.tar.gz" *
      - name: "Core dump: Upload tool cache artifact"
        if: env.ENABLE_CORE_DUMPS == 'true' && failure() && steps.upload_core_dump.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: tool_cache-python-${{ matrix.os }}-${{ matrix.python-version }}
          path: ${{runner.temp}}/tool_cache_python.tar.gz
          compression-level: 0  # no compression
      - name: "Core dump: Upload app artifact"
        if: env.ENABLE_CORE_DUMPS == 'true' && failure() && steps.upload_core_dump.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: setup-dist-${{ matrix.os }}-${{ matrix.python-version }}
          path: setup/dist
      
      # NOTE: Must remove the --app-only option from make-mac.sh above to
      #       reinstate build of *.dmg disk image
      #- name: Upload distribution artifact
      #  # Only export distribution artifact for earliest supported Python and OS
      #  if: (matrix.python-version == '3.8') && (matrix.os == 'macos-10.14')
      #  uses: actions/upload-artifact@v3
      #  with:
      #    name: dist-mac
      #    path: setup/dist-mac/*.dmg
      #    if-no-files-found: warn

  build-linux:
    strategy:
      matrix:
        os:
          # NOTE: Earliest Linux supported by Crystal is Ubuntu 22.04
          # NOTE: When adding new Linux versions, you may need
          #       to compile new wxPython .wgn files
          - ubuntu-22.04
        # Test the earliest supported Python version only
        python-version:
          - "3.8.18"
          #- "3.9.19"
      fail-fast: false
    runs-on: ${{ matrix.os }}
    timeout-minutes: 9  # 150% of normal time: 5 min 29 sec, as of 2024-02-24
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Poetry
        # NOTE: Keeping pip and setuptools updated avoids error: Backend 'setuptools.build_meta:__legacy__' is not available
        #       https://github.com/pypa/pip/issues/6264
        run: |
          python -m pip install -U pip setuptools
          python -m pip install -U "poetry>=1.4.0,<2.0.0"

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: poetry

      - name: Update APT packages
        run: sudo apt-get update

      # HACK: Suppress warning "Error retrieving accessibility bus
      #       address: org.freedesktop.DBus.Error.ServiceUnknown:
      #       The name org.a11y.Bus was not provided by any .service files"
      #       when running tests later
      - name: Install at-spi2-core
        run: sudo apt-get install -y at-spi2-core
      
      # NOTE: Needed for screenshot support while running tests
      - name: Install scrot
        run: sudo apt-get install scrot

      - name: Install wxPython dependencies
        run: sudo apt-get install -y libgtk-3-dev

      # Install wxPython from precompiled wagon because installing
      # wxPython from source takes about 40 minutes on GitHub Actions
      # 
      # NOTE: To recompile the .wgn, see instructions in: doc/how_to_make_wxpython_wagon.md
      - name: Install dependency wxPython from wagon (Python 3.8)
        if: startsWith(matrix.python-version, '3.8.')
        run: |
          poetry run pip3 install wagon
          wget --no-verbose https://github.com/davidfstr/Crystal-Web-Archiver/releases/download/v1.4.0b/wxPython-4.2.1-py38-none-linux_x86_64.wgn
          poetry run wagon install wxPython-4.2.1-py38-none-linux_x86_64.wgn
      - name: Install dependency wxPython from wagon (Python 3.9)
        if: startsWith(matrix.python-version, '3.9.')
        run: |
          poetry run pip3 install wagon
          wget --no-verbose https://github.com/davidfstr/Crystal-Web-Archiver/releases/download/v1.4.0b/wxPython-4.2.1-py39-none-linux_x86_64.wgn
          poetry run wagon install wxPython-4.2.1-py39-none-linux_x86_64.wgn

      - name: Install remaining dependencies with Poetry
        # If build takes a very long time, the precompiled .wgn in the 
        # previous build step may no longer be installing the correct
        # dependency versions that are consistent with pyproject.toml and
        # poetry.lock. In that case you'll need to rebuild the .wgn using
        # instructions from: doc/how_to_make_wxpython_wagon.md
        timeout-minutes: 2  # normally takes 6s, as of 2023-03-03
        run: poetry install
 
      - name: Display SQLite version and JSON support
        run: |
          python3 -c "import sqlite3; print('SQLite %s' % sqlite3.sqlite_version)"
          poetry run python -c "from crystal.util.xsqlite3 import sqlite_has_json_support; print('JSON Support: ' + ('yes' if sqlite_has_json_support else 'NO'))"

      - name: Run non-UI tests
        run: poetry run python -m pytest

      - name: Run UI tests
        env:
          CRYSTAL_FAULTHANDLER: 'True'
          # Enable a version of malloc() that looks for heap corruption specially:
          # https://stackoverflow.com/a/3718867/604063
          MALLOC_CHECK_: '2'
        run: |
          CRYSTAL_SCREENSHOTS_DIRPATH=$GITHUB_WORKSPACE/screenshots poetry run xvfb-run crystal --test

      - name: Upload screenshot if test failure
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: screenshots
          path: screenshots
          if-no-files-found: ignore

  build-windows:
    strategy:
      matrix:
        os:
          # NOTE: Earliest Windows supported by Crystal is Windows 7
          #- windows-7  # available only as a self-hosted runner
          #- windows-2012-r2  # based on Windows 7  # no longer supported by GitHub Actions
          #- windows-2016  # based on Windows 8.1  # no longer supported by GitHub Actions
          - windows-2019  # based on Windows 10
          #- windows-latest
        # Test the earliest supported Python version only
        python-version:
          - "3.8.10"
          #- "3.9.13"
      fail-fast: false
    runs-on: ${{ matrix.os }}
    timeout-minutes: 35  # 150% of normal time: 23 min, as of 2024-02-22
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Poetry
        # NOTE: Keeping pip and setuptools updated avoids error: Backend 'setuptools.build_meta:__legacy__' is not available
        #       https://github.com/pypa/pip/issues/6264
        run: |
          python -m pip install -U pip setuptools
          python -m pip install -U "poetry>=1.4.0,<2.0.0"

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: poetry
      
      - name: Install Poetry (again)
        # NOTE: Keeping pip and setuptools updated avoids error: Backend 'setuptools.build_meta:__legacy__' is not available
        #       https://github.com/pypa/pip/issues/6264
        run: |
          python -m pip install -U pip setuptools
          python -m pip install -U "poetry>=1.4.0,<2.0.0"

      - name: Install dependencies with Poetry
        # If build takes a very long time, then it's likely that the version
        # of wxPython installed does not offer a precompiled wheel for this
        # version of Python. Check the wxPython PyPI page to confirm.
        timeout-minutes: 2  # normally takes 15s, as of 2023-03-03
        run: poetry install

      - name: Display SQLite version and JSON support
        run: |
          python3 -c "import sqlite3; print('SQLite %s' % sqlite3.sqlite_version)"
          poetry run python -c "from crystal.util.xsqlite3 import sqlite_has_json_support; print('JSON Support: ' + ('yes' if sqlite_has_json_support else 'NO'))"

      - name: Run non-UI tests
        run: poetry run python -m pytest

      - name: Display Inno Setup in Program Files
        run: |
          dir "C:\Program Files (x86)"
          dir "C:\Program Files (x86)\Inno Setup 6"

      - name: Build .exe and installer
        working-directory: ".\\setup"
        run: "poetry run .\\make-win.bat"

      - name: Run UI tests
        working-directory: ".\\setup"
        run: |
          $LOGDIR = "$HOME\AppData\Local\DaFoster\Crystal Web Archiver\Logs"
          $env:CRYSTAL_SCREENSHOTS_DIRPATH = "$env:GITHUB_WORKSPACE\screenshots"
          $env:CRYSTAL_FAULTHANDLER = "True"
          poetry run python run_exe.py "--argsfile=arguments.txt" "--stdoutfile=$LOGDIR\stdout.log" "--stderrfile=$LOGDIR\stderr.log" "dist\Crystal Web Archiver.exe" "---" "--test"

      - name: Upload screenshot if test failure
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: screenshots
          path: screenshots
          if-no-files-found: ignore
      
      - name: Upload distribution artifact
        # Only export distribution artifact for earliest supported Python and OS
        if: (matrix.python-version == '3.8') && (matrix.os == 'windows-7')
        uses: actions/upload-artifact@v3
        with:
          name: dist-win
          path: "setup\\dist-win\\*.exe"
          if-no-files-found: warn