diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index 1a5a5c2d7..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh new file mode 100755 index 000000000..022df6a8a --- /dev/null +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e +set -e + + +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file +else + sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file +fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1e058aa49..699d45391 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,6 +10,8 @@ name: Adapter Integration Tests +run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}" + on: push: branches: @@ -24,10 +26,31 @@ on: workflow_dispatch: inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" + name: + description: "Name to associate with run (example: 'dbt-adapters-242')" required: false type: string + default: "Adapter Integration Tests" + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: "main" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: "main" + dbt_core_branch: + description: "The branch of dbt-core to use" + type: string + required: false + default: "main" + dbt_common_branch: + description: "The branch of dbt-common to use" + type: string + required: false + default: "main" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -74,12 +97,19 @@ jobs: DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' + - name: Check out the repository (push) + if: github.event_name == 'push' uses: actions/checkout@v4 with: persist-credentials: false + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ inputs.adapter_branch }} + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request` event - name: Check out the repository (PR) @@ -95,17 +125,20 @@ jobs: with: python-version: "3.11" + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: github.event_name == 'workflow_dispatch' + run: | + ./.github/scripts/update_dev_dependency_branches.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} + cat dev-requirements.txt + - name: Install python dependencies run: | python -m pip install --user --upgrade pip python -m pip --version python -m pip install -r dagger/requirements.txt - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - name: Run tests for ${{ matrix.test }} run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }} diff --git a/dagger/spark-container/install_spark.sh b/dagger/spark-container/install_spark.sh index 476f362a9..a9816020d 100755 --- a/dagger/spark-container/install_spark.sh +++ b/dagger/spark-container/install_spark.sh @@ -4,7 +4,7 @@ SPARK_VERSION=3.1.3 HADOOP_VERSION=3.2 apt-get update && \ -apt-get install -y wget netcat procps libpostgresql-jdbc-java && \ +apt-get install -y wget netcat-openbsd procps libpostgresql-jdbc-java && \ wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ diff --git a/docker/spark.Dockerfile b/docker/spark.Dockerfile index bb4d378ed..49138af50 100644 --- a/docker/spark.Dockerfile +++ b/docker/spark.Dockerfile @@ -13,7 +13,7 @@ ENV SPARK_HOME /usr/spark ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}" RUN apt-get update && \ - apt-get install -y wget netcat procps libpostgresql-jdbc-java && \ + apt-get install -y wget netcat-openbsd procps libpostgresql-jdbc-java && \ wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \