Skip to content

Commit

Permalink
Merge branch 'master' into feat/PRD-789-new
Browse files Browse the repository at this point in the history
  • Loading branch information
Salman-Apptware authored Dec 25, 2023
2 parents 8cfe4ab + ed5bdfc commit 48689a3
Show file tree
Hide file tree
Showing 2,070 changed files with 106,945 additions and 75,269 deletions.
12 changes: 9 additions & 3 deletions .github/workflows/airflow-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,23 +32,29 @@ jobs:
strategy:
matrix:
include:
# Note: this should be kept in sync with tox.ini.
- python-version: "3.8"
extra_pip_requirements: "apache-airflow~=2.1.4"
extra_pip_extras: plugin-v1
- python-version: "3.8"
extra_pip_requirements: "apache-airflow~=2.2.4"
extra_pip_extras: plugin-v1
- python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.4.0"
extra_pip_requirements: 'apache-airflow~=2.4.0 pluggy==1.0.0 "pendulum<3.0"'
extra_pip_extras: plugin-v2
- python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.6.0"
extra_pip_requirements: 'apache-airflow~=2.6.0 "pendulum<3.0"'
extra_pip_extras: plugin-v2
- python-version: "3.10"
extra_pip_requirements: "apache-airflow>=2.7.0"
extra_pip_requirements: "apache-airflow>=2.7.0 pydantic==2.4.2"
extra_pip_extras: plugin-v2
fail-fast: false
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ jobs:
with:
timezoneLinux: ${{ matrix.timezone }}
- uses: hsheth2/sane-checkout-action@v1
- name: Set up JDK 11
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
java-version: 17
- uses: actions/setup-python@v4
with:
python-version: "3.10"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/check-datahub-jars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: hsheth2/sane-checkout-action@v1
- name: Set up JDK 11
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
java-version: 17
- uses: actions/setup-python@v4
with:
python-version: "3.10"
Expand Down
55 changes: 45 additions & 10 deletions .github/workflows/docker-unified.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ jobs:
runs-on: ubuntu-latest
needs: setup
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- name: Check out the repo
uses: hsheth2/sane-checkout-action@v1
- name: Pre-build artifacts for docker image
Expand Down Expand Up @@ -135,6 +140,11 @@ jobs:
runs-on: ubuntu-latest
needs: setup
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- name: Check out the repo
uses: hsheth2/sane-checkout-action@v1
- name: Pre-build artifacts for docker image
Expand Down Expand Up @@ -191,6 +201,11 @@ jobs:
runs-on: ubuntu-latest
needs: setup
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- name: Check out the repo
uses: hsheth2/sane-checkout-action@v1
- name: Pre-build artifacts for docker image
Expand Down Expand Up @@ -247,6 +262,11 @@ jobs:
runs-on: ubuntu-latest
needs: setup
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- name: Check out the repo
uses: hsheth2/sane-checkout-action@v1
- name: Pre-build artifacts for docker image
Expand Down Expand Up @@ -303,6 +323,11 @@ jobs:
runs-on: ubuntu-latest
needs: setup
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- name: Check out the repo
uses: hsheth2/sane-checkout-action@v1
- name: Pre-build artifacts for docker image
Expand Down Expand Up @@ -537,6 +562,11 @@ jobs:
needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
needs: [setup, datahub_ingestion_base_slim_build]
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- name: Check out the repo
uses: hsheth2/sane-checkout-action@v1
- uses: dorny/paths-filter@v2
Expand Down Expand Up @@ -618,6 +648,11 @@ jobs:
needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
needs: [setup, datahub_ingestion_base_full_build]
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- name: Check out the repo
uses: hsheth2/sane-checkout-action@v1
- uses: dorny/paths-filter@v2
Expand Down Expand Up @@ -696,7 +731,12 @@ jobs:
strategy:
fail-fast: false
matrix:
test_strategy: ["no_cypress", "cypress_suite1", "cypress_rest"]
test_strategy: [
"no_cypress_suite0",
"no_cypress_suite1",
"cypress_suite1",
"cypress_rest"
]
needs:
[
setup,
Expand All @@ -715,11 +755,11 @@ jobs:
run: df -h . && docker images
- name: Check out the repo
uses: actions/checkout@v3
- name: Set up JDK 11
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
java-version: 17
- uses: actions/setup-python@v4
with:
python-version: "3.10"
Expand Down Expand Up @@ -792,11 +832,6 @@ jobs:
ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
run: |
./smoke-test/run-quickstart.sh
- name: sleep 60s
run: |
# we are doing this because gms takes time to get ready
# and we don't have a better readiness check when bootstrap is done
sleep 60s
- name: Disk Check
run: df -h . && docker images
- name: Disable ES Disk Threshold
Expand Down Expand Up @@ -876,13 +911,13 @@ jobs:
]
steps:
- uses: aws-actions/configure-aws-credentials@v1
if: ${{ needs.setup.outputs.publish != 'false' }}
if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
with:
aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SQS_ACCESS_KEY }}
aws-region: us-west-2
- uses: isbang/[email protected]
if: ${{ needs.setup.outputs.publish != 'false' }}
if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
with:
sqs-url: ${{ secrets.DATAHUB_HEAD_SYNC_QUEUE }}
message: '{ "command": "git-sync", "args" : {"repoName": "${{ needs.setup.outputs.repository_name }}", "repoOrg": "${{ github.repository_owner }}", "repoBranch": "${{ needs.setup.outputs.branch_name }}", "repoShaShort": "${{ needs.setup.outputs.short_sha }}" }}'
4 changes: 2 additions & 2 deletions .github/workflows/documentation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up JDK 11
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
java-version: 17
- uses: actions/setup-python@v4
with:
python-version: "3.10"
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/metadata-ingestion.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ jobs:
- python-version: "3.10"
fail-fast: false
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/metadata-io.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ jobs:
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
- name: Set up JDK 11
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
java-version: 17
- uses: actions/setup-python@v4
with:
python-version: "3.10"
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/metadata-model.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ jobs:
runs-on: ubuntu-latest
needs: setup
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/publish-datahub-jars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ jobs:
if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
steps:
- uses: hsheth2/sane-checkout-action@v1
- name: Set up JDK 11
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
java-version: 17
- uses: actions/setup-python@v4
with:
python-version: "3.10"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/spark-smoke-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: hsheth2/sane-checkout-action@v1
- name: Set up JDK 11
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
java-version: 17
- uses: actions/setup-python@v4
with:
python-version: "3.10"
Expand Down
Loading

0 comments on commit 48689a3

Please sign in to comment.