Merge branch 'master' into feat/PRD-789-new

Salman-Apptware · Jan 5, 2024 · 36c39ea · 36c39ea
2 parents d50bd4f + c3d3dbb
commit 36c39ea
Show file tree

Hide file tree

Showing 207 changed files with 6,959 additions and 12,101 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -55,6 +55,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
@@ -75,6 +76,7 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
+            !**/binary/**
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -29,11 +29,15 @@ jobs:
             "except_metadata_ingestion",
             "frontend",
           ]
-        timezone: ["UTC", "America/New_York"]
+        timezone: ["UTC"]
+        include:
+          # We only need the timezone variation for frontend tests.
+          - command: "frontend"
+            timezone: "America/New_York"
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
-      - uses: szenius/set-timezone@v1.0
+      - uses: szenius/set-timezone@v1.2
         with:
           timezoneLinux: ${{ matrix.timezone }}
       - uses: hsheth2/sane-checkout-action@v1
@@ -42,13 +46,13 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
           cache: pip
       - name: Gradle build (and test) for metadata ingestion
-        # we only need the timezone runs for frontend tests
-        if: ${{  matrix.command == 'except_metadata_ingestion' && matrix.timezone == 'America/New_York' }}
+        if: ${{  matrix.command == 'except_metadata_ingestion' }}
         run: |
           ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel
       - name: Gradle build (and test) for frontend
@@ -65,15 +69,9 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
+            !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
-      - name: Slack failure notification
-        if: failure() && github.event_name == 'push'
-        uses: kpritam/slack-job-status-action@v1
-        with:
-          job-status: ${{ job.status }}
-          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
-          channel: github-activities
 
   quickstart-compose-validation:
     runs-on: ubuntu-latest
@@ -82,10 +80,6 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
-      - name: Download YQ
-        uses: chrisdickinson/[email protected]
-        with:
-          yq-version: v4.28.2
       - name: Quickstart Compose Validation
         run: ./docker/quickstart/generate_and_compare.sh
 

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -33,6 +33,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -4,12 +4,14 @@ on:
     branches:
       - master
     paths-ignore:
+      - "docs-website/**"
       - "docs/**"
       - "**.md"
   pull_request:
     branches:
       - "**"
     paths-ignore:
+      - "docs-website/**"
       - "docs/**"
       - "**.md"
   release:
@@ -84,6 +86,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
@@ -145,6 +148,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
@@ -206,6 +210,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
@@ -267,6 +272,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
@@ -328,6 +334,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
@@ -567,6 +574,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
@@ -653,6 +661,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
@@ -731,12 +740,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test_strategy: [
-          "no_cypress_suite0",
-          "no_cypress_suite1",
-          "cypress_suite1", 
-          "cypress_rest"
-        ]
+        test_strategy:
+          [
+            "no_cypress_suite0",
+            "no_cypress_suite1",
+            "cypress_suite1",
+            "cypress_rest",
+          ]
     needs:
       [
         setup,
@@ -760,6 +770,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -894,21 +905,11 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Slack failure notification
-        if: failure() && github.event_name == 'push'
-        uses: kpritam/slack-job-status-action@v1
-        with:
-          job-status: ${{ job.status }}
-          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
-          channel: github-activities
+            !**/binary/**
   deploy_datahub_head:
     name: Deploy to Datahub HEAD
     runs-on: ubuntu-latest
-    needs:
-      [
-        setup,
-        smoke_test
-      ]
+    needs: [setup, smoke_test]
     steps:
       - uses: aws-actions/configure-aws-credentials@v1
         if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -32,6 +32,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -49,6 +49,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
@@ -78,6 +79,7 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
+            !**/binary/**
       - name: Upload coverage to Codecov
         if: ${{ always() && matrix.python-version == '3.10' }}
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
@@ -34,13 +34,12 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: "pip"
       - name: Gradle build (and test)
-        # there is some race condition in gradle build, which makes gradle never terminate in ~30% of the runs
-        # running build first without datahub-web-react:yarnBuild and then with it is 100% stable
-        # datahub-frontend:unzipAssets depends on datahub-web-react:yarnBuild but gradle does not know about it
         run: |
           ./gradlew :metadata-io:test
       - uses: actions/upload-artifact@v3
@@ -51,6 +50,7 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
+            !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
 

diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
@@ -34,10 +34,12 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run model generation

diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -14,3 +14,63 @@ jobs:
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           configuration-path: ".github/pr-labeler-config.yml"
+      - uses: actions-ecosystem/[email protected]
+        # only add names of Acryl Data team members here
+        if: 
+          ${{ 
+            !contains(
+              fromJson('[
+                "anshbansal",
+                "asikowitz",
+                "chriscollins3456",
+                "david-leifker",
+                "shirshanka",
+                "sid-acryl",
+                "swaroopjagadish",
+                "treff7es",
+                "yoonhyejin",
+                "eboneil",
+                "ethan-cartwright",
+                "gabe-lyons",
+                "hsheth2",
+                "jjoyce0510",
+                "maggiehays",
+                "mrjefflewis",
+                "pedro93",
+                "RyanHolstien"
+              ]'), 
+              github.actor
+            ) 
+          }}
+        with:
+          github_token: ${{ github.token }}
+          labels: |
+            community-contribution
+      - uses: actions-ecosystem/[email protected]
+        # only add names of champions here. Confirm with DevRel Team
+        if: 
+          ${{ 
+            contains(
+              fromJson('[
+                "skrydal",
+                "siladitya2",
+                "sgomezvillamor",
+                "ngamanda",
+                "HarveyLeo",
+                "frsann",
+                "bossenti",
+                "nikolakasev",
+                "PatrickfBraz",
+                "cuong-pham",
+                "sudhakarast",
+                "tkdrahn",
+                "rtekal",
+                "sgm44"
+              ]'), 
+              github.actor
+            ) 
+          }}
+        with:
+          github_token: ${{ github.token }}
+          labels: |
+            datahub-community-champion
diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
@@ -54,9 +54,11 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: "pip"
       - name: checkout upstream repo
         run: |
           git remote add upstream https://github.com/datahub-project/datahub.git

diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
@@ -35,9 +35,11 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Remove images
@@ -49,10 +51,15 @@ jobs:
              -x :datahub-web-react:yarnLint \
              -x :datahub-web-react:yarnGenerate \
              -x :datahub-web-react:yarnInstall \
-             -x :datahub-web-react:yarnQuickBuild \
-             -x :datahub-web-react:copyAssets \
+             -x :datahub-web-react:yarnBuild \
              -x :datahub-web-react:distZip \
              -x :datahub-web-react:jar
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: docker logs
+          path: "docker/build/container-logs/*.log"
       - uses: actions/upload-artifact@v3
         if: always()
         with:
@@ -61,10 +68,4 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Slack failure notification
-        if: failure() && github.event_name == 'push'
-        uses: kpritam/slack-job-status-action@v1
-        with:
-          job-status: ${{ job.status }}
-          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
-          channel: github-activities
+            !**/binary/**
diff --git a/build.gradle b/build.gradle
@@ -325,6 +325,10 @@ subprojects {
   }
 
   plugins.withType(JavaPlugin).configureEach {
+    if (project.name == 'datahub-web-react') {
+      return
+    }
+
     dependencies {
       implementation externalDependency.annotationApi
       constraints {