Merge branch 'datahub-project:master' into master

acryldata · Jan 20, 2024 · c663f24 · c663f24
2 parents 6a89727 + ce65b9c
commit c663f24
Show file tree

Hide file tree

Showing 170 changed files with 3,844 additions and 2,429 deletions.
diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml
@@ -0,0 +1,79 @@
+name: 'Identify CI Optimizations'
+description: 'Determine if code changes are specific to certain modules.'
+
+outputs:
+  frontend-only:
+    description: "Frontend only change"
+    value: ${{ steps.filter.outputs.frontend == 'true' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'false' }}
+  ingestion-only:
+    description: "Ingestion only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'true' && steps.filter.outputs.backend == 'false' }}
+  backend-only:
+    description: "Backend only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
+  backend-change:
+    description: "Backend code has changed"
+    value: ${{ steps.filter.outputs.backend == 'true' }}
+  ingestion-change:
+    description: "Ingestion code has changed"
+    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+  frontend-change:
+    description: "Frontend code has changed"
+    value: ${{ steps.filter.outputs.frontend == 'true' }}
+  docker-change:
+    description: "Docker code has changed"
+    value: ${{ steps.filter.outputs.docker == 'true' }}
+  kafka-setup-change:
+    description: "Kafka setup docker change"
+    value: ${{ steps.filter.outputs.kafka-setup == 'true' }}
+  mysql-setup-change:
+    description: "Mysql setup docker change"
+    value: ${{ steps.filter.outputs.mysql-setup == 'true' }}
+  postgres-setup-change:
+    description: "Postgres setup docker change"
+    value: ${{ steps.filter.outputs.postgres-setup == 'true' }}
+  elasticsearch-setup-change:
+    description: "Elasticsearch setup docker change"
+    value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }}
+runs:
+  using: "composite"
+  steps:
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          frontend:
+            - "datahub-frontend/**"
+            - "datahub-web-react/**"
+            - "smoke-test/tests/cypress/**"
+            - "docker/datahub-frontend/**"
+          ingestion:
+            - "metadata-ingestion-modules/airflow-plugin/**"
+            - "metadata-ingestion/**"
+            - "metadata-models/**"
+            - "smoke-test/**"
+            - "docker/datahub-ingestion-**"
+          docker:
+            - "docker/**"
+          backend:
+            - "metadata-models/**"
+            - "datahub-upgrade/**"
+            - "entity-registry/**"
+            - "li-utils/**"
+            - "metadata-auth/**"
+            - "metadata-dao-impl/**"
+            - "metadata-events/**"
+            - "metadata-io/**"
+            - "metadata-jobs/**"
+            - "metadata-service/**"
+            - "metadata-utils/**"
+            - "smoke-test/**"
+            - "docker/**"
+          kafka-setup:
+            - "docker/kafka-setup/**"
+          mysql-setup:
+            - "docker/mysql-setup/**"
+          postgres-setup:
+            - "docker/postgres-setup/**"
+          elasticsearch-setup:
+            - "docker/elasticsearch-setup/**"
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -40,13 +40,16 @@ jobs:
             extra_pip_requirements: "apache-airflow~=2.2.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.4.0 pluggy==1.0.0 "pendulum<3.0"'
+            extra_pip_requirements: 'apache-airflow~=2.4.0 pluggy==1.0.0 "pendulum<3.0" "Flask-Session<0.6.0"'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.6.0 "pendulum<3.0"'
+            extra_pip_requirements: 'apache-airflow~=2.6.0 "pendulum<3.0" "Flask-Session<0.6.0"'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow>=2.7.0 pydantic==2.4.2"
+            extra_pip_requirements: 'apache-airflow~=2.7.0 pydantic==2.4.2 "Flask-Session<0.6.0"'
+            extra_pip_extras: plugin-v2
+          - python-version: "3.10"
+            extra_pip_requirements: 'apache-airflow>=2.8.0 pydantic>=2.4.2 "Flask-Session<0.6.0"'
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -20,6 +20,25 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
+      ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }}
+      backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }}
+      docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' }}
+      frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }}
+      ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }}
+      kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }}
+      mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }}
+      postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }}
+      elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
+    steps:
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
+      - uses: ./.github/actions/ci-optimization
+        id: ci-optimize
+
   build:
     strategy:
       fail-fast: false
@@ -36,11 +55,13 @@ jobs:
             timezone: "America/New_York"
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    needs: setup
     steps:
       - uses: szenius/[email protected]
         with:
           timezoneLinux: ${{ matrix.timezone }}
-      - uses: hsheth2/sane-checkout-action@v1
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -51,12 +72,12 @@ jobs:
         with:
           python-version: "3.10"
           cache: pip
-      - name: Gradle build (and test) for metadata ingestion
-        if: ${{  matrix.command == 'except_metadata_ingestion' }}
+      - name: Gradle build (and test) for NOT metadata ingestion
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
           ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel
       - name: Gradle build (and test) for frontend
-        if: ${{  matrix.command == 'frontend' }}
+        if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         run: |
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
@@ -75,8 +96,11 @@ jobs:
 
   quickstart-compose-validation:
     runs-on: ubuntu-latest
+    needs: setup
+    if: ${{ needs.setup.outputs.docker_change == 'true' }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"