cleanup

.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: test
 
 on:
   push: # Run on pushes to the default branch
-    branches: [main]
+    branches: [main, kube_mode_ci]
   pull_request_target: # Also run on pull requests originated from forks
-    branches: [main]
+    branches: [main, kube_mode_ci]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -163,7 +163,6 @@ jobs:
           POSTGRES_DB: postgres
           POSTGRES_SCHEMA: public
           POSTGRES_PORT: 5432
-          SOURCE_RENDERING_BEHAVIOR: all
 
       - name: Upload coverage to Github
         uses: actions/upload-artifact@v2
@@ -235,7 +234,6 @@ jobs:
           POSTGRES_DB: postgres
           POSTGRES_SCHEMA: public
           POSTGRES_PORT: 5432
-          SOURCE_RENDERING_BEHAVIOR: all
 
       - name: Upload coverage to Github
         uses: actions/upload-artifact@v2
@@ -379,7 +377,6 @@ jobs:
           POSTGRES_DB: postgres
           POSTGRES_SCHEMA: public
           POSTGRES_PORT: 5432
-          SOURCE_RENDERING_BEHAVIOR: all
 
       - name: Upload coverage to Github
         uses: actions/upload-artifact@v2
@@ -461,12 +458,75 @@ jobs:
       AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
       PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
 
+  Run-Kubernetes-Tests:
+    needs: Authorize
+      runs-on: ubuntu-latest
+      strategy:
+        matrix:
+          python-version: [ "3.11" ]
+          airflow-version: [ "2.8" ]
+      steps:
+        - uses: actions/checkout@v3
+          with:
+            ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        - uses: actions/cache@v3
+          with:
+            path: |
+              ~/.cache/pip
+              .local/share/hatch/
+            key: coverage-integration-kubernetes-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
+
+        - name: Set up Python ${{ matrix.python-version }}
+          uses: actions/setup-python@v4
+          with:
+            python-version: ${{ matrix.python-version }}
+
+        - name: Create KinD cluster
+          uses: container-tools/kind-action@v1
+
+        - name: Install packages and dependencies
+          run: |
+            python -m pip install uv
+            uv pip install --system hatch
+            hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
+
+        - name: Run kubernetes tests agains
+          run: |
+            sh ./scripts/test/kubernetes-setup.sh
+            hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes
+          env:
+            AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
+            AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
+            AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }}
+            AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }}
+            AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }}
+            AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
+            PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
+            COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
+            DATABRICKS_CLUSTER_ID: mock
+            DATABRICKS_HOST: mock
+            DATABRICKS_WAREHOUSE_ID: mock
+            DATABRICKS_TOKEN: mock
+            POSTGRES_HOST: localhost
+            POSTGRES_USER: postgres
+            POSTGRES_PASSWORD: postgres
+            POSTGRES_DB: postgres
+            POSTGRES_SCHEMA: public
+            POSTGRES_PORT: 5432
+
+        - name: Upload coverage to Github
+          uses: actions/upload-artifact@v2
+          with:
+            name: coverage-integration-kubernetes-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
+            path: .coverage
+
   Code-Coverage:
     if: github.event.action != 'labeled'
     needs:
       - Run-Unit-Tests
       - Run-Integration-Tests
       - Run-Integration-Tests-Expensive
+      - Run-Kubernetes-Tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

dev/Dockerfile.postgres_profile_docker_k8s

@@ -4,7 +4,7 @@ RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz
 
 ENV POSTGRES_DATABASE=postgres
 ENV POSTGRES_HOST=postgres.default.svc.cluster.local
-ENV POSTGRES_PASSWORD=<postgres_password>
+ENV POSTGRES_PASSWORD=postgres
 ENV POSTGRES_PORT=5432
 ENV POSTGRES_SCHEMA=public
 ENV POSTGRES_USER=postgres

dev/dags/dbt/jaffle_shop/profiles.yml

@@ -16,9 +16,9 @@ postgres_profile:
   outputs:
     dev:
       type: postgres
-      dbname: postgres #'{{ env_var(''POSTGRES_DATABASE'') }}'
-      host: postgres.default.svc.cluster.local #'{{ env_var(''POSTGRES_HOST'') }}'
-      pass: postgres #'{{ env_var(''POSTGRES_PASSWORD'') }}'
-      port: 5432 #'{{ env_var(''POSTGRES_PORT'') | as_number }}'
-      schema: postgres # '{{ env_var(''POSTGRES_SCHEMA'') }}'
-      user: postgres #'{{ env_var(''POSTGRES_USER'') }}'
+      dbname: '{{ env_var(''POSTGRES_DATABASE'') }}'
+      host: '{{ env_var(''POSTGRES_HOST'') }}'
+      pass: '{{ env_var(''POSTGRES_PASSWORD'') }}'
+      port: '{{ env_var(''POSTGRES_PORT'') | as_number }}'
+      schema: '{{ env_var(''POSTGRES_SCHEMA'') }}'
+      user: '{{ env_var(''POSTGRES_USER'') }}'

scripts/test/kubernetes-setup.sh

@@ -1,93 +1,34 @@
 #!/bin/bash
 
+# Print each command before executing it
+# Exit the script immediately if any command exits with a non-zero status (for debugging purposes)
 set -x
 set -e
 
-#check_nodes_ready() {
-#    # Get the list of node statuses
-#    node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}')
-#    # Check if all nodes are in the "Ready" state
-#    for status in $node_statuses; do
-#        if [ "$status" != "Ready" ]; then
-#            return 1
-#        fi
-#    done
-#    return 0
-#}
-#
-#wait_for_nodes_ready() {
-#    local max_attempts=60
-#    local interval=5
-#    local attempt=0
-#
-#    echo "Waiting for nodes in the kind cluster to be in 'Ready' state..."
-#
-#    while [ $attempt -lt $max_attempts ]; do
-#        if check_nodes_ready; then
-#            echo "All nodes in the kind cluster are in 'Ready' state."
-#            return 0
-#        else
-#            echo "Nodes are not yet ready. Checking again in $interval seconds..."
-#            sleep $interval
-#            attempt=$((attempt + 1))
-#        fi
-#    done
-#
-#    echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state."
-#    return 1
-#}
-#
-#kubectl config set-context default
-#
-## Create a docker image containing the dbt project files and dbt profile
-#cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s .
-## Make the build image available in the Kind K8s cluster
-#kind load docker-image dbt-jaffle-shop:1.0.0
-#
-## Deploy a Postgres pod to Kind
-##helm repo add bitnami https://charts.bitnami.com/bitnami
-##helm repo update
-##helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml
-#
-## Retrieve the Postgres password and set it as an environment variable
-##POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d)
-##export POSTGRES_PASSWORD
-#
-#kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD
-#
-#sleep 120
-## Expose the Postgres to the host running Docker/Kind
-##kubectl port-forward --namespace default postgres-postgresql-0  5432:5432 &
-##kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 &
-##wait_for_nodes_ready
-##
-### Wait for the kind cluster to be in 'Ready' state
-##wait_for_nodes_ready
-#
-## For Debugging
-#echo "nodes"
-#kubectl get nodes
-#echo "helm"
-#helm list
-#echo "pod service"
-#kubectl get pods --namespace default
-#kubectl get svc --namespace default
-#echo "pg log"
-#kubectl logs postgres-postgresql-0 -c postgresql
-#kubectl describe pod postgres-postgresql-0
-
-
-kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=postgres
+# Create a Kubernetes secret named 'postgres-secrets' with the specified literals for host and password
+kubectl create secret generic postgres-secrets \
+  --from-literal=host=postgres-postgresql.default.svc.cluster.local \
+  --from-literal=password=postgres
 
+# Apply the PostgreSQL deployment configuration from the specified YAML file
 kubectl apply -f scripts/test/postgres-deployment.yaml
 
+# Build the Docker image with tag 'dbt-jaffle-shop:1.0.0' using the specified Dockerfile
 cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s .
+
+# Load the Docker image into the local KIND cluster
 kind load docker-image dbt-jaffle-shop:1.0.0
 
+# Retrieve the name of the PostgreSQL pod using the label selector 'app=postgres'
+# The output is filtered to get the first pod's name
 POD_NAME=$(kubectl get pods -n default -l app=postgres -o jsonpath='{.items[0].metadata.name}')
 
+# Print the name of the PostgreSQL pod
 echo "$POD_NAME"
 
-kubectl port-forward --namespace default "$POD_NAME"  5432:5432 &
+# Forward port 5432 from the PostgreSQL pod to the local machine's port 5432
+# This allows local access to the PostgreSQL instance running in the pod
+kubectl port-forward --namespace default "$POD_NAME" 5432:5432 &
 
+# List all pods in the default namespace to verify the status of pods
 kubectl get pod