Added tests

.github/workflows/run-pytest.yml

@@ -0,0 +1,30 @@
+name: Python application test with pytest
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build_test:
+    name: Build and Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.11
+
+      - name: Build Docker image
+        run: docker build -t test .
+
+      - name: Run tests with pytest and generate report
+        run: >
+          docker run
+          --name biproject
+          --network=host
+          -v "$(pwd)"/tests:/opt/airflow/tests
+          --entrypoint pytest test /opt/airflow/tests

.gitignore

@@ -8,6 +8,7 @@
 # Airflow
 logs
 airflow.db
+airflow.cfg
 standalone_admin_password.txt
 webserver_config.py
 

Dockerfile

@@ -8,11 +8,10 @@ ENV AIRFLOW__LOGGING__LOGGING_LEVEL=INFO
 
 # install your pip packages
 COPY requirements.txt ./requirements.txt
+COPY requirements-test.txt ./requirements-test.txt
 
 COPY dags ./dags
 COPY airflow.cfg ./airflow.cfg
 
-RUN pip install --upgrade pip &&\
-    pip install --no-cache-dir --upgrade setuptools==59.1.1 &&\
-    pip install --no-cache-dir --upgrade wheel &&\
-    pip install --no-cache-dir --user -r requirements.txt
+
+RUN pip install --no-cache-dir --user  -r requirements-test.txt -r requirements.txt

airflow.cfg

@@ -2,9 +2,9 @@
 # The folder where your airflow pipelines live, most likely a
 # subfolder in a code repository. This path must be absolute.
 #
-# Variable: AIRFLOW__CORE__DAGS_FOLDER
+# Variable: AIRFLOW_HOME
 #
-dags_folder = /Users/ernesta/work/projects/sis_workflows/monitoring/bi-dags/dags
+#dags_folder = /Users/ernesta/work/projects/sis_workflows/monitoring/bi-dags/dags
 
 # Hostname by providing a path to a callable, which will resolve the hostname.
 # The format is "package.function".
@@ -543,7 +543,7 @@ check_migrations = True
 #
 # Variable: AIRFLOW__LOGGING__BASE_LOG_FOLDER
 #
-base_log_folder = /Users/ernesta/work/projects/sis_workflows/monitoring/bi-dags/logs
+base_log_folder = {AIRFLOW_HOME}/logs
 
 # Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search.
 # Set this to True if you want to enable remote logging.
@@ -725,7 +725,7 @@ log_processor_filename_template = {{ filename }}.log
 #
 # Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION
 #
-dag_processor_manager_log_location = /Users/ernesta/work/projects/sis_workflows/monitoring/bi-dags/logs/dag_processor_manager/dag_processor_manager.log
+dag_processor_manager_log_location = {AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log
 
 # Name of handler to read task instance logs.
 # Defaults to use ``task`` handler.
@@ -1192,7 +1192,7 @@ access_denied_message = Access is Denied
 #
 # Variable: AIRFLOW__WEBSERVER__CONFIG_FILE
 #
-config_file = /Users/ernesta/work/projects/sis_workflows/monitoring/bi-dags/webserver_config.py
+config_file = {AIRFLOW_HOME}/webserver_config.py
 
 # The base url of your website as airflow cannot guess what domain or
 # cname you are using. This is used in automated emails that
@@ -1918,7 +1918,7 @@ orphaned_tasks_check_interval = 300.0
 #
 # Variable: AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY
 #
-child_process_log_directory = /Users/ernesta/work/projects/sis_workflows/monitoring/bi-dags/logs/scheduler
+child_process_log_directory = {AIRFLOW_HOME}/logs/scheduler
 
 # Local task jobs periodically heartbeat to the DB. If the job has
 # not heartbeat in this many seconds, the scheduler will mark the

requirements-test.txt

@@ -0,0 +1,4 @@
+pre-commit==3.6.0
+pytest==7.4.4
+coverage==7.4.0
+pytest-cov==4.1.0

tests/unit/test_test_dag.py

@@ -0,0 +1,9 @@
+from dags.example_dag.example_dag import fetch, pull
+
+
+def test_test_dag_fetch():
+    assert fetch() == "Test String"
+
+
+def test_test_dag_pull():
+    assert pull("Test String") == "Test String"