Airflow setup

.github/workflows/run-pytest.yml

@@ -0,0 +1,55 @@
+name: Python application test with pytest
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  AIRFLOW_HOME: /home/runner/work/bi-dags/bi-dags
+  REGISTRY: registry.cern.ch
+  IMAGE: cern-sis/bi-dags
+
+jobs:
+  build_test:
+    name: Build and Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.11
+
+      - name: Build Image
+        id: build
+        uses: cern-sis/gh-workflows/.github/actions/docker-build@v6
+        with:
+          registry: ${{ env.REGISTRY }}
+          image: ${{ env.IMAGE }}
+          tags: type=ref,event=pr
+          cache: false
+          username: ${{ secrets.HARBOR_USERNAME }}
+          password: ${{ secrets.HARBOR_PASSWORD }}
+
+      - name: Run tests with pytest and generate report
+        run: >
+          docker run
+          --name biproject
+          --network=host
+          -v "$(pwd)"/tests:/opt/airflow/tests
+          --entrypoint pytest
+          $REGISTRY/$IMAGE@${{ steps.build.outputs.image-digest }}
+          tests
+          --cov=./
+          --cov-report=xml
+
+      - name: Copy test coverage file to host machine
+        run: docker cp biproject:/opt/airflow/coverage.xml .
+
+      - name: Upload Coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          verbose: true

.gitignore

@@ -0,0 +1,18 @@
+# Local Dev
+*.log
+*.err
+*.pid
+*.out
+.DS_Store
+
+# Airflow
+logs
+airflow.db
+airflow.cfg
+standalone_admin_password.txt
+webserver_config.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: "22.8.0"
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v2.7.1"
+    hooks:
+      - id: prettier
+  - repo: https://github.com/pycqa/isort
+    rev: "5.12.0"
+    hooks:
+      - id: isort
+  - repo: https://github.com/pycqa/flake8
+    rev: "3.9.2"
+    hooks:
+      - id: flake8
+        args: ["--config=setup.cfg"]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

Dockerfile

@@ -0,0 +1,12 @@
+FROM apache/airflow:2.8.2-python3.8
+
+ENV PYTHONBUFFERED=0
+ENV AIRFLOW__LOGGING__LOGGING_LEVEL=INFO
+
+# install your pip packages
+COPY requirements.txt ./requirements.txt
+COPY requirements-test.txt ./requirements-test.txt
+
+COPY dags ./dags
+
+RUN pip install --no-cache-dir --user  -r requirements-test.txt -r requirements.txt

README.md

@@ -1 +1 @@
-# bi-dags
+# bi-dags