Airflow setup

.gitignore

@@ -0,0 +1,17 @@
+# Local Dev
+*.log
+*.err
+*.pid
+*.out
+.DS_Store
+
+# Airflow
+logs
+airflow.db
+standalone_admin_password.txt
+webserver_config.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: "22.8.0"
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v2.7.1"
+    hooks:
+      - id: prettier
+  - repo: https://github.com/pycqa/isort
+    rev: "5.12.0"
+    hooks:
+      - id: isort
+  - repo: https://github.com/pycqa/flake8
+    rev: "3.9.2"
+    hooks:
+      - id: flake8
+        args: ["--config=setup.cfg"]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

Dockerfile

@@ -0,0 +1,18 @@
+FROM apache/airflow:2.8.1-python3.10
+
+ENV AIRFLOW_HOME=/opt/airflow
+WORKDIR /opt/airflow
+
+ENV PYTHONBUFFERED=0
+ENV AIRFLOW__LOGGING__LOGGING_LEVEL=INFO
+
+# install your pip packages
+COPY requirements.txt ./requirements.txt
+
+COPY dags ./dags
+COPY airflow.cfg ./airflow.cfg
+
+RUN pip install --upgrade pip &&\
+    pip install --no-cache-dir --upgrade setuptools==59.1.1 &&\
+    pip install --no-cache-dir --upgrade wheel &&\
+    pip install --no-cache-dir --user -r requirements.txt