Initial (#1)

* initial

* fixes from initial testing

.github/workflows/build_image.yaml

@@ -0,0 +1,90 @@
+name: Build Airflow image
+
+on:
+  workflow_dispatch:
+    inputs:
+      airflow_version:
+        description: 'Airflow version'
+        required: true
+        default: 'default'
+        type: string
+      python_version:
+        description: 'Python version'
+        required: true
+        default: 'default'
+        type: string
+  push:
+    branches:
+      - main
+    paths:
+      - image/Dockerfile
+      - image/requirements.txt
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build_airflow:
+    runs-on: ubuntu-latest
+
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Set variables
+        run: |
+          if [ "${{ inputs.airflow_version }}" == "" ] || [ "${{ inputs.airflow_version }}" == "default" ]; then
+            AIRFLOW_VERSION=$(sed -n 's/^ARG AIRFLOW_VERSION=//p' image/Dockerfile)
+          else
+            AIRFLOW_VERSION=${{ inputs.airflow_version }}
+          fi
+          if [ "${{ inputs.python_version }}" == "" ] || [ "${{ inputs.python_version }}" == "default" ]; then
+            PYTHON_VERSION=$(sed -n 's/^ARG PYTHON_VERSION=//p' image/Dockerfile)
+          else
+            PYTHON_VERSION=${{ inputs.python_version }}
+          fi
+          echo "AIRFLOW_VERSION=${AIRFLOW_VERSION}" >> ${GITHUB_ENV}
+          echo "PYTHON_VERSION=${PYTHON_VERSION}" >> ${GITHUB_ENV}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Github Packages container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          logout: false
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/airflow
+          tags: |
+            type=raw,value=${{ env.AIRFLOW_VERSION }}-python${{ env.PYTHON_VERSION }}-{{date 'YYYYMMDD-HHmm'}}
+            type=raw,value=${{ env.AIRFLOW_VERSION }}-python${{ env.PYTHON_VERSION }}
+          labels: |
+            org.opencontainers.image.description=Onyx Airflow image
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: image
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          sbom: false
+          provenance: false
+          build-args: |
+            AIRFLOW_VERSION=${{ env.AIRFLOW_VERSION }}
+            PYTHON_VERSION=${{ env.PYTHON_VERSION }}

.github/workflows/publish.yaml

@@ -0,0 +1,28 @@
+name: build and publish
+'on':
+  workflow_dispatch: {}
+  push:
+    branches:
+    - main
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/nebari-plugin-airflow
+    permissions:
+      id-token: write
+    steps:
+    - name: Checkout Image
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install Hatch
+      run: pip install hatch
+    - name: Build
+      run: hatch build
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

image/.dockerignore

@@ -0,0 +1,4 @@
+Dockerfile
+.dockerignore
+
+dags

image/Dockerfile

@@ -0,0 +1,10 @@
+ARG AIRFLOW_VERSION=2.8.0
+ARG PYTHON_VERSION=3.10
+
+FROM apache/airflow:slim-${AIRFLOW_VERSION}-python${PYTHON_VERSION}
+
+COPY requirements.txt .
+
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt && \
+    rm requirements.txt

image/requirements.txt

@@ -0,0 +1,20 @@
+s3fs
+scikit-image
+tensorflow
+keras
+mlflow>=2.0
+scikit-learn
+pandas
+numpy
+boto3
+aiohttp==3.7.2
+botocore
+aiobotocore
+Pillow
+protobuf
+psycopg2-binary
+tqdm
+tables
+authlib
+apache-airflow-providers-celery
+apache-airflow-providers-cncf-kubernetes

pyproject.toml

@@ -0,0 +1,167 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "nebari-plugin-airflow"
+dynamic = ["version"]
+description = "MetroStar Onyx - Airflow plugin for Nebari platform"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+keywords = []
+authors = [
+  { name = "Scott Blair", email = "[email protected]" },
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
+]
+dependencies = [
+  "nebari",
+]
+
+[project.urls]
+Documentation = "https://github.com/MetroStar/nebari-airflow#readme"
+Issues = "https://github.com/MetroStar/nebari-airflow/issues"
+Source = "https://github.com/MetroStar/nebari-airflow"
+
+[tool.hatch.build.targets.sdist]
+include = ["src"]
+
+[tool.hatch.build.targets.wheel]
+include = ["src"]
+
+[tool.hatch.version]
+path = "src/nebari_plugin_airflow/__about__.py"
+
+[tool.hatch.envs.default]
+dependencies = [
+  "coverage[toml]>=6.5",
+  "cryptography",
+  "pytest",
+  "kubernetes",
+]
+[tool.hatch.envs.default.scripts]
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+cov-report = [
+  "- coverage combine",
+  "coverage report",
+]
+cov = [
+  "test-cov",
+  "cov-report",
+]
+
+[[tool.hatch.envs.all.matrix]]
+python = ["3.10", "3.11"]
+
+[tool.hatch.envs.lint]
+detached = true
+dependencies = [
+  "black>=23.1.0",
+  "mypy>=1.0.0",
+  "ruff>=0.0.243",
+]
+[tool.hatch.envs.lint.scripts]
+typing = "mypy --install-types --non-interactive {args:src/nebari_plugin_airflow tests}"
+style = [
+  "ruff {args:.}",
+  "black --check --diff {args:.}",
+]
+fmt = [
+  "black {args:.}",
+  "ruff --fix {args:.}",
+  "style",
+]
+all = [
+  "style",
+  "typing",
+]
+
+[tool.black]
+target-version = ["py310"]
+line-length = 120
+skip-string-normalization = true
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+select = [
+  "A",
+  "ARG",
+  "B",
+  "C",
+  "DTZ",
+  "E",
+  "EM",
+  "F",
+  "FBT",
+  "I",
+  "ICN",
+  "ISC",
+  "N",
+  "PLC",
+  "PLE",
+  "PLR",
+  "PLW",
+  "Q",
+  "RUF",
+  "S",
+  "T",
+  "TID",
+  "UP",
+  "W",
+  "YTT",
+]
+ignore = [
+  # Allow non-abstract empty methods in abstract base classes
+  "B027",
+  # Allow boolean positional values in function calls, like `dict.get(... True)`
+  "FBT003",
+  # Ignore checks for possible passwords
+  "S105", "S106", "S107",
+  # Ignore complexity
+  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+]
+unfixable = [
+  # Don't touch unused imports
+  "F401",
+]
+
+[tool.ruff.isort]
+known-first-party = ["nebari_plugin_airflow"]
+
+[tool.ruff.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.per-file-ignores]
+# Tests can use magic values, assertions, and relative imports
+"tests/**/*" = ["PLR2004", "S101", "TID252"]
+
+[tool.coverage.run]
+source_pkgs = ["nebari_plugin_airflow", "tests"]
+branch = true
+parallel = true
+omit = [
+  "src/nebari_plugin_airflow/__about__.py",
+]
+
+[tool.coverage.paths]
+nebari_plugin_airflow = ["src/nebari_plugin_airflow", "*/nebari-plugin-airflow/src/nebari_plugin_airflow"]
+tests = ["tests", "*/nebari-plugin-airflow/tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+  "no cov",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:",
+]
+
+[project.entry-points.nebari]
+nebari-plugin-airflow = "src.nebari_plugin_airflow"

src/nebari_plugin_airflow/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

src/nebari_plugin_airflow/__init__.py

@@ -0,0 +1,10 @@
+from nebari.hookspecs import NebariStage, hookimpl
+from typing import List
+
+from .plugin import AirflowStage
+
+@hookimpl
+def nebari_stage() -> List[NebariStage]:
+    return [
+        AirflowStage,
+    ]