WIP global: everything is working

.github/workflows/lint.yml → .github/workflows/backoffice/lint.yml

@@ -16,6 +16,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
+          working-directory: backoffice
 
       - name: Set up Python
         uses: actions/setup-python@v4

.github/workflows/test.yml → .github/workflows/backoffice/test.yml

@@ -18,15 +18,15 @@ jobs:
     with:
       ref: ${{ inputs.ref }}
       image: cern-sis/inspire/backoffice
-      context: .
+      context: ./backoffice
       dockerfile: ./compose/production/django/Dockerfile
     secrets: inherit
 
   test:
     needs: build
     strategy:
       fail-fast: false
-    uses: ./.github/workflows/integration-tests.yml
+    uses: ./.github/workflows/backoffice/integration-tests.yml
     with:
       ref: ${{ inputs.ref }}
       image: "registry.cern.ch/cern-sis/inspire/backoffice@${{ needs.build.outputs.image-id }}"

.github/workflows/pull-request-main.yml → .github/workflows/pull-request-main-backoffice.yml

@@ -6,11 +6,11 @@ on:
 
 jobs:
   lint:
-    uses: ./.github/workflows/lint.yml
+    uses: ./.github/workflows/backoffice/lint.yml
     with:
       ref: ${{ github.ref }}
   test:
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/backoffice/test.yml
     with:
       ref: ${{ github.event.pull_request.head.sha }}
     secrets: inherit

.github/workflows/push-main-backoffice.yml

@@ -0,0 +1,34 @@
+name: Push main
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'backoffice/**'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  lint:
+    uses: ./.github/workflows/backoffice/lint.yml
+    with:
+      ref: ${{ github.ref }}
+  test:
+    needs: lint
+    uses: ./.github/workflows/backoffice/test.yml
+    with:
+      ref: ${{ github.ref }}
+    secrets: inherit
+  deploy:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: send event
+        uses: cern-sis/gh-workflows/.github/actions/[email protected]
+        with:
+          event-type: update
+          images: |
+            cern-sis/inspire/backoffice@${{ needs.test.outputs.image-id }}
+          token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES }}

.github/workflows/push-main.yml → .github/workflows/push-main-workflows.yml

@@ -3,6 +3,8 @@ name: Push main
 on:
   push:
     branches: [main]
+    paths:
+      - 'workflows/**'
 
 defaults:
   run:

.github/workflows/workflows/lint.yml

@@ -0,0 +1,22 @@
+name: Lint
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: The reference to build
+        type: string
+        required: true
+
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          working-directory: workflows
+
+      - name: Pre-commit check
+        uses: pre-commit/[email protected]

Makefile

@@ -0,0 +1,28 @@
+# Makefile
+
+# Set the AIRFLOW_HOME environment variable
+export AIRFLOW_HOME=$(shell pwd)/workflows
+export AIRFLOW_PROJ_DIR=$(AIRFLOW_HOME)
+
+# Default target
+all: run
+
+run:
+	docker-compose up
+
+clean:
+	docker-compose down
+
+init: backoffice-init workflows-init
+
+backoffice-init:
+	docker-compose exec backoffice-webserver python manage.py create_groups
+	docker-compose exec backoffice-webserver python manage.py loaddata backoffice/users/fixtures/users.json
+	docker-compose exec backoffice-webserver python manage.py loaddata backoffice/users/fixtures/tokens.json
+	echo "Backoffice initialized"
+
+workflows-init:
+	docker-compose exec airflow-webserver /entrypoint airflow connections import ./scripts/connections/connections.json
+	docker-compose exec airflow-webserver /entrypoint airflow variables import ./scripts/variables/variables.json
+	echo "\033[31mCHANGE inspire_token in Admin->Variables\033[0m"
+	echo "Workflows initialized"

.envs/local/.django → backoffice/.envs/local/.django

@@ -10,6 +10,7 @@ REDIS_URL=redis://redis:6379/0
 # Celery
 CELERY_BROKER_URL=amqp://guest:guest@mq:5672/
 CELERY_RESULT_BACKEND=redis://redis:6379/2
+CELERY_TASK_ALWAYS_EAGER=True
 # Flower
 CELERY_FLOWER_USER=debug
 CELERY_FLOWER_PASSWORD=debug
@@ -19,5 +20,5 @@ OPENSEARCH_HOST=opensearch:9200
 OPENSEARCH_INDEX_PREFIX=backoffice-backend-local
 
 # Airflow
-AIRFLOW_BASE_URL=http://host.docker.internal:8082
-AIRFLOW_TOKEN=CHANGE_ME
+AIRFLOW_BASE_URL=http://host.docker.internal:8080
+AIRFLOW_TOKEN=YWlyZmxvdzphaXJmbG93Cg

.envs/docker/.postgres → backoffice/.envs/local/.postgres

@@ -1,7 +1,7 @@
 # PostgreSQL
 # ------------------------------------------------------------------------------
-POSTGRES_HOST=postgres
+POSTGRES_HOST=postgres-backoffice
 POSTGRES_PORT=5432
 POSTGRES_DB=inspire
 POSTGRES_USER=inspire
-POSTGRES_PASSWORD=inspire
+POSTGRES_PASSWORD=inspire

backoffice/backoffice/users/fixtures/tokens.json

@@ -0,0 +1,10 @@
+[
+{
+    "model": "authtoken.token",
+    "pk": "2e04111a61e8f5ba6ecec52af21bbb9e81732085",
+    "fields": {
+        "user": 1,
+        "created": "2024-07-23T15:08:17.326Z"
+    }
+}
+]

backoffice/backoffice/users/fixtures/users.json

@@ -0,0 +1,38 @@
+[
+{
+    "model": "users.user",
+    "pk": 1,
+    "fields": {
+        "password": "argon2$argon2id$v=19$m=102400,t=2,p=8$M3hDdTNkSHpSTTd0SE5TNW4zNzBQSw$RZmZYp0e0xIReaEvvtg23TX5Pu+OAL2Bl9DPiC0hf/k",
+        "last_login": "2024-07-23T15:05:59Z",
+        "is_superuser": true,
+        "is_staff": true,
+        "is_active": true,
+        "date_joined": "2024-07-23T15:05:30Z",
+        "name": "",
+        "email": "[email protected]",
+        "groups": [
+            1
+        ],
+        "user_permissions": []
+    }
+},
+{
+    "model": "users.user",
+    "pk": 2,
+    "fields": {
+        "password": "argon2$argon2id$v=19$m=102400,t=2,p=8$ckljTUU5WVhhV1YzNXF2Qk92Z2lzdw$F/uGQYwEhmsLf8CMMTHGW7ttU2Q+EdLxlegqCPGyX7o",
+        "last_login": null,
+        "is_superuser": true,
+        "is_staff": true,
+        "is_active": true,
+        "date_joined": "2024-07-23T15:08:00Z",
+        "name": "",
+        "email": "[email protected]",
+        "groups": [
+            2
+        ],
+        "user_permissions": []
+    }
+}
+]

backoffice/workflows/admin.py → backoffice/backoffice/workflows/admin.py

@@ -1,8 +1,8 @@
 from django.contrib import admin
-from django_json_widget.widgets import JSONEditorWidget
 
 from backoffice.management.permissions import IsAdminOrCuratorUser
-from backoffice.workflows.models import Workflow
+
+from .models import Workflow
 
 
 class WorkflowsAdminSite(admin.AdminSite):
@@ -34,10 +34,6 @@ class WorkflowAdmin(admin.ModelAdmin):
     Admin class for Workflow model. Define get, update and delete permissions.
     """
 
-    formfield_overrides = {
-        "data": {"widget": JSONEditorWidget},
-    }
-
     ordering = ("-_updated_at",)
     search_fields = ["id", "data"]
     list_display = (

backoffice/workflows/airflow_utils.py → backoffice/backoffice/workflows/airflow_utils.py

@@ -1,4 +1,5 @@
 from os import environ
+import logging
 
 import requests
 from django.http import JsonResponse
@@ -9,6 +10,7 @@
 
 AIRFLOW_HEADERS = {"Content-Type": "application/json", "Authorization": f"Basic {environ.get('AIRFLOW_TOKEN')}"}
 
+logger = logging.getLogger(__name__)
 
 def trigger_airflow_dag(dag_id, workflow_id, extra_data=None):
     """Triggers an airflow dag.
@@ -26,9 +28,10 @@ def trigger_airflow_dag(dag_id, workflow_id, extra_data=None):
     url = f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/dagRuns"
 
     try:
+        logger.info(f"Triggering DAG %s with data: %s and %s %s", dag_id, data, AIRFLOW_HEADERS, url)
         response = requests.post(url, json=data, headers=AIRFLOW_HEADERS)
         response.raise_for_status()
         return JsonResponse(response.json())
     except RequestException as req_err:
-        data = {"error": req_err}
-        return JsonResponse(data, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        data = {"error": response.json()}
+        return JsonResponse(data, status=status.HTTP_502_BAD_GATEWAY)

config/api_router.py → backoffice/config/api_router.py

@@ -17,9 +17,13 @@
 router.register("users", UserViewSet)
 
 # Workflows
-router.register("workflows/authors", AuthorWorkflowViewSet, basename="workflows-authors"),
+router.register(
+    "workflows/authors", AuthorWorkflowViewSet, basename="workflows-authors"
+),
 router.register("workflows", WorkflowViewSet, basename="workflows")
-router.register("workflow-update", WorkflowPartialUpdateViewSet, basename="workflow-update")
+router.register(
+    "workflow-update", WorkflowPartialUpdateViewSet, basename="workflow-update"
+)
 router.register("workflow-ticket", WorkflowTicketViewSet, basename="workflow-ticket"),
 app_name = "api"
 urlpatterns = router.urls

config/settings/base.py → backoffice/config/settings/base.py

@@ -58,7 +58,9 @@
             "NAME": env("POSTGRES_DB"),
             "USER": env("POSTGRES_USER"),
             "PASSWORD": env("POSTGRES_PASSWORD"),
-            "HOST": env("POSTGRES_HOST") if platform.system() == "Linux" else "localhost",
+            "HOST": env("POSTGRES_HOST")
+            if platform.system() == "Linux"
+            else "localhost",
             "PORT": env("POSTGRES_PORT"),
         }
     }
@@ -139,7 +141,9 @@
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#auth-password-validators
 AUTH_PASSWORD_VALIDATORS = [
-    {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
+    },
     {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
     {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
     {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
@@ -342,9 +346,11 @@
     "DEFAULT_AUTHENTICATION_CLASSES": (
         "rest_framework.authentication.SessionAuthentication",
         "rest_framework.authentication.TokenAuthentication",
-        'rest_framework_simplejwt.authentication.JWTAuthentication',
+        "rest_framework_simplejwt.authentication.JWTAuthentication",
+    ),
+    "DEFAULT_PERMISSION_CLASSES": (
+        "backoffice.management.permissions.IsAdminOrCuratorUser",
     ),
-    "DEFAULT_PERMISSION_CLASSES": ("backoffice.management.permissions.IsAdminOrCuratorUser",),
     "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
     "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
 }

config/settings/production.py → backoffice/config/settings/production.py

@@ -51,11 +51,15 @@
 # TODO: set this to 60 seconds first and then to 518400 once you prove the former works
 SECURE_HSTS_SECONDS = 60
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-include-subdomains
-SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True)
+SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool(
+    "DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True
+)
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-preload
 SECURE_HSTS_PRELOAD = env.bool("DJANGO_SECURE_HSTS_PRELOAD", default=True)
 # https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff
-SECURE_CONTENT_TYPE_NOSNIFF = env.bool("DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True)
+SECURE_CONTENT_TYPE_NOSNIFF = env.bool(
+    "DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True
+)
 # Metrics endpoint needs to be http for prometheus to access it
 SECURE_REDIRECT_EXEMPT = [r"^metrics$"]
 

config/settings/test.py → backoffice/config/settings/test.py

@@ -52,4 +52,3 @@
         "timeout": 30,
     },
 }
-

config/urls.py → backoffice/config/urls.py

@@ -11,7 +11,9 @@
 
 urlpatterns = [
     path("", TemplateView.as_view(template_name="pages/home.html"), name="home"),
-    path("about/", TemplateView.as_view(template_name="pages/about.html"), name="about"),
+    path(
+        "about/", TemplateView.as_view(template_name="pages/about.html"), name="about"
+    ),
     # Django Admin, use {% url 'admin:index' %}
     path(settings.ADMIN_URL, admin.site.urls),
     # User management