Fix harvest configs path

.github/workflows/main.yml

@@ -10,6 +10,15 @@ env:
   PROJECT_NAME: wri-odp
   BRANCH_NAME: dev
   ENABLE_TRIVY_SCANNING: true
+  ENABLE_BUILD_CKAN: false
+  ENABLE_BUILD_FRONTEND: false
+  ENABLE_BUILD_DATAPUSHER: false
+  ENABLE_BUILD_MIGRATION: false
+  ENABLE_INT_TESTS: false
+  ENABLE_UNIT_TESTS: true
+  ENABLE_FRONTEND_TESTS: false
+  ENABLE_DEBUG_LOGS: false
+  ENABLE_FRONTEND_DOCKER: false
 permissions:
   id-token: write
   contents: read
@@ -38,6 +47,7 @@ jobs:
           sudo echo "127.0.0.1 frontend" | sudo tee -a /etc/hosts
           sudo echo "127.0.0.1 prefect" | sudo tee -a /etc/hosts
       - name: Build and push CKAN image to ECR
+        if: ${{ env.ENABLE_BUILD_CKAN == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           CKAN_REPO: ${{ secrets.ECR_CKAN_REPO}}
@@ -47,6 +57,7 @@ jobs:
           docker build -t $REGISTRY/$CKAN_REPO:$IMAGE_TAG deployment/ckan
           docker push $REGISTRY/$CKAN_REPO:$IMAGE_TAG
       - name: Build and push Frontend image to ECR
+        if: ${{ env.ENABLE_BUILD_FRONTEND == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           FRONTEND_REPO: ${{ secrets.ECR_FRONTEND_REPO}}
@@ -66,6 +77,7 @@ jobs:
           deployment/frontend
           docker push $REGISTRY/$FRONTEND_REPO:$IMAGE_TAG
       - name: Build and push Datapusher image to ECR
+        if: ${{ env.ENABLE_BUILD_DATAPUSHER == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}}
@@ -75,6 +87,7 @@ jobs:
           datapusher
           docker push $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG
       - name: Build and push Migration image to ECR
+        if: ${{ env.ENABLE_BUILD_MIGRATION == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}}
@@ -85,14 +98,15 @@ jobs:
           docker push $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG-migration
       - name: Set up Docker Containers
         env:
-          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
+          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-680fae2f68e4e68e283617c1036b730572ffece8'
           SCRIPT_NAME: '/private-admin'
         run: |
           docker compose -f docker-compose.test.yml --env-file .env.example up --build -d
         working-directory: ./ckan-backend-dev
       - name: Initialize the Extensions
         run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Cypress Install and CKAN setup
+        if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           wait-on: 'http://localhost:5000/private-admin/en'
@@ -103,18 +117,21 @@ jobs:
       - name: Create sysadmin API for Authorization
         run: bash ./ckan-backend-dev/ckan/scripts/cypress_setup.sh
       - name: Set up Frontend Docker Container
+        if: ${{ env.ENABLE_FRONTEND_DOCKER == 'true' }}
         env:
-          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
+          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-680fae2f68e4e68e283617c1036b730572ffece8'
         run: docker compose -f docker-compose.test.yml --env-file .env.example up --build -d frontend
         working-directory: ./ckan-backend-dev
       - name: Initialize the Extensions
         run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Print Logs
+        if: ${{ env.ENABLE_DEBUG_LOGS == 'true' }}
         env:
-          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
+          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-680fae2f68e4e68e283617c1036b730572ffece8'
         run: docker compose -f docker-compose.test.yml --env-file .env.example logs
         working-directory: ./ckan-backend-dev
       - name: Cypress Install and Frontend setup
+        if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           wait-on: 'http://localhost:3000'
@@ -123,6 +140,7 @@ jobs:
           runTests: false
           working-directory: ./e2e-tests
       - name: Run Integration tests 🧪
+        if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           command: node test.js
@@ -137,6 +155,7 @@ jobs:
           # Fail this action after this many failed attempts
           max-attempts: 3 # Optional, defaults to 1
       - name: Run frontend tests 🧪
+        if: ${{ env.ENABLE_FRONTEND_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           command: npm run test
@@ -150,7 +169,7 @@ jobs:
       - name: Print Error Logs
         if: failure()
         env:
-          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
+          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-680fae2f68e4e68e283617c1036b730572ffece8'
         run: docker compose -f docker-compose.test.yml --env-file .env.example logs
         working-directory: ./ckan-backend-dev
       - name: Copy run_unit_tests.sh
@@ -172,6 +191,13 @@ jobs:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
         run: docker compose -f docker-compose.test.yml --env-file .env.example exec -T ckan-dev /bin/bash -c "/srv/app/run_unit_tests.sh"
         working-directory: ./ckan-backend-dev
+        timeout-minutes: 10
+      - name: Print Error Logs After Unit Tests
+        if: failure()
+        env:
+          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
+        run: docker compose -f docker-compose.test.yml --env-file .env.example logs
+        working-directory: ./ckan-backend-dev
       - name: Run Trivy Vulnerability Scanner for CKAN Container 🧪
         if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }}
         uses: aquasecurity/trivy-action@master

ckan-backend-dev/ckan/Dockerfile.dev

@@ -123,10 +123,11 @@ RUN ckan config-tool ${CKAN_INI} "ckan.root_path = /private-admin/{{LANG}}"
 
 COPY setup/who.ini ${APP_DIR}/src/ckan/who.ini
 
-COPY setup/supervisord.conf /etc/supervisord.d/supervisord.conf
+COPY setup/supervisord.conf /etc/supervisord.conf
 COPY setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
 COPY setup/supervisor.harvest.conf /etc/supervisord.d/harvest.conf
 
 RUN chown -R ckan:ckan /etc/supervisord.d
+RUN chown -R ckan:ckan /etc/supervisord.conf
 
 CMD ["sh", "-c", "${APP_DIR}/start_ckan_development.sh"]

ckan-backend-dev/ckan/scripts/run_unit_tests.sh

@@ -16,7 +16,7 @@ echo "Test Summary" > "$ROOT_DIR/test_summary.txt"
 if [ -d "src_extensions/ckanext-wri" ]; then
   cd src_extensions/ckanext-wri
 
-  pytest --ckan-ini=test.ini ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+  pytest --ckan-ini=test.ini -v --tb=long ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
   PYTEST_EXIT_CODE=${PIPESTATUS[0]}
 
   if [ $PYTEST_EXIT_CODE -eq 0 ]; then
@@ -30,7 +30,7 @@ fi
 if [ -d "$ROOT_DIR/src/ckanext-wri" ]; then
   cd $ROOT_DIR/src/ckanext-wri
 
-  pytest --ckan-ini=test.ini ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+  pytest --ckan-ini=test.ini -v --tb=long ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
   PYTEST_EXIT_CODE=${PIPESTATUS[0]}
 
   if [ $PYTEST_EXIT_CODE -eq 0 ]; then
@@ -52,7 +52,7 @@ for dir in ckanext-*; do
 
     cd $dir
 
-    pytest --ckan-ini=test.ini ckanext/${dir#ckanext-}/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+    pytest --ckan-ini=test.ini -v --tb=long ckanext/${dir#ckanext-}/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
     PYTEST_EXIT_CODE=${PIPESTATUS[0]}
 
     if [ $PYTEST_EXIT_CODE -eq 0 ]; then

ckan-backend-dev/ckan/setup/start_ckan_development.sh.override

@@ -94,7 +94,7 @@ then
 fi
 
 # Start supervisord
-supervisord --configuration /etc/supervisord.d/supervisord.conf &
+supervisord --configuration /etc/supervisord.conf &
 
 # Start the development server with automatic reload
 ckan -c $CKAN_INI run -H 0.0.0.0 --prefix /private-admin

ckan-backend-dev/docker-compose.test.yml

@@ -30,60 +30,6 @@ services:
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://ckan-dev:5000/private-admin/en"]
-
-  frontend:
-    container_name: wri-frontend
-    build:
-      context: ../deployment/frontend/
-      args:
-        - NEXT_PUBLIC_CKAN_URL=http://ckan-dev:5000/private-admin/en
-        - NEXT_PUBLIC_NEXTAUTH_URL=http://localhost:3000
-        - NEXT_PUBLIC_GTM_ID=AAAAAAAAA
-        - NEXT_PUBLIC_DISABLE_HOTJAR=disabled
-        - NEXT_PUBLIC_HOTJAR_ID=AAAAAAAAA
-        - NEXT_PUBLIC_GFW_API_KEY="1111"
-        - OSANO_URL="YOUR_OSANO_URL/osano.js"
-        - OKTA_CLIENT_ID=ffffffffffffffffffffffffffffffff
-        - OKTA_ISSUER=https://dev-000000.okta.com/oauth2/default
-        - OKTA_CLIENT_SECRET=ffffffffffffffffffffffffffffffff
-        - NEXT_PUBLIC_DEPLOYMENT_TYPE="dev"
-    environment:
-      - NEXTAUTH_SECRET=secret
-      - NEXTAUTH_URL=http://localhost:3000
-      - CKAN_URL=http://ckan-dev:5000/private-admin/en
-      - S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
-      - S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
-      - S3_BUCKET_NAME=ckan
-      - S3_BUCKET_REGION=us-east-1
-      - RW_API_KEY=${RW_API_KEY}
-      - AZURE_AD_TENANT_ID=${AZURE_AD_TENANT_ID}
-      - AZURE_AD_CLIENT_ID=${AZURE_AD_CLIENT_ID}
-      - AZURE_AD_CLIENT_SECRET=${AZURE_AD_CLIENT_SECRET}
-      - SYS_ADMIN_API_KEY=${SYS_ADMIN_API_KEY}
-      - SMTP_SERVER=${SMTP_SERVER}
-      - SMTP_PORT=${SMTP_PORT}
-      - SMTP_USER=${SMTP_USER}
-      - SMTP_PASSWORD=${SMTP_PASSWORD}
-      - SMTP_FROM=${SMTP_FROM}
-      - PREFECT_INTERNAL_URL=http://prefect:4200
-      - NEXT_PUBLIC_GFW_API_KEY="1111"
-      - OSANO_URL="YOUR_OSANO_URL/osano.js"
-      - OKTA_CLIENT_ID=ffffffffffffffffffffffffffffffff
-      - OKTA_ISSUER=https://dev-000000.okta.com/oauth2/default
-      - OKTA_CLIENT_SECRET=ffffffffffffffffffffffffffffffff
-      - NEXT_PUBLIC_DEPLOYMENT_TYPE="dev"
-    depends_on:
-      ckan-dev:
-        condition: service_healthy
-      prefect:
-        condition: service_healthy
-      my-datapusher:
-        condition: service_started
-    ports:
-      - "0.0.0.0:3000:3000"
-    healthcheck:
-      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:3000"]
-
   prefect:
     container_name: prefect-server
     build:
@@ -106,24 +52,6 @@ services:
       ckan-dev:
         condition: service_healthy
 
-  my-datapusher:
-    container_name: my-datapusher
-    build:
-      context: ../datapusher
-    restart: always
-    environment:
-      - PREFECT_API_URL=http://prefect:4200/api
-      - DEPLOYMENT_NAME=datapusher-dev
-      - CKAN_DATASTORE_READ_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
-      - CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
-      - S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
-      - S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
-      - S3_BUCKET_NAME=ckan
-      - S3_BUCKET_REGION=us-east-1
-    depends_on:
-      prefect:
-        condition: service_healthy
-
   db:
     container_name: ${POSTGRESQL_CONTAINER_NAME}
     build:
@@ -157,11 +85,11 @@ services:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
 
   redis:
-    container_name: ${REDIS_CONTAINER_NAME}
-    image: redis:${REDIS_VERSION}
-    restart: unless-stopped
+    container_name: redis
+    image: redis:3
     healthcheck:
-      test: ["CMD", "redis-cli", "-e", "QUIT"]
+      test: ["CMD", "redis-cli","ping"]
+    restart: unless-stopped
 
   minio:
     container_name: ${MINIO_CONTAINER_NAME}

deployment/ckan/Dockerfile

@@ -77,6 +77,10 @@ RUN ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}"
 RUN ckan config-tool ${CKAN_INI} "ckan.auth.create_user_via_web = false"
 RUN ckan config-tool ${CKAN_INI} "ckan.root_path = /private-admin/{{LANG}}"
 
+RUN if [ "$GITHUB_ACTIONS" = "true" ]; then \
+      ckan config-tool ${CKAN_INI} "ckan.cors.origin_allow_all = True"; \
+    fi
+
 USER root
 
 RUN chown -R ckan:ckan ${APP_DIR}/src/

deployment/ckan/setup/start_ckan.sh.override

@@ -39,7 +39,7 @@ export SCRIPT_NAME="/private-admin"
 if [ $? -eq 0 ]
 then
     # Start supervisord
-    supervisord --configuration /etc/supervisord.d/supervisord.conf &
+    supervisord --configuration /etc/supervisord.conf &
     # Start uwsgi
     uwsgi $UWSGI_OPTS
 else

deployment/ckan/setup/start_ckan_development.sh.override

@@ -93,7 +93,7 @@ then
 fi
 
 # Start supervisord
-supervisord --configuration /etc/supervisord.d/supervisord.conf &
+supervisord --configuration /etc/supervisord.conf &
 
 # Start the development server with automatic reload
 sudo -u ckan -EH ckan -c $CKAN_INI run -H 0.0.0.0 --prefix /private-admin