From 8a671851e980ca2c5cc882705922fcc76de31e65 Mon Sep 17 00:00:00 2001 From: Henning Bredel <881756+ridoo@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:27:23 +0200 Subject: [PATCH] Create setup for datapackage developement (#12) Create development setup based on https://github.com/GeoNodeUserGroup-DE/geonode-blueprint-docker/ and GeoNode `v4.3.0`. --- .devcontainer/.env.local | 3 +- .devcontainer/_vscode/launch.json | 6 +-- .devcontainer/docker-compose.yml | 11 ++-- .devcontainer/docker/geoserver/Dockerfile | 2 +- .devcontainer/docker/postgresql/Dockerfile | 2 +- .gitmodules | 7 +-- README.md | 12 ++--- docker-compose.yml | 14 +++-- docker/geonode/Dockerfile | 13 ++++- docker/geonode/geonode-mapstore-client | 1 + docker/geonode/requirements.txt | 1 + docker/geoserver/Dockerfile | 2 +- docker/geoserver_data/Dockerfile | 2 +- docker/nginx/Dockerfile | 2 +- docker/postgresql/Dockerfile | 2 +- geonode/apps/customizations/__init__.py | 32 ----------- geonode/apps/customizations/apps.py | 32 +++++++++++ .../_geonode_config.html | 54 ++++++++++++------- geonode/apps/externalapplications | 1 - geonode/geonode/settings_override.py | 23 +++++--- sample.env | 8 ++- 21 files changed, 135 insertions(+), 95 deletions(-) create mode 160000 docker/geonode/geonode-mapstore-client create mode 100644 geonode/apps/customizations/apps.py delete mode 160000 geonode/apps/externalapplications diff --git a/.devcontainer/.env.local b/.devcontainer/.env.local index e06f7ba..621e992 100644 --- a/.devcontainer/.env.local +++ b/.devcontainer/.env.local @@ -4,11 +4,12 @@ # Docker sets placeholders using a pre-context (e.g. shell or the .env-file). # However, placeholders in this file will be used when variables are missing. +DEBUG=True # only use http in dev mode HTTP_HOST=${SITE_HOST_NAME} SITEURL="http://172.18.0.1:8001/" -# behinde the nginx +# behind the nginx GEONODE_LB_HOST_IP="django" GEONODE_LB_PORT="8001" diff --git a/.devcontainer/_vscode/launch.json b/.devcontainer/_vscode/launch.json index 2cde691..3245e28 100644 --- a/.devcontainer/_vscode/launch.json +++ b/.devcontainer/_vscode/launch.json @@ -3,7 +3,7 @@ "configurations": [ { "name": "Python: Django", - "type": "python", + "type": "debugpy", "request": "launch", "program": "./manage.py", "args": [ @@ -15,7 +15,7 @@ }, { "name": "Python: Celery", - "type": "python", + "type": "debugpy", "request": "launch", "program": "/usr/local/bin/celery", "args": [ @@ -39,7 +39,7 @@ }, { "name": "Django Test", - "type": "python", + "type": "debugpy", "request": "launch", "program": "${workspaceFolder}/manage.py", "args": [ diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index bf750a5..fa5e053 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,17 +1,14 @@ -version: '3.4' - services: django: env_file: - .devcontainer/.env.local restart: "no" volumes: - #- ./.devcontainer/docker/geonode/custom_entrypoint.sh:/usr/src/geonode/custom_entrypoint.sh + #- ../geonode:/usr/src/geonode - ./.devcontainer/_gitignore:/usr/src/geonode/.gitignore - ./.devcontainer/_vscode:/usr/src/geonode/.vscode ports: - "8001:8001" - #command: /usr/src/geonode/custom_entrypoint.sh command: sleep infinity celery: @@ -20,8 +17,6 @@ services: restart: "no" depends_on: - django - # volumes: - # - './src:/usr/src/geonode' geonode: env_file: @@ -36,7 +31,7 @@ services: context: ./.devcontainer/docker/geoserver dockerfile: Dockerfile args: - - BASE_IMAGE_VERSION=${GEONODE_VERSION} + - BASE_IMAGE_VERSION=${GEOSERVER_BASE_IMAGE_VERSION} environment: - OAUTH2_CLIENT_ID=${OAUTH2_CLIENT_ID} - OAUTH2_CLIENT_SECRET=${OAUTH2_CLIENT_SECRET} @@ -48,6 +43,6 @@ services: context: ./.devcontainer/docker/postgresql dockerfile: Dockerfile args: - - BASE_IMAGE_VERSION=${GEONODE_VERSION} + - BASE_IMAGE_VERSION=${POSTGRES_BASE_IMAGE_VERSION} ports: - "5432:5432" diff --git a/.devcontainer/docker/geoserver/Dockerfile b/.devcontainer/docker/geoserver/Dockerfile index ba0311e..544f9de 100644 --- a/.devcontainer/docker/geoserver/Dockerfile +++ b/.devcontainer/docker/geoserver/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE_VERSION -FROM 52north/geonode-geoserver:${BASE_IMAGE_VERSION} +FROM geonode/geoserver:${BASE_IMAGE_VERSION} # ENV GEOSERVER_DATA_DIR="/geoserver_data/data" diff --git a/.devcontainer/docker/postgresql/Dockerfile b/.devcontainer/docker/postgresql/Dockerfile index 1fbc6e3..1db028f 100644 --- a/.devcontainer/docker/postgresql/Dockerfile +++ b/.devcontainer/docker/postgresql/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE_VERSION -FROM 52north/geonode-postgis:${BASE_IMAGE_VERSION} +FROM geonode/postgis:${BASE_IMAGE_VERSION} COPY ./prepare-testdb.sh /docker-entrypoint-initdb.d/prepare-testdb.sh RUN chmod +x /docker-entrypoint-initdb.d/prepare-testdb.sh diff --git a/.gitmodules b/.gitmodules index e524410..faf2f87 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ -[submodule "geonode/externalapplications"] - path = geonode/apps/externalapplications - url = https://github.com/GeoNodeUserGroup-DE/contrib_externalapplications.git +[submodule "docker/geonode/geonode-mapstore-client"] + path = docker/geonode/geonode-mapstore-client + url = https://github.com/GeoNodeUserGroup-DE/geonode-mapstore-client/ + branch = datapackage_tabular-data diff --git a/README.md b/README.md index b3124e5..68898a2 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ All needed components are available as [Docker](https://www.docker.com/) images The whole setup is based on the [Docker Blueprint for a GeoNode Installation](https://github.com/GeoNodeUserGroup-DE/geonode-blueprint-docker). The blueprint is an opnionated GeoNode setup, but helps to keep everything necessary in one place to develop the datapackage feature which adds changes on multiple repositories: -- https://github.com/GeoNodeUserGroup-DE/geonode/ +- [GeoNodeUserGroup-DE/geonode/](https://github.com/GeoNodeUserGroup-DE/geonode/tree/datapackage_tabular-data) (branch `datapackage_tabular-data`) +- [GeoNodeUserGroup-DE/geonode-mapstore-client/](https://github.com/GeoNodeUserGroup-DE/geonode-mapstore-client/tree/datapackage_tabular-data) (Branch `datapackage_tabular-data`) - https://github.com/GeoNodeUserGroup-DE/geonode-importer/ -- https://github.com/GeoNodeUserGroup-DE/geonode-mapstore-client/ - https://github.com/GeoNodeUserGroup-DE/importer-datapackage/ Under `./.devcontainer` you find a configuration to run and debug the project as [`devcontainer`](https://containers.dev/). @@ -22,11 +22,11 @@ For detailed background information about the genesis of the blueprint and how t Make sure you have installed `git`, `Docker` and `docker compose`. -Clone the [repository containing a GeoNode Docker setup]( https://github.com/GeoNodeUserGroup-DE/geonode-blueprint-docker) and change directory your local working copy: +Clone the [repository containing a GeoNode Docker setup]( https://github.com/GeoNodeUserGroup-DE/geonode-dev-datapackage) and change directory your local working copy: ``` -git clone --recurse-submodules https://github.com/GeoNodeUserGroup-DE/geonode-blueprint-docker geonode -cd geonode +git clone --recurse-submodules https://github.com/GeoNodeUserGroup-DE/geonode-dev-datapackage geonode-dev-datapackage +cd geonode-dev-datapackage ``` ## Configuration @@ -35,7 +35,7 @@ cd geonode > > Settings (e.g. geodatabase parameters) are mainly configured in the `.env` file. > To review in-built default settings of an image, run the `env` command on an image. -> For example `docker run 52north/geonode-geoserver env | sort`. +> For example `docker run geonode/geoserver env | sort`. > > For a complete set of available options take the [GeoNode Settings](https://docs.geonode.org/en/master/basic/settings/index.html#settings) documentation as a reference. diff --git a/docker-compose.yml b/docker-compose.yml index d6d3c7a..64c1b53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,12 @@ -version: '3.4' - # Common Django template for GeoNode and Celery services below x-common-django: &default-common-django - image: ${DOCKER_REGISTRY}${DOCKER_REPOSITORY}${COMPOSE_PROJECT_NAME}_django:${GEONODE_VERSION}-${PROJECT_VERSION} + image: ${DOCKER_REGISTRY}${DOCKER_REPOSITORY}${COMPOSE_PROJECT_NAME}_django:${GEONODE_BASE_IMAGE_VERSION}-${PROJECT_VERSION} build: context: ./docker/geonode dockerfile: Dockerfile args: - - BASE_IMAGE_VERSION=${GEONODE_VERSION} + - BASE_IMAGE_VERSION=${GEONODE_BASE_IMAGE_VERSION} restart: no env_file: - .env @@ -71,7 +69,7 @@ services: context: ./docker/nginx dockerfile: Dockerfile args: - - BASE_IMAGE_VERSION=${GEONODE_VERSION} + - BASE_IMAGE_VERSION=${NGINX_BASE_IMAGE_VERSION} container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: ./.env environment: @@ -99,7 +97,7 @@ services: context: ./docker/geoserver dockerfile: Dockerfile args: - - BASE_IMAGE_VERSION=${GEONODE_VERSION} + - BASE_IMAGE_VERSION=${GEOSERVER_BASE_IMAGE_VERSION} container_name: geoserver4${COMPOSE_PROJECT_NAME} env_file: ./.env healthcheck: @@ -128,7 +126,7 @@ services: context: ./docker/geoserver_data dockerfile: Dockerfile args: - - BASE_IMAGE_VERSION=${GEONODE_VERSION} + - BASE_IMAGE_VERSION=${GEOSERVER_DATA_BASE_IMAGE_VERSION} container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: @@ -145,7 +143,7 @@ services: context: ./docker/postgresql dockerfile: Dockerfile args: - - BASE_IMAGE_VERSION=${GEONODE_VERSION} + - BASE_IMAGE_VERSION=${POSTGRES_BASE_IMAGE_VERSION} command: postgres -c "max_connections=${POSTGRES_MAX_CONNECTIONS}" container_name: db4${COMPOSE_PROJECT_NAME} env_file: ./.env diff --git a/docker/geonode/Dockerfile b/docker/geonode/Dockerfile index b803a00..d53f52b 100644 --- a/docker/geonode/Dockerfile +++ b/docker/geonode/Dockerfile @@ -1,7 +1,16 @@ ARG BASE_IMAGE_VERSION -FROM 52north/geonode:${BASE_IMAGE_VERSION} +FROM geonode/geonode:${BASE_IMAGE_VERSION} + +# we override geonode from base image here +# FIXME a submodule fails to build (error -> cannot replace directory .git to file) +RUN git remote add usergroup https://github.com/GeoNodeUserGroup-DE/geonode.git \ + && git fetch usergroup \ + && git stash save \ + && git switch datapackage_tabular-data \ + && git stash pop COPY ./requirements.txt /usr/src/more_requirements.txt RUN yes w | pip install --src /usr/src -r /usr/src/more_requirements.txt -#RUN rm .env* +COPY ./geonode-mapstore-client /usr/src/django_geonode_mapstore_client_dev +RUN yes w | pip install -e /usr/src/django_geonode_mapstore_client_dev diff --git a/docker/geonode/geonode-mapstore-client b/docker/geonode/geonode-mapstore-client new file mode 160000 index 0000000..716adf0 --- /dev/null +++ b/docker/geonode/geonode-mapstore-client @@ -0,0 +1 @@ +Subproject commit 716adf0b3c9d46c80383a95775da9fc6e2222eda diff --git a/docker/geonode/requirements.txt b/docker/geonode/requirements.txt index e69de29..aaf6755 100644 --- a/docker/geonode/requirements.txt +++ b/docker/geonode/requirements.txt @@ -0,0 +1 @@ +-e git+https://github.com/GeoNodeUserGroup-DE/importer-datapackage.git@7_support-geonode-430#egg=importer_datapackage diff --git a/docker/geoserver/Dockerfile b/docker/geoserver/Dockerfile index 36582d5..b469cfd 100644 --- a/docker/geoserver/Dockerfile +++ b/docker/geoserver/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE_VERSION -FROM 52north/geonode-geoserver:${BASE_IMAGE_VERSION} +FROM geonode/geoserver:${BASE_IMAGE_VERSION} ## GeoServer plugins # diff --git a/docker/geoserver_data/Dockerfile b/docker/geoserver_data/Dockerfile index b86292c..c1c60c2 100644 --- a/docker/geoserver_data/Dockerfile +++ b/docker/geoserver_data/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE_VERSION -FROM 52north/geonode-geoserver_data:${BASE_IMAGE_VERSION} +FROM geonode/geoserver_data:${BASE_IMAGE_VERSION} # configure logging RUN sed 's|false|true|' -i "/geoserver_data/data/logging.xml" diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 54e6378..1c64cbd 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE_VERSION -FROM 52north/geonode-nginx:${BASE_IMAGE_VERSION} +FROM geonode/nginx:${BASE_IMAGE_VERSION} COPY ./cert.pem /geonode-certificates/autoissued/fullchain.pem COPY ./key.pem /geonode-certificates/autoissued/privkey.pem diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile index 6c1b3f1..1c7d2fb 100644 --- a/docker/postgresql/Dockerfile +++ b/docker/postgresql/Dockerfile @@ -1,2 +1,2 @@ ARG BASE_IMAGE_VERSION -FROM 52north/geonode-postgis:${BASE_IMAGE_VERSION} \ No newline at end of file +FROM geonode/postgis:${BASE_IMAGE_VERSION} \ No newline at end of file diff --git a/geonode/apps/customizations/__init__.py b/geonode/apps/customizations/__init__.py index ba5503d..e69de29 100644 --- a/geonode/apps/customizations/__init__.py +++ b/geonode/apps/customizations/__init__.py @@ -1,32 +0,0 @@ -import os -from django.apps import AppConfig - - -def run_setup_hooks(*args, **kwargs): - from django.conf import settings - from django.conf.urls import url - from django.views.generic import TemplateView - from geonode.urls import urlpatterns - - LOCAL_ROOT = os.path.abspath(os.path.dirname(__file__)) - template_dir = os.path.join(LOCAL_ROOT, "templates") - settings.TEMPLATES[0]["DIRS"].insert(0, template_dir) - - urlpatterns += [ - url(r'^legal_notice/$', - TemplateView.as_view(template_name='legal-notice.html'), - name='legal-notice'), - url(r'^accessibility/$', - TemplateView.as_view(template_name='accessibility.html'), - name='accessibility'), - ] - - -class CustomizationsAppConfig(AppConfig): - name = 'customizations' - - def ready(self): - super().ready() - run_setup_hooks() - -default_app_config = 'customizations.CustomizationsAppConfig' diff --git a/geonode/apps/customizations/apps.py b/geonode/apps/customizations/apps.py new file mode 100644 index 0000000..0192a3f --- /dev/null +++ b/geonode/apps/customizations/apps.py @@ -0,0 +1,32 @@ +import os +from django.apps import AppConfig + + +def run_setup_hooks(*args, **kwargs): + from django.conf import settings + from django.urls import re_path + from django.views.generic import TemplateView + from geonode.urls import urlpatterns + + LOCAL_ROOT = os.path.abspath(os.path.dirname(__file__)) + template_dir = os.path.join(LOCAL_ROOT, "templates") + settings.TEMPLATES[0]["DIRS"].insert(0, template_dir) + + urlpatterns += [ + re_path(r'^legal_notice/$', + TemplateView.as_view(template_name='legal-notice.html'), + name='legal-notice'), + re_path(r'^accessibility/$', + TemplateView.as_view(template_name='accessibility.html'), + name='accessibility'), + ] + + +class CustomizationsAppConfig(AppConfig): + name = 'customizations' + + def ready(self): + super().ready() + run_setup_hooks() + +default_app_config = 'customizations.CustomizationsAppConfig' diff --git a/geonode/apps/customizations/templates/geonode-mapstore-client/_geonode_config.html b/geonode/apps/customizations/templates/geonode-mapstore-client/_geonode_config.html index be9be25..c3291e3 100644 --- a/geonode/apps/customizations/templates/geonode-mapstore-client/_geonode_config.html +++ b/geonode/apps/customizations/templates/geonode-mapstore-client/_geonode_config.html @@ -21,13 +21,13 @@ labelId: 'gnhome.uploadDataset', value: 'layer', type: 'link', - href: '/catalogue/#/upload/dataset' + href: '{context.getCataloguePath("/catalogue/#/upload/dataset")}' }, { labelId: 'gnhome.uploadDocument', value: 'document', type: 'link', - href: '/catalogue/#/upload/document' + href: '{context.getCataloguePath("/catalogue/#/upload/document")}' }, { labelId: 'gnhome.createDataset', @@ -40,19 +40,19 @@ labelId: 'gnhome.createMap', value: 'map', type: 'link', - href: '/catalogue/#/map/new' + href: '{context.getCataloguePath("/catalogue/#/map/new")}' }, { labelId: 'gnhome.createGeostory', value: 'geostory', type: 'link', - href: '/catalogue/#/geostory/new' + href: '{context.getCataloguePath("/catalogue/#/geostory/new")}' }, { labelId: 'gnhome.createDashboard', value: 'dashboard', type: 'link', - href: '/catalogue/#/dashboard/new' + href: '{context.getCataloguePath("/catalogue/#/dashboard/new")}' }, { labelId: 'gnhome.remoteServices', @@ -68,6 +68,9 @@ ] const resourcesGridPluginFiltersFormItems = [ + { + type: 'search' + }, { type: 'group', labelId: 'gnhome.customFiltersTitle', @@ -138,6 +141,11 @@ labelId: 'gnhome.maps', type: 'filter' }, + { + id: 'mapviewer', + labelId: 'gnhome.mapviewers', + type: 'filter' + }, { id: 'geostory', labelId: 'gnhome.geostories', @@ -155,28 +163,38 @@ disableIf: '{!state("user")}' }, { - labelId: 'gnhome.categories', - placeholderId: 'gnhome.categoriesPlaceholder', type: 'select', - suggestionsRequestKey: 'categories' + facet: "category" + }, + { + type: 'select', + facet: "keyword" }, { - labelId: 'gnhome.keywords', - placeholderId: 'gnhome.keywordsPlaceholder', type: 'select', - suggestionsRequestKey: 'keywords' + facet: 'place' }, { - labelId: 'gnhome.regions', - placeholderId: 'gnhome.regionsPlaceholder', type: 'select', - suggestionsRequestKey: 'regions' + facet: 'user' }, { - labelId: 'gnhome.owners', - placeholderId: 'gnhome.ownersPlaceholder', type: 'select', - suggestionsRequestKey: 'owners' + facet: "group" + }, + { + type: "accordion", + style: "facet", // style can be facet or filter (checkbox) + facet: "thesaurus" + }, + { + type: 'date-range', + filterKey: 'date', + labelId: 'gnviewer.dateFilter' + }, + { + labelId: 'gnviewer.extent', + type: 'extent' } ]; @@ -197,7 +215,7 @@ const plugin = localConfig.plugins[pluginPageName][i] const filtersFormItems = resourcesGridPluginFiltersFormItems - filtersFormItems[0].items.push({ + filtersFormItems[1].items.push({ id: "externalapplication", labelId: "externalapplications.filter", type: "filter" diff --git a/geonode/apps/externalapplications b/geonode/apps/externalapplications deleted file mode 160000 index f31c3e1..0000000 --- a/geonode/apps/externalapplications +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f31c3e140bfcfbec3684c46e9b8b6b87c1b85076 diff --git a/geonode/geonode/settings_override.py b/geonode/geonode/settings_override.py index a3be0da..e14376b 100644 --- a/geonode/geonode/settings_override.py +++ b/geonode/geonode/settings_override.py @@ -8,13 +8,19 @@ import sys -# DEBUG = "False" - -X_FRAME_OPTIONS = "SAMEORIGIN" - # sets defaults settings and from .env from geonode.settings import * -from geonode.settings import TEMPLATES, INSTALLED_APPS +from geonode.settings import ( + DEBUG, + TEMPLATES, + IMPORTER_HANDLERS, + INSTALLED_APPS, +) + +X_FRAME_OPTIONS = "SAMEORIGIN" +SECURE_CROSS_ORIGIN_OPENER_POLICY = None if DEBUG else "same-origin" +# required for geonode-mapstore-client development +CSRF_TRUSTED_ORIGINS = ["http://localhost:8081"] if DEBUG else [] STATIC_ROOT = "/mnt/volumes/statics/static/" @@ -103,8 +109,13 @@ }, } +IMPORTER_HANDLERS = ( + "importer_datapackage.handlers.datapackage.handler.DataPackageFileHandler", + *IMPORTER_HANDLERS, +) + INSTALLED_APPS += ( - "externalapplications", + "importer_datapackage", "customizations", ) diff --git a/sample.env b/sample.env index d7c87ca..7248a25 100644 --- a/sample.env +++ b/sample.env @@ -4,12 +4,18 @@ COMPOSE_PROJECT_NAME="geonode_docker" DOCKER_REGISTRY= # (optional) with trailing slash e.g. sample/ or orga/ DOCKER_REPOSITORY=my-geonode/ -GEONODE_VERSION="4.2.2" +GEONODE_BASE_IMAGE_VERSION="4.3.0" +NGINX_BASE_IMAGE_VERSION="1.25.3-v1" +GEOSERVER_BASE_IMAGE_VERSION="2.24.3-v1" +GEOSERVER_DATA_BASE_IMAGE_VERSION="2.24.3-v1" +POSTGRES_BASE_IMAGE_VERSION="15.3-v1" + PROJECT_VERSION="v1" # set to 'true' to re-initialize setup FORCE_REINIT=false +INVOKE_LOG_STDOUT=True ## External Site Settings