From 372ef2682b9ebbaefa01eb627f18453e8e1e810e Mon Sep 17 00:00:00 2001 From: Marcel Clausen Date: Wed, 31 Jan 2024 10:05:21 +0100 Subject: [PATCH 1/4] PB-81: post prod deploy clean-up these changes were needed during the parallel operation vhost/k8s the new runtime is k8s and we dont need the infra-vhost mountpoints anymore --- Dockerfile | 3 --- scripts/docker-cmd.sh | 11 +---------- scripts/index-sync-rotate.sh | 22 +--------------------- 3 files changed, 2 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 69d88c8..afd261c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,9 +28,6 @@ RUN groupadd -r geodata -g 2500 && \ useradd -u 2500 -r -g geodata -s /sbin/nologin --create-home geodata && \ # create mountpoint for Amazon EFS CSI driver install -o geodata -g geodata -d /var/local/ && \ - # create mountpoint for EFS mount in infra-vhost - # TODO: this mountpoint can be removed after the migration to k8s - install -o geodata -p geodata -d /var/lib/sphinxsearch/data/index_efs/ && \ # create mountpoint folder for infra-vhost/k8s ebs/ssd volume install -o geodata -g geodata -d /var/lib/sphinxsearch/data/index/ && \ # change ownerships to geodata which will run the service or the maintenance scripts diff --git a/scripts/docker-cmd.sh b/scripts/docker-cmd.sh index c33db94..5b4c380 100755 --- a/scripts/docker-cmd.sh +++ b/scripts/docker-cmd.sh @@ -7,16 +7,7 @@ green='\e[0;32m' NC='\e[0m' # No Color SPHINXINDEX_VOLUME="/var/lib/sphinxsearch/data/index/" -SPHINXINDEX_EFS="/var/lib/sphinxsearch/data/index_efs/" -K8S_EFS="/var/local/geodata/service-sphinxsearch/${DBSTAGING}/index/" - -# TODO: This switch can be removed after the migration to k8s -# in k8s we have to use /var/local/ as mountpoint for the index files from geodata efs -# /var/local/geodata/service-sphinxsearch/${DBSTAGING}/index/ -if [ -d "${K8S_EFS}" ]; then - echo "service is running on k8s, index files have been found on ${K8S_EFS}." - SPHINXINDEX_EFS="${K8S_EFS}" -fi +SPHINXINDEX_EFS="/var/local/geodata/service-sphinxsearch/${DBSTAGING}/index/" # remove lock files in volume rm ${SPHINXINDEX_VOLUME}*.spl 2> /dev/null || : diff --git a/scripts/index-sync-rotate.sh b/scripts/index-sync-rotate.sh index 3729610..1d0c5bf 100755 --- a/scripts/index-sync-rotate.sh +++ b/scripts/index-sync-rotate.sh @@ -2,8 +2,7 @@ # executed as cronjob every 15 minutes # shellcheck disable=SC2068 set -eu -SPHINX_EFS="/var/lib/sphinxsearch/data/index_efs/" -K8S_EFS="/var/local/geodata/service-sphinxsearch/${DBSTAGING}/index/" +SPHINX_EFS="/var/local/geodata/service-sphinxsearch/${DBSTAGING}/index/" SPHINX_VOLUME="/var/lib/sphinxsearch/data/index/" SPHINXCONFIG="/etc/sphinxsearch/sphinx.conf" @@ -36,7 +35,6 @@ RSYNC_INCLUDE="/tmp/include.txt" # .spp # .sps - json_logger() { log_level=$1 timestamp=$(date --utc +%FT%T.%3NZ) @@ -79,22 +77,6 @@ exlock_now() { _lock xn; } # obtain an exclusive lock immediately or fai exlock_now || { echo "locked" | json_logger INFO; exit 1; } echo "start" | json_logger INFO -# TODO: This switch can be removed after the migration to k8s -# in k8s we have to use /var/local/ as mountpoint for the index files from geodata efs -# /var/local/geodata/service-sphinxsearch/${DBSTAGING}/index/ -set_efs_source() { - # input: - # ${SPHINX_EFS} mountpoint of efs index files - #  - # output: SPHINX_EFS - # if the index files are available on the k8s mountpoint, the k8s mountpoint will be used - # as efs index source - if [ -d "${K8S_EFS}" ]; then - echo "service is running on k8s, index files have been found on ${K8S_EFS}." | json_logger INFO - SPHINX_EFS="${K8S_EFS}" - fi -} - check_if_efs_index_is_ready() { # input: # $1: index name @@ -159,8 +141,6 @@ check_if_local_index_is_ready() { return ${ready} } -set_efs_source - # loop through all indexes from sphinx config and sync them if the have been fully updated on efs for sphinx_index in ${SPHINX_INDEXES[@]}; do # create include-from file from sphinx config for selective rsync from EFS -> LOCAL From 012f330c465500aad96dd4f18e426fad35586694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Cl=C3=A4usen?= <5286659+ltclm@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:32:45 +0100 Subject: [PATCH 2/4] plz ortschaften switch to new datasource / stac --- conf/stopo.conf.part | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/stopo.conf.part b/conf/stopo.conf.part index 6205ad6..50fa2e7 100644 --- a/conf/stopo.conf.part +++ b/conf/stopo.conf.part @@ -88,7 +88,7 @@ source src_ch_swisstopo_vd_ortschaftenverzeichnis_plz : def_searchable_features , box2d(st_transform(the_geom, 2056)) as geom_st_box2d_lv95 \ , plz::text as label \ , os_uuid::text as feature_id \ - FROM vd.gabmo_plz + FROM vd.amtovz_ortschaften } source src_ch_swisstopo_vec200_names_namedlocation : def_searchable_features From 4cd040fc1da3758537a3e15723a583b7207a3022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Cl=C3=A4usen?= <5286659+ltclm@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:38:07 +0100 Subject: [PATCH 3/4] stable id is os_uuid (zip_id) --- conf/stopo.conf.part | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/stopo.conf.part b/conf/stopo.conf.part index 50fa2e7..2e728a1 100644 --- a/conf/stopo.conf.part +++ b/conf/stopo.conf.part @@ -77,7 +77,7 @@ source src_ch_swisstopo_vd_ortschaftenverzeichnis_plz : def_searchable_features { sql_db = stopo_${DBSTAGING} sql_query = \ - SELECT bgdi_id::int as id \ + SELECT os_uuid::int as id \ , 'feature' as origin \ , remove_accents(concat_ws(' ', plz, langtext)) as detail \ , 'ch.swisstopo-vd.ortschaftenverzeichnis_plz' as layer \ From ac5fc82dfec8573fef60091389875d69272f4d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Cl=C3=A4usen?= <5286659+ltclm@users.noreply.github.com> Date: Mon, 11 Mar 2024 09:07:28 +0100 Subject: [PATCH 4/4] BGDIDIC-642: add old feature ids to index this will allow to search for old feature ids once the migration to stac has been finished --- conf/lubis.conf.part | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/lubis.conf.part b/conf/lubis.conf.part index 6478578..842c8b2 100644 --- a/conf/lubis.conf.part +++ b/conf/lubis.conf.part @@ -26,7 +26,7 @@ source src_ch_swisstopo_lubis_luftbilder_schraegaufnahmen : def_searchable_featu SELECT row_number() OVER(ORDER BY ebkey asc) as id \ , concat_ws(' ',flightdate::date, ebkey::text) as label \ , 'feature' as origin \ - , concat_ws(' ',flightdate::date, ebkey::text) as detail \ + , concat_ws(' ',flightdate::date, ebkey::text, ebkey_old) as detail \ , 'ch.swisstopo.lubis-luftbilder_schraegaufnahmen' as layer \ , bgdi_quadindex as geom_quadindex \ , st_y(st_transform(st_centroid(the_geom),4326)) as lat \ @@ -81,7 +81,7 @@ source src_ch_swisstopo_lubis_luftbilder_infrarot: def_searchable_features_with_ sql_db = lubis_${DBSTAGING} sql_query = \ SELECT row_number() OVER(ORDER BY ebkey asc) as id \ - , concat(flugdatum, ' ', bildnummer, ' (', concat_ws(', ', ort, ebkey), ')' ) as label \ + , concat(flugdatum, ' ', bildnummer, ' (', concat_ws(', ', ort, ebkey, ebkey_old), ')' ) as label \ , 'feature' as origin \ , remove_accents(concat_ws(' ', ebkey, ort)) as detail \ , 'ch.swisstopo.lubis-luftbilder_infrarot' as layer \ @@ -140,7 +140,7 @@ source src_ch_swisstopo_lubis_terrestrische_aufnahmen : def_searchable_features_ SELECT row_number() OVER(ORDER BY inventory_number asc) as id \ , lpad(bilder.inventory_number::text, 14, '0'::text) as label \ , 'feature' as origin \ - , remove_accents(concat_ws(' ', inventory_number, image_number, station, operate_name, year)) as detail \ + , remove_accents(concat_ws(' ', inventory_number, image_number, station, operate_name, year, inventarnummer_old)) as detail \ , 'ch.swisstopo.lubis-terrestrische_aufnahmen' as layer \ , bgdi_quadindex as geom_quadindex \ , st_y(st_transform(st_centroid(the_geom),4326)) as lat \ @@ -149,7 +149,7 @@ source src_ch_swisstopo_lubis_terrestrische_aufnahmen : def_searchable_features_ , box2d(st_transform(the_geom_hilight, 2056)) as geom_st_box2d_lv95 \ , year \ , lpad(bilder.inventory_number::text, 14, '0'::text)::text as feature_id \ - FROM swisstopo_terrestrial.bilder + FROM swisstopo_terrestrial.view_bilder bilder } ## INDICES