From d1f04c35def9deeb1ac1e297041214b0b6a2a846 Mon Sep 17 00:00:00 2001 From: Julia Haas Date: Tue, 23 Jan 2024 12:20:12 +0100 Subject: [PATCH 01/14] update administrative_boundaries() --- v.alkis.buildings.import.py | 58 ++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/v.alkis.buildings.import.py b/v.alkis.buildings.import.py index 1525ffe..345d72d 100644 --- a/v.alkis.buildings.import.py +++ b/v.alkis.buildings.import.py @@ -182,8 +182,8 @@ def url_response(url): return url -def administrative_boundaries(polygon_name): - """returns list of districts overlapping with AOI/region""" +def administrative_boundaries(aoi_name): + """Returns list of districts overlapping with AOI/region""" # url of administrative boundaries url = ( "https://daten.gdz.bkg.bund.de/produkte/vg/vg5000_0101/" @@ -196,53 +196,45 @@ def administrative_boundaries(polygon_name): "VG5000_KRS.shp", ) - # create tempdirectory for unzipping files - path_admin_boundaries = grass.tempdir() - # download administrative boundaries as .zip + # check if URL is reachable response = requests.get(url) if not response.status_code == 200: sys.exit( ( - "v.alkis.nutzung.import was stopped. The data of the" + "v.alkis.buildings.import was stopped. The data of the" "district boundaries are currently not available." ) ) - # unzip boundaries - zip_file = ZipFile(BytesIO(response.content)) - zip_file.extractall(path_admin_boundaries) - # import district shapefile - vec_all_district_vec = f"all_districts_vec_{os.getpid()}" - rmvecmaps.append(vec_all_district_vec) + # download and import administrative boundaries grass.run_command( - "v.import", - input=os.path.join(path_admin_boundaries, filename), - output=vec_all_district_vec, + "g.region", + vect=aoi_name, quiet=True, ) - # get district of AOI/region-polygon - vec_poly_district_vec = f"poly_districts_vec_{os.getpid()}" - rmvecmaps.append(vec_poly_district_vec) + vsi_command = f"/vsizip/vsicurl/{url}/{filename}" + districts_vec = f"all_districts_vec_{os.getpid()}" + rmvecmaps.append(districts_vec) grass.run_command( - "v.clip", - input=vec_all_district_vec, - clip=polygon_name, - output=vec_poly_district_vec, - flags="d", + "v.import", + input=vsi_command, + output=districts_vec, + extent="region", + overwrite=True, quiet=True, ) - if vec_poly_district_vec: - krs_list = list( - grass.parse_command( - "v.db.select", - map=vec_poly_district_vec, - columns="GEN", - flags="c", - ).keys() - ) - grass.message(krs_list) + # get district of AOI/region-polygon + krs_list = list( + grass.parse_command( + "v.db.select", + map=districts_vec, + columns="GEN", + flags="c", + ).keys() + ) + grass.message(krs_list) return krs_list From 408aef80a55c2570d9c30b232cf62efbb51b3f89 Mon Sep 17 00:00:00 2001 From: Julia Haas Date: Wed, 24 Jan 2024 17:36:23 +0100 Subject: [PATCH 02/14] add local file import and restructure addon --- download_urls.py | 87 +++++--- fs.py | 57 +++++ requirements.txt | 2 + v.alkis.buildings.import.html | 25 ++- v.alkis.buildings.import.py | 395 +++++++++++++++++++--------------- 5 files changed, 363 insertions(+), 203 deletions(-) create mode 100644 fs.py create mode 100644 requirements.txt diff --git a/download_urls.py b/download_urls.py index e7160ab..b2396ac 100644 --- a/download_urls.py +++ b/download_urls.py @@ -1,45 +1,66 @@ #!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: download_urls +# AUTHOR(S): Anika Weinmann, Julia Haas + +# PURPOSE: German Federal State download urls for ALKIS buildings +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS +# Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# URLS = { - "Brandenburg": None, - "Berlin": "https://datenbox.stadt-berlin.de/ssf/s/readFile/share" + "BW": None, + "BY": None, + "BE": "https://datenbox.stadt-berlin.de/ssf/s/readFile/share" "/14254/8799702476865788292/publicLink/SHP_BE_ALKIS.7z", - "Baden-Württemberg": None, - "Bayern": None, - "Bremen": None, - "Hessen": "https://gds.hessen.de/downloadcenter/DATE/" + "BB": None, + "HB": None, + "HH": None, + "HE": "https://gds.hessen.de/downloadcenter/DATE/" "Liegenschaftskataster/Hausumringe%20(shape)/Hausumringe%20Hessen.zip", - "Hamburg": None, - "Mecklenburg-Vorpommern": None, - "Niedersachsen": None, - "Nordrhein-Westfalen": "https://www.opengeodata.nrw.de/produkte/geobasis" + "MV": None, + "NI": None, + "NW": "https://www.opengeodata.nrw.de/produkte/geobasis" "/lk/akt/hu_shp/hu_EPSG4647_Shape.zip", - "Rheinland-Pfalz": None, - "Schleswig-Holstein": None, - "Saarland": None, - "Sachsen": "https://geocloud.landesvermessung.sachsen.de/index.php/s" + "RP": None, + "SL": None, + "SN": "https://geocloud.landesvermessung.sachsen.de/index.php/s" "/YgBfai4gXoiExJx/download?path=%2F&files=hu_sn_shape.zip", - "Sachsen-Anhalt": None, - "Thüringen": "https://geoportal.geoportal-th.de/hausko_umr/HU-TH.zip", + "ST": None, + "SH": None, + "TH": "https://geoportal.geoportal-th.de/hausko_umr/HU-TH.zip", } -filenames = { - "Brandenburg": None, - "Berlin": "SHP_BE_ALKIS_Merged/Gebaeude_Bauteile_Flaechen.shp", - "Baden-Württemberg": None, - "Bayern": None, - "Bremen": None, - "Hessen": "gebaeude-he.shp", - "Hamburg": None, - "Mecklenburg-Vorpommern": None, - "Niedersachsen": None, - "Nordrhein-Westfalen": "hu_shp.shp", - "Rheinland-Pfalz": None, - "Schleswig-Holstein": None, - "Saarland": None, - "Sachsen": "hu_sn_gebaeude_20231123.shp", - "Sachsen-Anhalt": None, - "Thüringen": "gebaeude-th.shp", +BUILDINGS_FILENAMES = { + "BW": None, + "BY": None, + "BE": "SHP_BE_ALKIS_Merged/Gebaeude_Bauteile_Flaechen.shp", + "BB": None, + "HB": None, + "HH": None, + "HE": "gebaeude-he.shp", + "MV": None, + "NI": None, + "NW": "hu_shp.shp", + "RP": None, + "SL": None, + "SN": "hu_sn_gebaeude_20231123.shp", + "ST": None, + "SH": None, + "TH": "gebaeude-th.shp", } BB_districts = { diff --git a/fs.py b/fs.py new file mode 100644 index 0000000..959ccba --- /dev/null +++ b/fs.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: fs +# AUTHOR(S): Anika Weinmann, Julia Haas + +# PURPOSE: German Federal State information as dictionary +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS +# Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# + +FS_ABBREVIATION = { + "Baden-Württemberg": "BW", + "BW": "BW", + "Bayern": "BY", + "BY": "BY", + "Berlin": "BE", + "BE": "BE", + "Brandenburg": "BB", + "BB": "BB", + "Bremen": "HB", + "HB": "HB", + "Hamburg": "HH", + "HH": "HH", + "Hessen": "HE", + "HE": "HE", + "Mecklenburg-Vorpommern": "MV", + "MV": "MV", + "Niedersachsen": "NI", + "NI": "NI", + "Nordrhein-Westfalen": "NW", + "NW": "NW", + "Rheinland-Pfalz": "RP", + "RP": "RP", + "Saarland": "SL", + "SL": "SL", + "Sachsen": "SN", + "SN": "SN", + "Sachsen-Anhalt": "ST", + "ST": "ST", + "Schleswig-Holstein": "SH", + "SH": "SH", + "Thüringen": "TH", + "TH": "TH", +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b7ffe62 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +py7zr +grass_gis_helpers \ No newline at end of file diff --git a/v.alkis.buildings.import.html b/v.alkis.buildings.import.html index 5d2cff3..39c89ac 100644 --- a/v.alkis.buildings.import.html +++ b/v.alkis.buildings.import.html @@ -1,6 +1,7 @@

DESCRIPTION

v.alkis.buildings.import downloads ALKIS building data and imports them into GRASS. +Alternatively, local data can be imported. To download the data, the federal_state(s) of the area of interest has to be given. Either given in a text file, comma-separated for multiple federal_states (file option). Or given directly as input, comma-separated for multiple federal_states (federal_state option). @@ -13,6 +14,7 @@

DESCRIPTION

Implemented federal state options are:

+For local data import the parameter local_data_dir has to be given and +the folder structure has to be as follows: +
+/path/to/ALKIS_buildings/
+├── BW/*.gpkg
+└── NW/*.shp
+└── ...
+
+If local data does not overlap with AOI, data will be downloaded from Open Data +portals if federal state supports Open Data. +

REQUIREMENTS

-
py7zr
needs to be installed. +
py7zr
, +
grass-gis-helpers
need to be installed.

EXAMPLE

@@ -57,6 +71,13 @@

Load ALKIS building data for current set region

v.alkis.buildings.import output=alkis_buildings federal_state=Nordrhein-Westfalen -r +

Load ALKIS building data from local file

+ +
+v.alkis.buildings.import aoi_map=AOI_BW output=buildings_bw federal_state=Baden-Württemberg local_data_dir=/path/to/ALKIS_buildings/
+
+ +

SEE ALSO

@@ -66,3 +87,5 @@

SEE ALSO

AUTHORS

Lina Krisztian, mundialis +Anika Weinmann, mundialis +Julia Haas, mundialis diff --git a/v.alkis.buildings.import.py b/v.alkis.buildings.import.py index 345d72d..087c888 100644 --- a/v.alkis.buildings.import.py +++ b/v.alkis.buildings.import.py @@ -40,6 +40,12 @@ # % description: Vector map to restrict ALKIS building import to # %end +# %option G_OPT_M_DIR +# % key: local_data_dir +# % required: no +# % description: Directory with vector map of ALKIS buildings to import +# %end + # %option G_OPT_F_INPUT # % key: file # % required: no @@ -93,13 +99,13 @@ from io import BytesIO from zipfile import ZipFile from time import sleep -import shutil from multiprocessing.pool import ThreadPool from datetime import datetime from datetime import timedelta import grass.script as grass import py7zr import requests +from grass_gis_helpers.cleanup import general_cleanup sys.path.insert( 1, @@ -108,53 +114,31 @@ ), ) # pylint: disable=wrong-import-position -from download_urls import URLS, filenames, BB_districts, download_dict +from download_urls import ( + URLS, + BUILDINGS_FILENAMES, + BB_districts, + download_dict, +) +from fs import FS_ABBREVIATION -ORIG_REGION = None +orig_region = None OUTPUT_ALKIS_TEMP = None -DLDIR = None +dldir = None PID = None currentpath = os.getcwd() -rmvecmaps = [] +rm_vectors = [] def cleanup(): """removes created objects when finished or failed""" - grass.message(_("Cleaning up...")) - os.chdir(currentpath) - nulldev = open(os.devnull, "w") - # if aoi_map is given: - if ORIG_REGION: - # set region back and delete saved region: - grass.run_command("g.region", region=ORIG_REGION) - grass.run_command( - "g.remove", - type="region", - name=ORIG_REGION, - flags="f", - quiet=True, - stderr=nulldev, - ) - # remove temp_output (if aoi_map given) - grass.run_command( - "g.remove", - type="vector", - name=OUTPUT_ALKIS_TEMP, - flags="f", - quiet=True, - stderr=nulldev, - ) - for rm_v in rmvecmaps: - grass.run_command( - "g.remove", - flags="f", - type="vector", - name=rm_v, - quiet=True, - stderr=nulldev, - ) + rm_dirs = [] if not flags["d"]: - shutil.rmtree(DLDIR) + rm_dirs.append(dldir) + + general_cleanup( + orig_region=orig_region, rm_vectors=rm_vectors, rm_dirs=rm_dirs + ) def url_response(url): @@ -215,7 +199,7 @@ def administrative_boundaries(aoi_name): vsi_command = f"/vsizip/vsicurl/{url}/{filename}" districts_vec = f"all_districts_vec_{os.getpid()}" - rmvecmaps.append(districts_vec) + rm_vectors.append(districts_vec) grass.run_command( "v.import", input=vsi_command, @@ -238,14 +222,14 @@ def administrative_boundaries(aoi_name): return krs_list -def download_brandenburg(aoi_map): +def download_alkis_buildings_bb(aoi_map): """Download and prepare data for Brandenburg""" # TODO check if data area already downloaded # select Landkreise if not aoi_map: aoi_map = f"aoi_region_{grass.tempname(12)}" - rmvecmaps.append(aoi_map) + rm_vectors.append(aoi_map) grass.run_command("v.in.region", output=aoi_map) krs_list = administrative_boundaries(aoi_map) filtered_urls = [] @@ -261,13 +245,13 @@ def download_brandenburg(aoi_map): ][0] kbs_zip = os.path.basename(kbs_url) kbs_zips.append(kbs_zip) - if not os.path.isfile(os.path.join(DLDIR, kbs_zip)): + if not os.path.isfile(os.path.join(dldir, kbs_zip)): filtered_urls.append(kbs_url) grass.message( _(f"Downloading {len(filtered_urls)} files from {len(kbs_zips)}...") ) - os.chdir(DLDIR) + os.chdir(dldir) pool = ThreadPool(3) results = pool.imap_unordered(url_response, filtered_urls) for result in results: @@ -279,10 +263,10 @@ def download_brandenburg(aoi_map): # for Brandenburg shape files shp_files = [] globstring = "ALKIS_Shape_*.zip" - zip_files = glob.glob(os.path.join(DLDIR, globstring)) + zip_files = glob.glob(os.path.join(dldir, globstring)) for zip_file in zip_files: zip_base_name = os.path.basename(zip_file) - shp_dir = os.path.join(DLDIR, zip_base_name.rsplit(".", 1)[0]) + shp_dir = os.path.join(dldir, zip_base_name.rsplit(".", 1)[0]) if not os.path.isdir(shp_dir): os.makedirs(shp_dir) if zip_base_name in kbs_zips: @@ -301,6 +285,52 @@ def download_brandenburg(aoi_map): return shp_files +def download_alkis_buildings(fs, url): + """download alkis building data""" + # create tempdirectory for unzipping files + # file of interest in zip + buildings_filename = BUILDINGS_FILENAMES[fs] + alkis_source = os.path.join(dldir, buildings_filename) + if not os.path.isfile(alkis_source): + grass.message(_("Downloading ALKIS building data...")) + if fs == "HE": + # insert current date into download URL + # try dates of yesterday and tomorrow if it's not working + today = datetime.now().strftime("%Y%m%d") + yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y%m%d") + tomorrow = (datetime.now() + timedelta(days=1)).strftime("%Y%m%d") + dates = [today, yesterday, tomorrow] + url = url.replace("DATE", today) + response = requests.get(url) + if not response.status_code == 200: + url = url.replace(dates[0], dates[1]) + response = requests.get(url) + if not response.status_code == 200: + url = url.replace(dates[1], dates[2]) + response = requests.get(url) + else: + response = requests.get(url) + + if not response.status_code == 200: + grass.fatal( + _( + "v.alkis.buildings.import was stopped." + "The data are currently not available." + ) + ) + # unzip boundaries + if url.endswith(".zip"): + zip_file = zipfile.ZipFile(BytesIO(response.content)) + zip_file.extractall(dldir) + elif url.endswith(".7z"): + zip_file = py7zr.SevenZipFile(BytesIO(response.content)) + zip_file.extractall(dldir) + else: + grass.fatal(_("Zip format not (yet) supported.")) + + return alkis_source + + def import_single_alkis_source( alkis_source, aoi_map, load_region, output_alkis, f_state ): @@ -409,7 +439,7 @@ def import_shapefiles(shape_files, output_alkis, aoi_map=None): grass.message(_(f"Importing {shape_file}")) out_temp = f"""out_temp_{PID}_ {os.path.splitext(os.path.basename(shape_file))[0]}""" - rmvecmaps.append(out_temp) + rm_vectors.append(out_temp) grass.run_command( "v.import", input=shape_file, @@ -463,157 +493,184 @@ def import_shapefiles(shape_files, output_alkis, aoi_map=None): ) +def patch_vector(vector_list, output): + # patch output from several federal states + if len(vector_list) > 1: + grass.run_command( + "v.patch", + input=vector_list, + output=output, + flags="e", + quiet=True, + ) + else: + grass.run_command("g.rename", vector=f"{vector_list[0]},{output}") + + +def import_local_data(aoi_map, local_data_dir, fs, output_alkis_fs): + """Import of data from local file path + + Args: + aoi_map (str): name of vector map defining AOI + local_data_dir (str): path to local data + fs (str): federal state abbreviation + output_alkis_fs (str): output for federal state + + Returns: + imported_local_data (bool): True if local data imported, otherwise False + """ + imported_local_data = False + # get files (GPKG or SHP) + buildings_files = glob.glob( + os.path.join(local_data_dir, fs, "**", "*.gpkg"), + recursive=True, + ) + shp_files = glob.glob( + os.path.join(local_data_dir, fs, "**", "*.shp"), recursive=True + ) + buildings_files.extend(shp_files) + + # import data for AOI + imported_buildings_list = [] + for i, buildings_file in enumerate(buildings_files): + if aoi_map: + grass.run_command( + "g.region", + vector=aoi_map, + quiet=True, + ) + grass.run_command( + "v.import", + input=buildings_file, + output=f"{output_alkis_fs}_{i}", + extent="region", + quiet=True, + ) + imported_buildings_list.append(f"{output_alkis_fs}_{i}") + rm_vectors.append(f"{output_alkis_fs}_{i}") + + # patch outputs + patch_vector(imported_buildings_list, output_alkis_fs) + + # check if result is not empty + buildings_info = grass.parse_command( + "v.info", + map=output_alkis_fs, + flags="gt", + ) + if int(buildings_info["centroids"]) == 0 and fs in ["BW"]: + grass.fatal(_("Local data does not overlap with AOI.")) + elif int(buildings_info["centroids"]) == 0: + grass.message( + _( + "Local data does not overlap with AOI. Data will be downloaded" + "from Open Data portal." + ) + ) + else: + imported_local_data = True + + return imported_local_data + + def main(): """main function for processing""" - global ORIG_REGION, OUTPUT_ALKIS_TEMP, PID, DLDIR + global orig_region, OUTPUT_ALKIS_TEMP, PID, dldir PID = os.getpid() # parser options: aoi_map = options["aoi_map"] file_federal_state = options["file"] load_region = flags["r"] - DLDIR = options["dldir"] + local_data_dir = options["local_data_dir"] + dldir = options["dldir"] OUTPUT_ALKIS_TEMP = f"OUTPUT_ALKIS_TEMP_{PID}" - rmvecmaps.append(OUTPUT_ALKIS_TEMP) + rm_vectors.append(OUTPUT_ALKIS_TEMP) output_alkis = options["output"] - # temp download path, if not explicite path given - if not DLDIR: - DLDIR = grass.tempdir() + # temp download path, if not explicit path given + if not dldir: + dldir = grass.tempdir() else: - if not os.path.exists(DLDIR): + if not os.path.exists(dldir): grass.message( - _(f"Download folder {DLDIR} does not exist. Creating it...") + _(f"Download folder {dldir} does not exist. Creating it...") ) - os.makedirs(DLDIR) + os.makedirs(dldir) # get federal state if file_federal_state: with open(file_federal_state) as file: - federal_states = file.read() + federal_states = file.read().strip() else: - federal_states = options["federal_state"] + federal_states = options["federal_state"].strip() + + # get list of local input folders for federal states + local_fs_list = os.listdir(local_data_dir) # region - ORIG_REGION = f"ORIG_REGION{PID}" + orig_region = f"ORIG_REGION{PID}" # save current region for setting back later in cleanup - grass.run_command("g.region", save=ORIG_REGION, quiet=True) - - # get URL for corresponding federal state - url = None - f_state = None - # prevent error caused by "\n" in federal states input file - if "\n" in federal_states: - federal_states = federal_states[:-1] + grass.run_command("g.region", save=orig_region, quiet=True) + + # loop over federal state and import data + output_alkis_list = [] for federal_state in federal_states.split(","): - if federal_state in URLS: - if federal_state in [ - "Nordrhein-Westfalen", - "Berlin", - "Hessen", - "Thüringen", - "Sachsen", - ]: - url = URLS[federal_state] - f_state = federal_state - elif federal_state == "Brandenburg": - f_state = federal_state - url = URLS[federal_state] - else: - grass.warning( - _(f"Support for {federal_state} is not yet implemented.") - ) - else: - if options["file"]: - grass.fatal( - _( - "Non valid name of federal state," - " in 'file'-option given" - ) - ) - elif options["federal_state"]: - grass.fatal( - _( - "Non valid name of federal state," - " in 'federal_states'-option given" - ) - ) - # so far, just Berlin, Brandenburg, Hessen, NRW and Sachsen are implemented; - # in case single federal state given, and not NRW: - # skips following part - # + grass.message: see above - # in case multiple federal states given, and at least one of them is NRW: - # import data only for NRW area - if not url and f_state == "Brandenburg": - alkis_source = download_brandenburg(aoi_map) - elif not url and not f_state: - grass.fatal( - _( - "AOI is located in federal state(s)," - "which are not yet implemented." + if federal_state not in FS_ABBREVIATION: + grass.fatal(_(f"Non valid name of federal state: {federal_state}")) + fs = FS_ABBREVIATION[federal_state] + output_alkis_fs = f"{output_alkis}_{fs}" + output_alkis_list.append(output_alkis_fs) + rm_vectors.append(output_alkis_fs) + + # check if local data for federal state given + imported_local_data = False + if fs in local_fs_list: + imported_local_data = import_local_data( + aoi_map, local_data_dir, fs, output_alkis_fs + ) + elif fs in ["BW"]: + grass.fatal( + _(f"No local data for {fs} available. Is the path correct?") ) - ) - if url: - """download alkis building data""" - # create tempdirectory for unzipping files - # file of interest in zip - filename = filenames[f_state] - alkis_source = os.path.join(DLDIR, filename) - if not os.path.isfile(alkis_source): - grass.message(_("Downloading ALKIS building data...")) - if federal_state == "Hessen": - # insert current date into download URL - # try dates of yesterday and tomorrow if it's not working - today = datetime.now().strftime("%Y%m%d") - yesterday = (datetime.now() - timedelta(days=1)).strftime( - "%Y%m%d" - ) - tomorrow = (datetime.now() + timedelta(days=1)).strftime( - "%Y%m%d" + # check if federal state is supported + if not imported_local_data: + if fs in ["NW", "BE", "HE", "TH", "SN"]: + url = URLS[fs] + elif fs in ["BB"]: + pass + else: + grass.warning(_(f"Support for {fs} is not yet implemented.")) + + # so far, just Berlin, Brandenburg, Hessen, NRW and Sachsen are implemented; + # in case single federal state given, and not NRW: + # skips following part + # + grass.message: see above + # in case multiple federal states given, and at least one of them is NRW: + # import data only for NRW area + if fs in ["BB"]: + alkis_source = download_alkis_buildings_bb(aoi_map) + else: + alkis_source = download_alkis_buildings(fs, url) + + # import to GRASS DB + grass.message(_("Importing ALKIS buildings data...")) + if isinstance(alkis_source, str): + import_single_alkis_source( + alkis_source, + aoi_map, + load_region, + output_alkis_fs, + federal_state, ) - dates = [today, yesterday, tomorrow] - url = url.replace("DATE", today) - response = requests.get(url) - if not response.status_code == 200: - url = url.replace(dates[0], dates[1]) - response = requests.get(url) - if not response.status_code == 200: - url = url.replace(dates[1], dates[2]) - response = requests.get(url) else: - response = requests.get(url) + import_shapefiles(alkis_source, output_alkis_fs, aoi_map) - if not response.status_code == 200: - sys.exit( - ( - "v.alkis.buildings.import was stopped." - "The data are currently not available." - ) - ) - # unzip boundaries - if federal_state in [ - "Nordrhein-Westfalen", - "Hessen", - "Thüringen", - "Sachsen", - ]: - zip_file = zipfile.ZipFile(BytesIO(response.content)) - zip_file.extractall(DLDIR) - elif federal_state == "Berlin": - zip_file = py7zr.SevenZipFile(BytesIO(response.content)) - zip_file.extractall(DLDIR) - """ import to GRASS DB - """ - grass.message(_("Importing ALKIS building data...")) - if isinstance(alkis_source, str): - import_single_alkis_source( - alkis_source, aoi_map, load_region, output_alkis, federal_state - ) - else: - import_shapefiles(alkis_source, output_alkis, aoi_map) + # patch output from several federal states + patch_vector(output_alkis_list, output_alkis) - grass.message(_("Done importing ALKIS building data.")) + grass.message(_(f"Importing ALKIS buildings data <{output_alkis}> done.")) if __name__ == "__main__": From de107306e67ad15ff9a02659432b62d2a13fa42f Mon Sep 17 00:00:00 2001 From: Julia Haas Date: Tue, 30 Jan 2024 14:19:44 +0100 Subject: [PATCH 03/14] add requirements, change html --- requirements.txt | 2 +- v.alkis.buildings.import.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index b7ffe62..9c5da79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ py7zr -grass_gis_helpers \ No newline at end of file +grass_gis_helpers diff --git a/v.alkis.buildings.import.html b/v.alkis.buildings.import.html index 39c89ac..981a331 100644 --- a/v.alkis.buildings.import.html +++ b/v.alkis.buildings.import.html @@ -22,7 +22,7 @@

DESCRIPTION

  • Sachsen
    • -For local data import the parameter local_data_dir has to be given and +For local data import the parameter local_data_dir has to be given and the folder structure has to be as follows: 
     /path/to/ALKIS_buildings/
@@ -30,7 +30,7 @@ 
    DESCRIPTION
    
 └── NW/*.shp
 └── ...
    -If local data does not overlap with AOI, data will be downloaded from Open Data +If local data does not overlap with AOI, data will be downloaded from Open Data portals if federal state supports Open Data.

    REQUIREMENTS

    From 6a9a07acee2b8529e67eb1deb529a2473e5bdf59 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Tue, 30 Jan 2024 14:37:41 +0100 Subject: [PATCH 04/14] fix installation --- Makefile | 2 +- fs.py => federal_state_info.py | 2 +- requirements.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) rename fs.py => federal_state_info.py (97%) diff --git a/Makefile b/Makefile index 40cd6fe..9e2e8e0 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ MODULE_TOPDIR = ../.. PGM = v.alkis.buildings.import -ETCFILES = download_urls +ETCFILES = download_urls federal_state_info include $(MODULE_TOPDIR)/include/Make/Python.make include $(MODULE_TOPDIR)/include/Make/Script.make diff --git a/fs.py b/federal_state_info.py similarity index 97% rename from fs.py rename to federal_state_info.py index 959ccba..390fddd 100644 --- a/fs.py +++ b/federal_state_info.py @@ -2,7 +2,7 @@ # ############################################################################ # -# MODULE: fs +# MODULE: federal_state_info # AUTHOR(S): Anika Weinmann, Julia Haas # PURPOSE: German Federal State information as dictionary diff --git a/requirements.txt b/requirements.txt index 9c5da79..716a583 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ py7zr grass_gis_helpers +requests \ No newline at end of file From 2cedaed5d44560862c73fdf0e38f0b84f2ccd5e4 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Tue, 30 Jan 2024 14:56:01 +0100 Subject: [PATCH 05/14] fix local data dir --- v.alkis.buildings.import.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/v.alkis.buildings.import.py b/v.alkis.buildings.import.py index 087c888..0926c0f 100644 --- a/v.alkis.buildings.import.py +++ b/v.alkis.buildings.import.py @@ -120,7 +120,7 @@ BB_districts, download_dict, ) -from fs import FS_ABBREVIATION +from federal_state_info import FS_ABBREVIATION orig_region = None OUTPUT_ALKIS_TEMP = None @@ -606,7 +606,9 @@ def main(): federal_states = options["federal_state"].strip() # get list of local input folders for federal states - local_fs_list = os.listdir(local_data_dir) + local_fs_list = [] + if local_data_dir and local_data_dir != "": + local_fs_list = os.listdir(local_data_dir) # region orig_region = f"ORIG_REGION{PID}" From ef054921dae3c967c2a655b40149fef0b6fcbfea Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 31 Jan 2024 16:01:40 +0100 Subject: [PATCH 06/14] restructure old tests --- .../data/{singleFs.txt => fs_file_NW.txt} | 0 ...ea_cologne.geojson => test_aoi_NW.geojson} | 0 testsuite/test_v_alkis_buildings_import.py | 265 ------------------ testsuite/test_v_alkis_buildings_import_NW.py | 45 +++ .../test_v_alkis_buildings_import_multi_fs.py | 154 ++++++++++ testsuite/v_alkis_buildings_import_base.py | 198 +++++++++++++ v.alkis.buildings.import.py | 60 +++- 7 files changed, 449 insertions(+), 273 deletions(-) rename testsuite/data/{singleFs.txt => fs_file_NW.txt} (100%) rename testsuite/data/{area_cologne.geojson => test_aoi_NW.geojson} (100%) delete mode 100644 testsuite/test_v_alkis_buildings_import.py create mode 100644 testsuite/test_v_alkis_buildings_import_NW.py create mode 100644 testsuite/test_v_alkis_buildings_import_multi_fs.py create mode 100644 testsuite/v_alkis_buildings_import_base.py diff --git a/testsuite/data/singleFs.txt b/testsuite/data/fs_file_NW.txt similarity index 100% rename from testsuite/data/singleFs.txt rename to testsuite/data/fs_file_NW.txt diff --git a/testsuite/data/area_cologne.geojson b/testsuite/data/test_aoi_NW.geojson similarity index 100% rename from testsuite/data/area_cologne.geojson rename to testsuite/data/test_aoi_NW.geojson diff --git a/testsuite/test_v_alkis_buildings_import.py b/testsuite/test_v_alkis_buildings_import.py deleted file mode 100644 index 56fecc8..0000000 --- a/testsuite/test_v_alkis_buildings_import.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python3 -# -############################################################################ -# -# MODULE: v.alkis.buildings.import test -# AUTHOR(S): Lina Krisztian - -# PURPOSE: Tests v.alkis.buildings.import -# COPYRIGHT: (C) 2022 by mundialis GmbH & Co. KG and the GRASS Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -############################################################################# - -import os - -from grass.gunittest.case import TestCase -from grass.gunittest.main import test -from grass.gunittest.gmodules import SimpleModule -import grass.script as grass - - -class TestVGetAlkisBuildings(TestCase): - """tests functionality of v.alkis.buildings.import.py""" - - pid = os.getpid() - aoi_map_data = os.path.join("data", "area_beuel.geojson") - aoi_map = f"aoi_map_{pid}" - test_output = f"test_output_{pid}" - region_map_data = os.path.join("data", "area_cologne.geojson") - region_map = f"region_map_{pid}" - aoi_map_multi_data = os.path.join("data", "area_nw_he.geojson") - aoi_map_multi = f"aoi_map_multi_{pid}" - aoi_map_multi_c_data = os.path.join( - "data", "area_germany_netherlands.geojson" - ) - aoi_map_multi_c = f"aoi_map_multi_c_{pid}" - - @classmethod - # pylint: disable=invalid-name - def setUpClass(cls): - """Ensures expected computational region and generated data""" - # import vector map to set as region - cls.runModule( - "v.import", input=cls.region_map_data, output=cls.region_map - ) - cls.runModule("g.region", vector=cls.region_map) - # import vector map to test aoi_map option - cls.runModule("v.import", input=cls.aoi_map_data, output=cls.aoi_map) - # import vector map to test aoi_map located in multiple federal states - cls.runModule( - "v.import", - input=cls.aoi_map_multi_data, - output=cls.aoi_map_multi, - ) - # import vector map to test aoi_map located not only in Germany - cls.runModule( - "v.import", - input=cls.aoi_map_multi_c_data, - output=cls.aoi_map_multi_c, - ) - - @classmethod - # pylint: disable=invalid-name - def tearDownClass(cls): - """Remove the temporary region and generated data""" - cls.runModule( - "g.remove", type="vector", name=cls.region_map, flags="f" - ) - cls.runModule("g.remove", type="vector", name=cls.aoi_map, flags="f") - cls.runModule( - "g.remove", type="vector", name=cls.aoi_map_multi, flags="f" - ) - cls.runModule( - "g.remove", type="vector", name=cls.aoi_map_multi_c, flags="f" - ) - - # pylint: disable=invalid-name - def tearDown(self): - """Remove the outputs created - This is executed after each test run. - """ - self.runModule( - "g.remove", type="vector", name=self.test_output, flags="f" - ) - - def test_option_aoi_map(self): - """tests aoi_map as optional input""" - v_check = SimpleModule( - "v.alkis.buildings.import", - output=self.test_output, - federal_state="Nordrhein-Westfalen", - aoi_map=self.aoi_map, - ) - self.assertModule(v_check, "Using aoi_map as optional input fails") - # Data should have following columns: - # cat, AGS, OI, GFK, AKTUALITAE - atr_dict = grass.parse_command( - "v.info", map=self.test_output, flags="c" - ) - atr = list(atr_dict.keys()) - self.assertTrue( - "AGS" in atr[1], "Module failed, because of missins key 'AGS'" - ) - - def test_option_aoi_map_multi_fs(self): - """tests aoi_map as optional input - with aoi located in multiple federal states - """ - v_check = SimpleModule( - "v.alkis.buildings.import", - output=self.test_output, - federal_state=["Nordrhein-Westfalen", "Hessen"], - aoi_map=self.aoi_map_multi, - ) - self.assertModule( - v_check, - "Using aoi_map, which is located in" - "multiple federal states, fails", - ) - # Data should have following columns: - # cat, AGS, OI, GFK, AKTUALITAE - atr_dict = grass.parse_command( - "v.info", map=self.test_output, flags="c" - ) - atr = list(atr_dict.keys()) - self.assertTrue( - "AGS" in atr[1], "Module failed, because of missins key 'AGS'" - ) - - def test_option_aoi_map_multi_country(self): - """tests aoi_map as optional input - with aoi located only partly in Germany - """ - v_check = SimpleModule( - "v.alkis.buildings.import", - output=self.test_output, - federal_state="Nordrhein-Westfalen", - aoi_map=self.aoi_map_multi_c, - ) - self.assertModule( - v_check, - "Using aoi_map, which is located in" - "in Germany and the Netherlands fails", - ) - # Data should have following columns: - # cat, AGS, OI, GFK, AKTUALITAE - atr_dict = grass.parse_command( - "v.info", map=self.test_output, flags="c" - ) - atr = list(atr_dict.keys()) - self.assertTrue( - "AGS" in atr[1], "Module failed, because of missins key 'AGS'" - ) - - def test_flag(self): - """tests -r flag""" - # set some region in setUp class (here some region in cologne): - v_check = SimpleModule( - "v.alkis.buildings.import", - output=self.test_output, - federal_state="Nordrhein-Westfalen", - flags="r", - ) - self.assertModule(v_check, "Using -r flag fails") - # Data should have following columns: - # cat, AGS, OI, GFK, AKTUALITAE - atr_dict = grass.parse_command( - "v.info", map=self.test_output, flags="c" - ) - atr = list(atr_dict.keys()) - self.assertTrue( - "AGS" in atr[1], "Module failed, because of missins key 'AGS'" - ) - - def test_correct_region(self): - """tests if region is set back correctly""" - # set some region in setUp class (here some region in cologne): - # get this info before running the addon - region_exp = self.runModule("g.region", flags="g") - # then run v.alkis.buildings.import with aoi_map (which changes region) - self.runModule( - "v.alkis.buildings.import", - output=self.test_output, - federal_state="Nordrhein-Westfalen", - aoi_map=self.aoi_map, - ) - # region after running module - region_is = self.runModule("g.region", flags="g") - # check if current region is expected region - self.assertTrue( - region_exp == region_is, "Region was modified within addon." - ) - # Data should have following columns: - # cat, AGS, OI, GFK, AKTUALITAE - atr_dict = grass.parse_command( - "v.info", map=self.test_output, flags="c" - ) - atr = list(atr_dict.keys()) - self.assertTrue( - "AGS" in atr[1], "Module failed, because of missins key 'AGS'" - ) - - def test_file_input_single(self): - """tests file as input option for federal state(s) information - single federal state - case - """ - # single federal state in file - v_check = SimpleModule( - "v.alkis.buildings.import", - output=self.test_output, - aoi_map=self.aoi_map, - file=os.path.join("data", "singleFs.txt"), - ) - self.assertModule( - v_check, - "Module fails, when file-input with single federal state given", - ) - # Data should have following columns: - # cat, AGS, OI, GFK, AKTUALITAE - atr_dict = grass.parse_command( - "v.info", map=self.test_output, flags="c" - ) - atr = list(atr_dict.keys()) - self.assertTrue( - "AGS" in atr[1], "Module failed, because of missins key 'AGS'" - ) - - def test_file_input_multi(self): - """tests file as input option for federal state(s) information - multiple federal state - case - """ - # multiple (two) federal states in file - v_check = SimpleModule( - "v.alkis.buildings.import", - output=self.test_output, - aoi_map=self.aoi_map_multi, - file=os.path.join("data", "multiFs.txt"), - ) - self.assertModule( - v_check, - "Module fails, when file-input" - "with multiple federal states given", - ) - # Data should have following columns: - # cat, AGS, OI, GFK, AKTUALITAE - atr_dict = grass.parse_command( - "v.info", map=self.test_output, flags="c" - ) - atr = list(atr_dict.keys()) - self.assertTrue( - "AGS" in atr[1], "Module failed, because of missins key 'AGS'" - ) - - -if __name__ == "__main__": - test() diff --git a/testsuite/test_v_alkis_buildings_import_NW.py b/testsuite/test_v_alkis_buildings_import_NW.py new file mode 100644 index 0000000..512c0c2 --- /dev/null +++ b/testsuite/test_v_alkis_buildings_import_NW.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: v.alkis.buildings.import test for NW +# AUTHOR(S): Lina Krisztian, Anika Weinmann +# PURPOSE: Tests v.alkis.buildings.import for NW +# COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS +# Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# + +from grass.gunittest.main import test + +from v_alkis_buildings_import_base import VAlkisBuildingsImportTestFsBase + + +class VAlkisBuildingsImportTestNW(VAlkisBuildingsImportTestFsBase): + fs = "NW" + federal_state = "Nordrhein-Westfalen" + + def test_option_aoi_map(self): + """Tests aoi_map as optional input""" + self.option_aoi_map() + + def test_flag(self): + """Tests region as AOI input""" + self.flag() + + def test_file_input_single(self): + """Tests aoi_map as optional input and federal state input file""" + self.file_input_single() + +if __name__ == "__main__": + test() diff --git a/testsuite/test_v_alkis_buildings_import_multi_fs.py b/testsuite/test_v_alkis_buildings_import_multi_fs.py new file mode 100644 index 0000000..a7178ca --- /dev/null +++ b/testsuite/test_v_alkis_buildings_import_multi_fs.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: v.alkis.buildings.import test base +# AUTHOR(S): Lina Krisztian, Anika Weinmann +# PURPOSE: Testbase v.alkis.buildings.import +# COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS +# Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# + +import os + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test +from grass.gunittest.gmodules import SimpleModule +import grass.script as grass + +from v_alkis_buildings_import_base import VAlkisBuildingsImportTestBase + + +class VAlkisBuildingsImportTestMulitpleFS(VAlkisBuildingsImportTestBase): + """Test base for the test functionality of v.alkis.buildings.import with + multiple federal states""" + + aoi_map_multi_data = os.path.join("data", "area_nw_he.geojson") + aoi_map_multi_county_data = os.path.join( + "data", "area_germany_netherlands.geojson" + ) + + def test_option_aoi_map_multi_fs(self): + """Tests aoi_map as optional input + with aoi located in multiple federal states (NW, HE) + """ + print( + "Running tests with AOI in multiple federal states (NW and HE)..." + ) + self.runModule( + "v.import", + input=self.aoi_map_multi_data, + output=self.aoi_map, + overwrite=True, + ) + v_check = SimpleModule( + "v.alkis.buildings.import", + output=self.test_output, + federal_state=["Nordrhein-Westfalen", "Hessen"], + aoi_map=self.aoi_map, + ) + self.assertModule( + v_check, + "Using aoi_map, which is located in" + "multiple federal states, fails", + ) + # Data should have following columns: + # cat, AGS, OI, GFK + atr_dict = grass.parse_command( + "v.info", map=self.test_output, flags="c" + ) + atr = list(atr_dict.keys()) + self.assertTrue( + "AGS" in atr[1], "Module failed, because of missins key 'AGS'" + ) + print( + "Running tests with AOI in multiple federal states (NW and HE) " + "done." + ) + + def test_option_aoi_map_multi_country(self): + """Tests aoi_map as optional input + with aoi located only partly in Germany + """ + print("Running tests with AOI only partly in Germany...") + self.runModule( + "v.import", + input=self.aoi_map_multi_county_data, + output=self.aoi_map, + overwrite=True, + ) + v_check = SimpleModule( + "v.alkis.buildings.import", + output=self.test_output, + federal_state="Nordrhein-Westfalen", + aoi_map=self.aoi_map, + ) + self.assertModule( + v_check, + "Using aoi_map, which is located in" + "in Germany and the Netherlands fails", + ) + # Data should have following columns: + # cat, AGS, OI, GFK + atr_dict = grass.parse_command( + "v.info", map=self.test_output, flags="c" + ) + atr = list(atr_dict.keys()) + self.assertTrue( + "AGS" in atr[1], "Module failed, because of missins key 'AGS'" + ) + print("Running tests with AOI only partly in Germany done.") + + def test_file_input_multi(self): + """tests file as input option for federal state(s) information + multiple federal state - case + """ + print( + "Running tests with AOI only partly in Germany and federal " + "state input file..." + ) + self.runModule( + "v.import", + input=self.aoi_map_multi_data, + output=self.aoi_map, + overwrite=True, + ) + # multiple (two) federal states in file + v_check = SimpleModule( + "v.alkis.buildings.import", + output=self.test_output, + aoi_map=self.aoi_map, + file=os.path.join("data", "multiFs.txt"), + ) + self.assertModule( + v_check, + "Module fails, when file-input" + "with multiple federal states given", + ) + # Data should have following columns: + # cat, AGS, OI, GFK + atr_dict = grass.parse_command( + "v.info", map=self.test_output, flags="c" + ) + atr = list(atr_dict.keys()) + self.assertTrue( + "AGS" in atr[1], "Module failed, because of missins key 'AGS'" + ) + print( + "Running tests with AOI only partly in Germany and federal " + "state input file done." + ) + +if __name__ == "__main__": + test() diff --git a/testsuite/v_alkis_buildings_import_base.py b/testsuite/v_alkis_buildings_import_base.py new file mode 100644 index 0000000..c6e4a53 --- /dev/null +++ b/testsuite/v_alkis_buildings_import_base.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: v.alkis.buildings.import test base +# AUTHOR(S): Lina Krisztian, Anika Weinmann +# PURPOSE: Testbase v.alkis.buildings.import +# COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS +# Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# + +import os + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test +from grass.gunittest.gmodules import SimpleModule +import grass.script as grass + +from grass_gis_helpers.location import ( + create_tmp_location, + get_current_location, +) +from grass_gis_helpers.cleanup import cleaning_tmp_location + + +class VAlkisBuildingsImportTestBase(TestCase): + """Test base for the test functionality of v.alkis.buildings.import""" + + pid = os.getpid() + test_output = f"test_output_{pid}" + orig_region = f"orig_region_{pid}" + aoi_map = f"aoi_map_{pid}" + + GISDBASE = None + TGTGISRC = None + TMPLOC = None + SRCGISRC = None + + @classmethod + # pylint: disable=invalid-name + def setUpClass(cls): + # switch location + _, _, cls.GISDBASE, cls.TGTGISRC = get_current_location() + cls.TMPLOC, cls.SRCGISRC = create_tmp_location(epsg=25832) + # save region + grass.run_command("g.region", save=cls.orig_region) + + @classmethod + # pylint: disable=invalid-name + def tearDownClass(cls): + # remove AOI + if grass.find_file(name=cls.aoi_map, element="vector")["file"]: + cls.runModule( + "g.remove", type="vector", name=cls.aoi_map, flags="f" + ) + # cleanup temporary location + cleaning_tmp_location( + cls.TGTGISRC, cls.TMPLOC, cls.SRCGISRC, cls.SRCGISRC + ) + + # pylint: disable=invalid-name + def tearDown(self): + """Remove the outputs created + This is executed after each test run. + """ + if grass.find_file(name=self.test_output, element="vector")["file"]: + self.runModule( + "g.remove", type="vector", name=self.test_output, flags="f" + ) + + +class VAlkisBuildingsImportTestFsBase(VAlkisBuildingsImportTestBase): + """Test base for the test functionality of v.alkis.buildings.import + testing one federal state example""" + + fs = "" + federal_state = "" + + + @classmethod + # pylint: disable=invalid-name + def setUpClass(cls): + # switch location and save region + super().setUpClass() + # import vector map to set as region + cls.runModule( + "v.import", + input=os.path.join("data", f"test_aoi_{cls.fs}.geojson"), + output=cls.aoi_map + ) + # set region + grass.run_command("g.region", vector=cls.aoi_map, flags="a") + grass.run_command("g.region", n="n+200", s="n-100", w="e-100") + + def option_aoi_map(self): + """Tests aoi_map as optional input""" + print(f"Running test for {self.fs} AOI...") + # import pdb; pdb.set_trace() + # get this info before running the addon + region_exp = self.runModule("g.region", flags="g") + # run module + v_check = SimpleModule( + "v.alkis.buildings.import", + output=self.test_output, + federal_state=self.federal_state, + aoi_map=self.aoi_map, + # dldir= + ) + self.assertModule(v_check, "Using aoi_map as optional input fails") + # Data should have following columns: + # cat, AGS, OI, GFK, AKTUALITAE + atr_dict = grass.parse_command( + "v.info", map=self.test_output, flags="c" + ) + atr = list(atr_dict.keys()) + self.assertTrue( + "AGS" in atr[1], "Module failed, because of missins key 'AGS'" + ) + # check if current region is expected region + region_is = self.runModule("g.region", flags="g") + self.assertTrue( + region_exp == region_is, "Region was modified within addon." + ) + print(f"Running test for {self.fs} AOI done.") + + def flag(self): + """Tests -r flag""" + print(f"Running test for {self.fs} region flag...") + # set some region in setUp class (here some region in cologne): + v_check = SimpleModule( + "v.alkis.buildings.import", + output=self.test_output, + federal_state="Nordrhein-Westfalen", + flags="r", + ) + self.assertModule(v_check, "Using -r flag fails") + # Data should have following columns: + # cat, AGS, OI, GFK, AKTUALITAE + atr_dict = grass.parse_command( + "v.info", map=self.test_output, flags="c" + ) + atr = list(atr_dict.keys()) + self.assertTrue( + "AGS" in atr[1], "Module failed, because of missins key 'AGS'" + ) + # check extend of output + out_data_reg = grass.parse_command("v.info", map=self.test_output, flags="g") + g_reg = grass.region() + self.assertTrue( + ( + float(out_data_reg["north"]) >= g_reg["n"] and + float(out_data_reg["south"]) <= g_reg["s"] and + float(out_data_reg["east"]) >= g_reg["e"] and + float(out_data_reg["west"]) <= g_reg["w"] + + ), + "Output data extend is wrong.", + ) + + print(f"Running test for {self.fs} region flag done.") + + def file_input_single(self): + """Tests file as input option for federal state(s) information + single federal state - case + """ + print(f"Running test for {self.fs} AOI and federal state file input...") + # single federal state in file + v_check = SimpleModule( + "v.alkis.buildings.import", + output=self.test_output, + aoi_map=self.aoi_map, + file=os.path.join("data", f"fs_file_{self.fs}.txt"), + ) + self.assertModule( + v_check, + "Module fails, when file-input with single federal state given", + ) + # Data should have following columns: + # cat, AGS, OI, GFK + atr_dict = grass.parse_command( + "v.info", map=self.test_output, flags="c" + ) + atr = list(atr_dict.keys()) + self.assertTrue( + "AGS" in atr[1], "Module failed, because of missins key 'AGS'" + ) + print(f"Running test for {self.fs} AOI and federal state file input done.") \ No newline at end of file diff --git a/v.alkis.buildings.import.py b/v.alkis.buildings.import.py index 0926c0f..737e551 100644 --- a/v.alkis.buildings.import.py +++ b/v.alkis.buildings.import.py @@ -292,7 +292,7 @@ def download_alkis_buildings(fs, url): buildings_filename = BUILDINGS_FILENAMES[fs] alkis_source = os.path.join(dldir, buildings_filename) if not os.path.isfile(alkis_source): - grass.message(_("Downloading ALKIS building data...")) + grass.message(_(f"Downloading ALKIS building data ({fs})...")) if fs == "HE": # insert current date into download URL # try dates of yesterday and tomorrow if it's not working @@ -365,6 +365,12 @@ def import_single_alkis_source( if aoi_map: # set region to aoi_map grass.run_command("g.region", vector=aoi_map, quiet=True) + # if grass.find_file( + # name=OUTPUT_ALKIS_TEMP, element="vector" + # )["file"] != "": + # import pdb; pdb.set_trace() + # OUTPUT_ALKIS_TEMP += "_2" + # rm_vectors.append(OUTPUT_ALKIS_TEMP) grass.run_command( "v.import", input=alkis_source_fixed, @@ -372,6 +378,7 @@ def import_single_alkis_source( snap=snap, extent="region", quiet=True, + overwrite=True, ) grass.run_command( "v.clip", @@ -476,12 +483,7 @@ def import_shapefiles(shape_files, output_alkis, aoi_map=None): out = output_alkis if aoi_map: out = OUTPUT_ALKIS_TEMP - if len(out_tempall) > 1: - grass.run_command( - "v.patch", input=out_tempall, output=out, flags="e", quiet=True - ) - else: - grass.run_command("g.rename", vector=f"{out_tempall[0]},{out}") + patch_vector(out_tempall, out) if aoi_map: grass.run_command( "v.clip", @@ -572,6 +574,44 @@ def import_local_data(aoi_map, local_data_dir, fs, output_alkis_fs): return imported_local_data +def cleanup_columns(out_alkis): + """Remove additional columns""" + cols = grass.vector_columns(out_alkis) + rm_cols = [] + for col in cols: + if col not in ["cat", "AGS", "OI", "GFK"]: + rm_cols.append(col) + for needed_col in ["AGS", "OI", "GFK"]: + tmp_col = None + if needed_col in cols: + tmp_col = f"{needed_col}_tmp" + rm_cols.append(tmp_col) + grass.run_command( + "v.db.renamecolumn", + map=out_alkis, + column=f"{needed_col},{tmp_col}", + quiet=True, + ) + grass.run_command( + "v.db.addcolumn", + map=out_alkis, + columns=f"{needed_col} TEXT", + quiet=True, + ) + grass.run_command( + "v.db.update", + map=out_alkis, + column=needed_col, + query_column=tmp_col, + quiet=True, + ) + if len(rm_cols) > 0: + grass.run_command( + "v.db.dropcolumn", + map=out_alkis, + columns=rm_cols, + quiet=True, + ) def main(): """main function for processing""" @@ -657,7 +697,7 @@ def main(): alkis_source = download_alkis_buildings(fs, url) # import to GRASS DB - grass.message(_("Importing ALKIS buildings data...")) + grass.message(_(f"Importing ALKIS buildings data ({fs})...")) if isinstance(alkis_source, str): import_single_alkis_source( alkis_source, @@ -669,6 +709,10 @@ def main(): else: import_shapefiles(alkis_source, output_alkis_fs, aoi_map) + # cleanup columns of different federal state data + for out_alkis in output_alkis_list: + cleanup_columns(out_alkis) + # patch output from several federal states patch_vector(output_alkis_list, output_alkis) From 45bbedfc0d3234ee060ba4582f62d487395c6412 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 31 Jan 2024 20:55:36 +0100 Subject: [PATCH 07/14] add SN tests --- testsuite/v_alkis_buildings_import_base.py | 28 +++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/testsuite/v_alkis_buildings_import_base.py b/testsuite/v_alkis_buildings_import_base.py index c6e4a53..c3880be 100644 --- a/testsuite/v_alkis_buildings_import_base.py +++ b/testsuite/v_alkis_buildings_import_base.py @@ -41,6 +41,8 @@ class VAlkisBuildingsImportTestBase(TestCase): test_output = f"test_output_{pid}" orig_region = f"orig_region_{pid}" aoi_map = f"aoi_map_{pid}" + east = "" + west = "" GISDBASE = None TGTGISRC = None @@ -86,7 +88,7 @@ class VAlkisBuildingsImportTestFsBase(VAlkisBuildingsImportTestBase): fs = "" federal_state = "" - + alkis_data_dir = "" @classmethod # pylint: disable=invalid-name @@ -101,12 +103,16 @@ def setUpClass(cls): ) # set region grass.run_command("g.region", vector=cls.aoi_map, flags="a") - grass.run_command("g.region", n="n+200", s="n-100", w="e-100") + if cls.east == "" and cls.west == "": + grass.run_command("g.region", n="n+200", s="n-100", w="e-100") + else: + grass.run_command( + "g.region", n="n+200", s="n-100", w=cls.west, e=cls.east + ) def option_aoi_map(self): """Tests aoi_map as optional input""" print(f"Running test for {self.fs} AOI...") - # import pdb; pdb.set_trace() # get this info before running the addon region_exp = self.runModule("g.region", flags="g") # run module @@ -115,6 +121,7 @@ def option_aoi_map(self): output=self.test_output, federal_state=self.federal_state, aoi_map=self.aoi_map, + local_data_dir=self.alkis_data_dir, # dldir= ) self.assertModule(v_check, "Using aoi_map as optional input fails") @@ -141,7 +148,8 @@ def flag(self): v_check = SimpleModule( "v.alkis.buildings.import", output=self.test_output, - federal_state="Nordrhein-Westfalen", + federal_state=self.federal_state, + local_data_dir=self.alkis_data_dir, flags="r", ) self.assertModule(v_check, "Using -r flag fails") @@ -154,15 +162,16 @@ def flag(self): self.assertTrue( "AGS" in atr[1], "Module failed, because of missins key 'AGS'" ) - # check extend of output + # check extend of output (data should overlap with 50 percent of the + # region) out_data_reg = grass.parse_command("v.info", map=self.test_output, flags="g") g_reg = grass.region() self.assertTrue( ( - float(out_data_reg["north"]) >= g_reg["n"] and - float(out_data_reg["south"]) <= g_reg["s"] and - float(out_data_reg["east"]) >= g_reg["e"] and - float(out_data_reg["west"]) <= g_reg["w"] + abs(float(out_data_reg["north"]) - g_reg["n"]) < 25 and + abs(float(out_data_reg["south"]) - g_reg["s"]) < 25 and + abs(float(out_data_reg["east"]) - g_reg["e"]) < 25 and + abs(float(out_data_reg["west"]) - g_reg["w"]) < 25 ), "Output data extend is wrong.", @@ -180,6 +189,7 @@ def file_input_single(self): "v.alkis.buildings.import", output=self.test_output, aoi_map=self.aoi_map, + local_data_dir=self.alkis_data_dir, file=os.path.join("data", f"fs_file_{self.fs}.txt"), ) self.assertModule( From 9a099efc57c2214e157b757a089b1dadb2d1017d Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 31 Jan 2024 20:55:54 +0100 Subject: [PATCH 08/14] add GHA for tests --- .github/workflows/grass-test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/grass-test.yml diff --git a/.github/workflows/grass-test.yml b/.github/workflows/grass-test.yml new file mode 100644 index 0000000..033345f --- /dev/null +++ b/.github/workflows/grass-test.yml @@ -0,0 +1,11 @@ +name: Run tests for GRASS GIS addons +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + +jobs: + tests: + uses: mundialis/github-workflows/.github/workflows/grass-tests.yml@main From b6c3e9a1c487afdbc49f2adb24b9bc15a62eed19 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 31 Jan 2024 20:56:07 +0100 Subject: [PATCH 09/14] add SN tests --- testsuite/test_v_alkis_buildings_import_SN.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 testsuite/test_v_alkis_buildings_import_SN.py diff --git a/testsuite/test_v_alkis_buildings_import_SN.py b/testsuite/test_v_alkis_buildings_import_SN.py new file mode 100644 index 0000000..467152b --- /dev/null +++ b/testsuite/test_v_alkis_buildings_import_SN.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: v.alkis.buildings.import test for NW +# AUTHOR(S): Lina Krisztian, Anika Weinmann +# PURPOSE: Tests v.alkis.buildings.import for NW +# COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS +# Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# + +from grass.gunittest.main import test + +from v_alkis_buildings_import_base import VAlkisBuildingsImportTestFsBase + + +class VAlkisBuildingsImportTestSN(VAlkisBuildingsImportTestFsBase): + fs = "SN" + federal_state = "Sachsen" + east = "w+19200" + west = "w+19100" + + def test_option_aoi_map(self): + """Tests aoi_map as optional input""" + self.option_aoi_map() + + def test_flag(self): + """Tests region as AOI input""" + self.flag() + + def test_file_input_single(self): + """Tests aoi_map as optional input and federal state input file""" + self.file_input_single() + +if __name__ == "__main__": + test() From 0df44ecd27f626bf902cc749ed661c9c1b3100ce Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 31 Jan 2024 20:56:21 +0100 Subject: [PATCH 10/14] add BW tests --- testsuite/test_v_alkis_buildings_import_BW.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 testsuite/test_v_alkis_buildings_import_BW.py diff --git a/testsuite/test_v_alkis_buildings_import_BW.py b/testsuite/test_v_alkis_buildings_import_BW.py new file mode 100644 index 0000000..e59c1bb --- /dev/null +++ b/testsuite/test_v_alkis_buildings_import_BW.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: v.alkis.buildings.import test for NW +# AUTHOR(S): Lina Krisztian, Anika Weinmann +# PURPOSE: Tests v.alkis.buildings.import for NW +# COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS +# Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# + +import os +from grass.gunittest.main import test + +from v_alkis_buildings_import_base import VAlkisBuildingsImportTestFsBase + + +class VAlkisBuildingsImportTestBW(VAlkisBuildingsImportTestFsBase): + fs = "BW" + federal_state = "Baden-Württemberg" + alkis_data_dir = os.path.join("data", "ALKIS") + + def test_option_aoi_map(self): + """Tests aoi_map as optional input""" + self.option_aoi_map() + + def test_flag(self): + """Tests region as AOI input""" + self.flag() + + def test_file_input_single(self): + """Tests aoi_map as optional input and federal state input file""" + self.file_input_single() + +if __name__ == "__main__": + test() From bc3328a51aa6eeecdf74c85d8a5bffe548b12ce2 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 31 Jan 2024 21:16:43 +0100 Subject: [PATCH 11/14] fix tests --- .../data/ALKIS/BW/ALKIS_testGebaeude.gpkg | Bin 0 -> 135168 bytes testsuite/data/fs_file_BW.txt | 1 + testsuite/data/fs_file_SN.txt | 1 + testsuite/data/test_aoi_BW.geojson | 1 + testsuite/data/test_aoi_SN.geojson | 1 + v.alkis.buildings.import.py | 13 +++++++------ 6 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 testsuite/data/ALKIS/BW/ALKIS_testGebaeude.gpkg create mode 100644 testsuite/data/fs_file_BW.txt create mode 100644 testsuite/data/fs_file_SN.txt create mode 100644 testsuite/data/test_aoi_BW.geojson create mode 100644 testsuite/data/test_aoi_SN.geojson diff --git a/testsuite/data/ALKIS/BW/ALKIS_testGebaeude.gpkg b/testsuite/data/ALKIS/BW/ALKIS_testGebaeude.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..7845e964e88f419344b7275b75eab92030b58cb7 GIT binary patch literal 135168 zcmeI53wRsVm4GEXek!&CA-E7A6VcQbmTf)!QewL(w(Ka_l96N*$^NUXHJO0w2l-?aYP`l|J% z0y#WH0!RP}AOR$R1dsp{Kmter2_OL^fCS1%fZU3ov)l~d>nQlLp3?x|Yb;f!#>$3! z8)x7LevkkXKmter2_OL^fCP}hpE`j<8?)b~81aWj^4T`WkS{th2GVl^pJ8Y&7G(s15ha18Q?W2J z%?Jz^XJU~>QDV|Dm$%nzb412>_`{*V(3EW?9HAi1X>)XYdU{=+9*5i8+wSh_al73v zr_14+i0%l51JOw+0GzWqY&}kw*LKY{C33pmUA-h{JRBJFgGVC*!*KI#e8%Q*mJsjt zI$cD3A{dBH*%C|=#6hy%>FVk1^0>Y3E|=Ha>G70M^}5{d(mr(NdvNF@8G63v80*zz>5&oZ%z^c%~VF zO|UfNdky(woK7*ZX*y19Lmj+cZ?{2UnodFS3@u75m$W&$oa9qNg5w2gMizlwg-n`} z1OQX2Exb+zp>KD9#EWL#?~6pCf#k$s&R{SUj_xScZh*`~aK6t}({I{PyD2@vC75ZJ zV-i%7;gbTLnPKBpBpeBbg8n%zFZV+)6z6#%!E!V_>lL)nqeMCMrhcz|YPOj+m^aPl zOBtjEi4g&-R><#j#&bWYywEWUIe6Y*PuI0V4_jul7w;O0P`#bRy#yn%aT34lUeJ1v zl9}s0a7g+_r@n7!B1JvJ)74!nDtnwBXX&Wy_PD!A&Kc|(a^sJxfo*eWjjts0Tv85r zn?ntiqF&yqc%0HCJG4GR<>`wQ%TWm?&Zg-U>`DI>DW0RG8HQ5Xiz zB(sNR$Rw30(FtiTm5#-fjQnv70{$GdB|IdQvL>G_%5n`^V{H*Bn^tu4bdfB3Tt>G9C$;FybR;LbFDiS~fNB{{S0VIF~kN^@u0!RP} zAOR$BW(bhy|9Jm@W>{k>NB{{S0VIF~kN^@u0!RP}AOR$R1Xhax`TKu(|G!%Fv0Nm8 z1dsp{Kmter2_OL^fCP{L5bOwQ{(T}KHK=E#_QIa*WOV#V|mmvTXRF*rmDxP=GXLA{KEVx5c$KOrRX^| z8ziqT#^Eh(Am< z7v`6;(M)ak4~N2jYGQndD2IkAwLrNfDikcX(@cQ|<@XKlpu(YDU|=K=q$2)N|6r5~ z`vMVvi*I`<9Cc95ik}qEQ4+l;#Zb*fJ~cN}`&d2&wGpW}&xw*ivz)Zm=tP|6I9{Um zFjR^WMM|2XIm*>+r*`e|gXb+l|E_j}(|wf7zK!w+hn9xcDOk@rF03pkGJ;fy?ZmJu zHgZ@4!H7Q`Jw=R)%~iy>(^|9fN{G(QD`DKv^XXWM*~g?#Ha3eaC_ZEfZcc;zvvFu(A)LF@j$?W2CGFqdkPjl?)Q z%}`PQ?kE)u!8S2E>d;j+5RCfCh@8taj8-e<&M<6pMxu(c<+Fk^I+xB{NioANK>u8b ziR^xc8VXI23guiLQL)S;`4xQRG zsHM{}9W8R!35)~_E!e5Be;9`K;GjRE;D*zJ#Bh?RLv6PoyrjOSsjJI;NHJO1E-{fQ z8plbR<(REWB=og0wy3_&4p9(eY@)bv(kvHSD1u8;7mHKrg;KJRS}bcsnUvCE*;p1d zlWZHRM^5tj22$!<+&E%}mRLF!6PW3kxF}LBs-S(uQr$Gwb=>_F$QSoj(jIGZWUYju}h2Mo}Xo<}Mi%^?}+GH>I3I3QMTVo?lhd z)X`ziPUQoWPYTdl%6KI^piihS4oH2ru3e5X@JutbG$$}n${d$JISl!SeG{X~naA#{ zu5Oy>I3_NAxoW@DC)FIhsG_E+x!HVZK3_LOy|jbM;yP(r`anBIh2Tzxj*+fplyR^L z81&g`mJyT;C3+nRMvMz=Mq=TnLO$VYRc~%~q-ar!#o_KC3FkMsn~4U-@@F_o6KsK- zjA@BYGcC=Nfux zL1qi-n-0*lgKw9t?B7x^Ro9gDWIg7k^>UelBrtHFmYQWnxTM)bGjjgYG`cLC%Es zrM^vot}EMRUTQiW=mZ{a9m`xZ+2*@WD?GUfTZ_Ca2~Xi_U+TF8@Y%EW+NI{xfUmsZ zRAN^J!qExeC>52b7|I-WHfkag2#!#)=PhhPzLZ&zuUg1A%Pq>k7w!EQS2tbRddzDI zW&v)*ML55T>NR6Np(#6mQ4JYm4sMaL%NGFo?1IDd4Y5*?S#;f4bT8(}87_7Er8|ck zXJo`UUv!nJyW2`KF)7Ya#dk@DTPz1jmO3IB&SRDWuIje zrh3dnIJs1I5pT)aTXdb7h|UWCOL7ksQsUr=jYc`f7!$!P%0JPD(!yRo%`% z5Z!UE!nW!7*q&sd8r(h0MpT1ur+||D`~RBC?WW4@jjuJ>EdN=O5Mm!uYGyza&z6rtT(&9_5&U2uwLd$E9)iPWthLwxT$ND%1usn)JFM+qwv5z ze;T)u$2T@|OJplUEo@}zQIHCa4z=5+*@TVir^q5C<+m?5q}I$kMSV09Vu!h5-F{;d5WPf3)6g+q6&wDK|kvg<-`8{g%9Nv36 zqsLVS?q?)=TxV7Ezzwn-J|wEh&GNBhNV^LN!?W7)QQx4S%$0@8A5K^KMkio>h7!i4 z2ROj5F_zn{4{Qf8l4;o1s#+vw0$Qm01gbK!7+PH)3$Bq z54iGE!mwL}&mYkVbso7n*-25DI@kBB_Pk`sZ zv3+!Ej?w&fkhKBuEGnHT!ykj8+}@(S@(`%Ku%e?pme9&O zfWX21%a^H-oDONP<0$j<^aPV)B&H1GkT%hJnjWVserk`L)Cbh%eTL4b5=Bp(lsAg3 z&hzl)_b%JD)fCzS#4@+OvPhxx>~DV6QLX*8)f8oY(xi*>Mo{q=$PRj5trkiWu1mtKHV;BQC;p)O&Pq@ zz9K&s)BnqAcu!K4GT?_5ImY8Q%B8;9D8vHQ#;i4=# z_@fG!)8(=jF6CZ#amwM+dS=7r169i{>iO{||LIwfqnvg;WeD-pe0iigs)fdCo*z$a znb;~^T50{}PiAG>DxTr7Xe59HkN^@u0!RP}AOR$R1dsp{KmthM3=tsf|JPOfOjfb+ z`Nmxh-)h*l_Va5uS@zZ+t=~}XtNvciUA3*XzpXobhTOqokbq2};aUWN1dza4Patct zRGF5}H;;^)Dql>POf7F*`$igt|c;HgVY;pcy|J6Dr&B*Jsx5hU^{+^?+b;KcGGghc?$zHLx%5*UA&HKN+?!tCZ zxZ~g6{N)_%sjit9tzhVe4$*g!-W&1~mi60jeqk1bD&Kz7|1QYXl>Yha0}bcGam9zP zyT4awL^EFhetzbB$p7S_4|ath|C2qRySWLD@0}dEY9}0T=Xc$=5ssa|fA!8kgJZ4m z4_|p#dK!)|_Rfije zKOvFvL_^~yj<1jc4c=(NKl^IW0h0OB(Tj(_ty9vB{Gctmi&UwH4IUt2q!}yJ zUzfGi8@!>vbKkXJ2Ze+4hZ{&WAG+$s@c)WY-=i--^21-l-r$WU{MLu=dHHq7)RZ*g zSMP8A`~LwWkN@Y`+r-FYANs_@|DZEks9&GGqRtR^-~OfNr8hz0)_bczH4IQc^~3dd zz6^$18wW;S(|beWP80rz1y3Ex)RZ*gt-rd?MT|K9_T`PQK>lsl)p=Us_=e{``y4S^ z_0Z>ExD1ZpdcN{DlK=4|0sbO5{{G)Qe3SlIbHaY~@jrVDoZEiz)rX0GVfl60)>>eD zRc^f8b^q==t_Owp{PVk>dIE2 zzcy>BF+{?DC2#-YmqFos&o(~yUD%)ZySrZ@hPa1+{2c>(GP-NR3%A5>Bbih0da>y# zP|}1oBVD6=-M@tV&rNka_*I?JLj9VorP|=l@WbOj{wq*;{E^WEcf0 zOM6}f1^++2a^?9DA{Tts-umxg=x8AP;b#lI(S)^3O{wr$GcxIW{)5E$hr1r>Bt{GM ztFo3g25$}=c)tFFpfJ1je};Yl`<)*<_19!Al?HF7KJ%p?Gys#h+2(h`{?>;cf8%Fh=)f(H z-~aPMZ#3acAG`XqB=h!-zq*7dX~LS3P5-#J_Ib#^{Lb(Gj2PF97V1}KEfofDZhYSZ zw|x!VKm5}5VG}5Jyz;c282Y&Dg@=p0(S+Z~*4;@mH6=~>SCp-q7@6AA+DnXR@|scY zSkte_)|-hpb=nnE$46^^v>&`#TDSKX55xWwm)`I!VH1D!^T8r-emmFoHWBVU@V?7Q zrlzC`YsXJce(x8=h-O?fTBuL%|0^4BHCf}<3fSQX2_OL^fCP{L5sCY)M7ofYNhomll2~J7IyeS0!RP}AOR$R1dsp{Kmter2_OL^fCP@4 zz-3i1dto+ABCIfZQAt(7?1NF%pjUBCHJLo9J56&3^7?=4gC^^n)?ZtHX?@B1bL(@~ zXRSZ7{?Phe>x0%utY5Ky&H9A(TgP2>6hs0@00|%gB!C2v01`j~NB{{S0VIF~PL{y? zG{s6wrKzrRE%|C7UybC;O1{>SuX^%TN4{#wR}J~9CSO(LYYq9TBwrQe3;uabo&5Lz zENv$EA3sO{2_OL^fCP{L5khD%sHU}*6e+gKa+jbs3@*ddw z)bt8Sdn;g>sf08;Yy)a~C8VVT7WSYj|6VnHO~8`4JYbpr(9WZud=|F%j*2niL@IcG$k8rmutaEs);3^XT0SY`3WCR!HjtKlei#2b<0Xen zfX%L^;|>WRfm0wr?*DV6*(umTRV07}kN^@u0!RP}AOR$R1dsp{Kmter2_OL^fCP{L z5DIvidPdgUUslLrap~2|nxSt|9)cC~q(ZC?pypmco z)f}0OMEzsU1ND8f#y&<6S)S{+dD=bhc6Zw-o2G38^%T{|CKyg)r&&fA=`Hg3u#*Fx9ysh#)vE3K(kN86)IvJb8AB_0NwvYNjYfj)Z49!tdMi3Zi2q>5e zYwzf#B8#HLq$!sdEHj+Qr1zv4n_~=={oy~TJS;$)$-1^{PE^P1JP94wN$9vwLdSCw zI-Mt>(|r;;JtwN;b~;a(HRyDmcsVY@z&|<~7>|SkLsPbqaD+sM)8^>*^z^zsJr1|G zx82><<954UPM2fU9~_D9mfVc#! z0>g0gY<$M%a5{X!kx{>5A;Bae2NK(zuAa^=kK60+a(TU-9uFuJcl_g#5u2ma-PH>T zgQ4J6{%|A^3QpOK9cs)U-4O!us6cZf8Cn@?j1l6r#0xf^9jI3i@yZwWfd=Uc8BQ{U}R`ff`QTI=z@@RI!m{|(N1?MI~*@a zGqR!MvH{Hm?H!pKqy-qXS(FNZ zu(X20M8r4ZpNb5I#>prQ4aYHz08UfloUo5sbWjqX;gbTLnPKBpn$Bb(uO0f2FYNbe zBQLdwk>(-VsqSw)=%;8dLAl)UYb)iOlSlGx)OIG7WarY`s3C?+)50v(M)90D!wZZo z&QDLv2ivF-2F@?s;)?k}UYMsBDVB?;=E&J=MZrjhW;v3pnkGJP-yYgM<>_tj>2`EG zdOO=a9*2uWR=(Fkr}IEcUk5p*$){R44`u}Z+RlMivg@Cd(o`!stMz-_R4bE8WyF3M zKYE>1tHjSI#oo?-C-nE+e^e`pJh30NI6jtuQ)yp^Vo$R!!bMSH00x?_z7FMgK8Iuw zk4O&u)-vchh^xc^87Q`@`D&6TkzmBQz-Hu&j*=(jz7D;Jrbawt;zWy%3p}@&9)S8P z4jYp-1pvTiM4nA(^0H4oPL~(1Dhl&7C3x?=8c@?IIyo>+I$d8!L8>ea_10g5$k#0$ z>IF)l{33qsvP_kO2whn0w>fR($A0Ap%jJG7Xj#gS1e=D738<~uZ*zkR%PG20Lccr= zD|Bu0wG;#bT!LhH7%p=Sk%6wl=h}r_GD*;R;fhMxs}+WHhD*rTSvgsNs?&wbP@QQ& zw;;haFI+dv7vTfs#rmH Date: Wed, 31 Jan 2024 21:19:59 +0100 Subject: [PATCH 12/14] fix tests --- testsuite/test_v_alkis_buildings_import_BW.py | 1 + testsuite/test_v_alkis_buildings_import_NW.py | 1 + testsuite/test_v_alkis_buildings_import_SN.py | 1 + .../test_v_alkis_buildings_import_multi_fs.py | 2 +- testsuite/v_alkis_buildings_import_base.py | 24 +++++++++++-------- v.alkis.buildings.import.py | 2 ++ 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/testsuite/test_v_alkis_buildings_import_BW.py b/testsuite/test_v_alkis_buildings_import_BW.py index e59c1bb..94a506d 100644 --- a/testsuite/test_v_alkis_buildings_import_BW.py +++ b/testsuite/test_v_alkis_buildings_import_BW.py @@ -43,5 +43,6 @@ def test_file_input_single(self): """Tests aoi_map as optional input and federal state input file""" self.file_input_single() + if __name__ == "__main__": test() diff --git a/testsuite/test_v_alkis_buildings_import_NW.py b/testsuite/test_v_alkis_buildings_import_NW.py index 512c0c2..d70f8c9 100644 --- a/testsuite/test_v_alkis_buildings_import_NW.py +++ b/testsuite/test_v_alkis_buildings_import_NW.py @@ -41,5 +41,6 @@ def test_file_input_single(self): """Tests aoi_map as optional input and federal state input file""" self.file_input_single() + if __name__ == "__main__": test() diff --git a/testsuite/test_v_alkis_buildings_import_SN.py b/testsuite/test_v_alkis_buildings_import_SN.py index 467152b..e180501 100644 --- a/testsuite/test_v_alkis_buildings_import_SN.py +++ b/testsuite/test_v_alkis_buildings_import_SN.py @@ -43,5 +43,6 @@ def test_file_input_single(self): """Tests aoi_map as optional input and federal state input file""" self.file_input_single() + if __name__ == "__main__": test() diff --git a/testsuite/test_v_alkis_buildings_import_multi_fs.py b/testsuite/test_v_alkis_buildings_import_multi_fs.py index a7178ca..7f0b528 100644 --- a/testsuite/test_v_alkis_buildings_import_multi_fs.py +++ b/testsuite/test_v_alkis_buildings_import_multi_fs.py @@ -22,7 +22,6 @@ import os -from grass.gunittest.case import TestCase from grass.gunittest.main import test from grass.gunittest.gmodules import SimpleModule import grass.script as grass @@ -150,5 +149,6 @@ def test_file_input_multi(self): "state input file done." ) + if __name__ == "__main__": test() diff --git a/testsuite/v_alkis_buildings_import_base.py b/testsuite/v_alkis_buildings_import_base.py index c3880be..d10c6ef 100644 --- a/testsuite/v_alkis_buildings_import_base.py +++ b/testsuite/v_alkis_buildings_import_base.py @@ -23,7 +23,6 @@ import os from grass.gunittest.case import TestCase -from grass.gunittest.main import test from grass.gunittest.gmodules import SimpleModule import grass.script as grass @@ -99,7 +98,7 @@ def setUpClass(cls): cls.runModule( "v.import", input=os.path.join("data", f"test_aoi_{cls.fs}.geojson"), - output=cls.aoi_map + output=cls.aoi_map, ) # set region grass.run_command("g.region", vector=cls.aoi_map, flags="a") @@ -164,15 +163,16 @@ def flag(self): ) # check extend of output (data should overlap with 50 percent of the # region) - out_data_reg = grass.parse_command("v.info", map=self.test_output, flags="g") + out_data_reg = grass.parse_command( + "v.info", map=self.test_output, flags="g" + ) g_reg = grass.region() self.assertTrue( ( - abs(float(out_data_reg["north"]) - g_reg["n"]) < 25 and - abs(float(out_data_reg["south"]) - g_reg["s"]) < 25 and - abs(float(out_data_reg["east"]) - g_reg["e"]) < 25 and - abs(float(out_data_reg["west"]) - g_reg["w"]) < 25 - + abs(float(out_data_reg["north"]) - g_reg["n"]) < 25 + and abs(float(out_data_reg["south"]) - g_reg["s"]) < 25 + and abs(float(out_data_reg["east"]) - g_reg["e"]) < 25 + and abs(float(out_data_reg["west"]) - g_reg["w"]) < 25 ), "Output data extend is wrong.", ) @@ -183,7 +183,9 @@ def file_input_single(self): """Tests file as input option for federal state(s) information single federal state - case """ - print(f"Running test for {self.fs} AOI and federal state file input...") + print( + f"Running test for {self.fs} AOI and federal state file input..." + ) # single federal state in file v_check = SimpleModule( "v.alkis.buildings.import", @@ -205,4 +207,6 @@ def file_input_single(self): self.assertTrue( "AGS" in atr[1], "Module failed, because of missins key 'AGS'" ) - print(f"Running test for {self.fs} AOI and federal state file input done.") \ No newline at end of file + print( + f"Running test for {self.fs} AOI and federal state file input done." + ) diff --git a/v.alkis.buildings.import.py b/v.alkis.buildings.import.py index c63fa31..74e518a 100644 --- a/v.alkis.buildings.import.py +++ b/v.alkis.buildings.import.py @@ -574,6 +574,7 @@ def import_local_data(aoi_map, local_data_dir, fs, output_alkis_fs): return imported_local_data + def cleanup_columns(out_alkis): """Remove additional columns""" cols = grass.vector_columns(out_alkis) @@ -614,6 +615,7 @@ def cleanup_columns(out_alkis): quiet=True, ) + def main(): """main function for processing""" global orig_region, OUTPUT_ALKIS_TEMP, PID, dldir From c8380f0038773472693465144eae94d5561a79ee Mon Sep 17 00:00:00 2001 From: Anika Weinmann <37300249+anikaweinmann@users.noreply.github.com> Date: Thu, 1 Feb 2024 08:47:58 +0100 Subject: [PATCH 13/14] Apply suggestions from code review Co-authored-by: Julia Haas <83269984+juleshaas@users.noreply.github.com> --- requirements.txt | 2 +- testsuite/test_v_alkis_buildings_import_BW.py | 4 ++-- testsuite/test_v_alkis_buildings_import_SN.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 716a583..1debeec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ py7zr -grass_gis_helpers +grass-gis-helpers requests \ No newline at end of file diff --git a/testsuite/test_v_alkis_buildings_import_BW.py b/testsuite/test_v_alkis_buildings_import_BW.py index 94a506d..2bd6a92 100644 --- a/testsuite/test_v_alkis_buildings_import_BW.py +++ b/testsuite/test_v_alkis_buildings_import_BW.py @@ -2,9 +2,9 @@ # ############################################################################ # -# MODULE: v.alkis.buildings.import test for NW +# MODULE: v.alkis.buildings.import test for BW # AUTHOR(S): Lina Krisztian, Anika Weinmann -# PURPOSE: Tests v.alkis.buildings.import for NW +# PURPOSE: Tests v.alkis.buildings.import for BW # COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS # Development Team # diff --git a/testsuite/test_v_alkis_buildings_import_SN.py b/testsuite/test_v_alkis_buildings_import_SN.py index e180501..fe19151 100644 --- a/testsuite/test_v_alkis_buildings_import_SN.py +++ b/testsuite/test_v_alkis_buildings_import_SN.py @@ -2,9 +2,9 @@ # ############################################################################ # -# MODULE: v.alkis.buildings.import test for NW +# MODULE: v.alkis.buildings.import test for SN # AUTHOR(S): Lina Krisztian, Anika Weinmann -# PURPOSE: Tests v.alkis.buildings.import for NW +# PURPOSE: Tests v.alkis.buildings.import for SN # COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS # Development Team # From 292335aaf645538f87eb8507b75d10cc24fbeed9 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 1 Feb 2024 09:00:11 +0100 Subject: [PATCH 14/14] add readme for test data --- testsuite/data/readme.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/testsuite/data/readme.md b/testsuite/data/readme.md index 736700f..7da01bb 100644 --- a/testsuite/data/readme.md +++ b/testsuite/data/readme.md @@ -4,9 +4,12 @@ Test area: located in Bonn-Beuel settlement area (SIE01) -## area_cologne.geojson +## test_aoi_NW.geojson, test_aoi_BW.geojson, test_aoi_SN.geojson -Test area: located in a part of Cologne (created at geojson.io) +Test area for: +* NW - located in a part of Cologne (created at geojson.io) +* BW +* SN ## area_nw_rp.geojson @@ -16,10 +19,14 @@ Test area: located partly in Nordrhein-Westfalen and partly in Rheinland-Pfalz ( Test area: located partly in Germany and partly in the Netherlands -## singleFs +## fs_file_NW.txt, fs_file_BW.txt, fs_file_SN.txt Test text-file containing single federal state name -## multiFs +## multiFs.txt Test text-file containing multiple federal state names + +## ALKIS/BW/ALKIS_testGebaeude.gpkg + +Polygons generated in BW to simulate local ALKIS buildings data \ No newline at end of file