Use asyncio also for invoking contour generation, use pyhgtmap for el…

…evation generation

Temporarily remove srg.. support, since it involves fetching files using login cookie
and that seems to run into trouble. Got trouble even with semaphore(1), so do not think
it is related to running many instances in parallel, need to check how the pyhgtmap
is handling the download.
The pyhgtmap should possibly be changed to rather use M2M API with token, instead of
simulating logging in as a web client.

wahoomc/main.py

@@ -78,17 +78,15 @@ def run(run_level):
         # Filter tags from country osm.pbf files'
         asyncio.run(o_osm_maps.filter_tags_from_country_osm_pbf_files())
 
-
         # Generate land
         asyncio.run(o_osm_maps.generate_land())
 
-
         # Generate sea
         o_osm_maps.generate_sea()
 
         # Generate elevation
         if o_input_data.contour:
-            o_osm_maps.generate_elevation(o_input_data.use_srtm1)
+            asyncio.run(o_osm_maps.generate_elevation(o_input_data.use_srtm1))
 
         # Split filtered country files to tiles
         asyncio.run(o_osm_maps.split_filtered_country_files_to_tiles())

wahoomc/osm_maps_functions.py

@@ -319,7 +319,7 @@ def generate_sea(self):
 
         log.info('+ Generate sea for each coordinate: OK, %s', timings.stop_and_return())
 
-    def generate_elevation(self, use_srtm1):
+    async def generate_elevation(self, use_srtm1):
         """
         Generate contour lines for all tiles
         """
@@ -330,6 +330,7 @@ def generate_elevation(self, use_srtm1):
 
         hgt_path = os.path.join(USER_DL_DIR, 'hgt')
 
+        semaphore = asyncio.Semaphore(28)
         tasks = set()
         timings = Timings()
         tile_count = 1
@@ -350,35 +351,36 @@ def generate_elevation(self, use_srtm1):
                 # 2) set source
                 elevation_source = '--source=srtm1,view1,view3,srtm3'
             else:
-                # 1) search vor view1 elevation files
+                # 1) search for view1 elevation files
                 out_file_elevation_existing = glob.glob(os.path.join(
                     USER_OUTPUT_DIR, str(tile["x"]), str(tile["y"]), 'elevation*view1*.osm'))
                 # 2) set source
-                elevation_source = '--source=view1,view3,srtm3'
+                elevation_source = '--source=view1,view3'
 
             # check for already existing elevation .osm file (the ones matched via glob)
             if not (len(out_file_elevation_existing) == 1 and os.path.isfile(out_file_elevation_existing[0])) \
                     or self.o_osm_data.force_processing is True:
-                self.log_tile_info(tile["x"], tile["y"], tile_count)
-                timings_tile = Timings()
-                cmd = ['phyghtmap']
-                cmd.append('-a ' + f'{tile["left"]}' + ':' + f'{tile["bottom"]}' +
-                           ':' + f'{tile["right"]}' + ':' + f'{tile["top"]}')
-                cmd.extend(['-o', f'{out_file_elevation}', '-s 10', '-c 100,50', elevation_source,
-                            '--jobs=8', '--viewfinder-mask=1', '--start-node-id=20000000000',
-                            '--max-nodes-per-tile=0', '--start-way-id=2000000000', '--write-timestamp',
-                            '--no-zero-contour', '--hgtdir=' + hgt_path])
-                cmd.append('--earthexplorer-user=' + username)
-                cmd.append('--earthexplorer-password=' + password)
-
-                run_subprocess_and_log_output(
-                    cmd, f'! Error in phyghtmap with tile: {tile["x"]},{tile["y"]}. Win_macOS/elevation')
-                self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return())
+#                self.log_tile_info(tile["x"], tile["y"], tile_count)
+                tasks.add(asyncio.create_task(self.invoke_generate_elevation_for_tile(semaphore, tile, elevation_source, hgt_path, username, password, out_file_elevation)))
 
             tile_count += 1
 
+        await asyncio.gather(*tasks)
+
         log.info('+ Generate contour lines for each coordinate: OK, %s', timings.stop_and_return())
 
+    async def invoke_generate_elevation_for_tile(self, semaphore, tile, elevation_source, hgt_path, username, password, out_file_elevation):
+        cmd = 'pyhgtmap'
+        args = ['-a ' + f'{tile["left"]}' + ':' + f'{tile["bottom"]}' + ':' + f'{tile["right"]}' + ':' + f'{tile["top"]}']
+        args.extend(['-o', f'{out_file_elevation}', '-s 10', '-c 100,50', elevation_source,
+                            '--jobs=1', '--viewfinder-mask=1', '--start-node-id=20000000000',
+                            '--max-nodes-per-tile=0', '--start-way-id=2000000000', '--write-timestamp',
+                            '--no-zero-contour', '--hgtdir=' + hgt_path])
+        args.append('--earthexplorer-user=' + username)
+        args.append('--earthexplorer-password=' + password)
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in phyghtmap with tile: {tile["x"]},{tile["y"]}')
+
     async def split_filtered_country_files_to_tiles(self):
         """
         Split filtered country files to tiles

wahoomc/setup_functions.py

@@ -134,9 +134,9 @@ def check_installation_of_programs_credentials_for_contour_lines():
                     \nPlease refer to the Quickstart Guide of wahooMapsCreator for instructions:\n- https://github.com/treee111/wahooMapsCreator/blob/develop/docs/QUICKSTART_ANACONDA.md#additional-programs-for-generating-contour-lines \
                     \nor create an issue:\n- https://github.com/treee111/wahooMapsCreator/issues"
 
-    if not is_program_installed("phyghtmap"):
+    if not is_program_installed("pyhgtmap"):
         sys.exit(
-            f"phyghtmap is not installed. {text_to_docu}")
+            f"pyhgtmap is not installed. {text_to_docu}")
 
     username, password = read_earthexplorer_credentials()