From 773396015c49969be4faa08babf452e1dfa1284e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 10 Jan 2025 09:32:50 -0600 Subject: [PATCH] Switch to converting to NETCDF3_64BIT_DATA with ncks Writing SCRIP from pyremap descriptors using NETCDF3_64BIT_DATA proved to be way too slow. --- .../tests/utility/combine_topo/__init__.py | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/compass/ocean/tests/utility/combine_topo/__init__.py b/compass/ocean/tests/utility/combine_topo/__init__.py index 8f9659d68..bd605d2df 100644 --- a/compass/ocean/tests/utility/combine_topo/__init__.py +++ b/compass/ocean/tests/utility/combine_topo/__init__.py @@ -325,6 +325,7 @@ def _create_bedmachine_scrip_file(self): section = config['combine_topo'] antarctic_name = section.get('antarctic_filename').strip('.nc') in_filename = f'{antarctic_name}_mod.nc' + netcdf4_filename = f'{antarctic_name}.scrip.netcdf4.nc' out_filename = f'{antarctic_name}.scrip.nc' # Define projection @@ -337,8 +338,18 @@ def _create_bedmachine_scrip_file(self): bedmachine_descriptor = ProjectionGridDescriptor.read( projection, in_filename, 'BedMachineAntarctica500m', ) - bedmachine_descriptor.format = 'NETCDF3_64BIT_DATA' - bedmachine_descriptor.to_scrip(out_filename) + bedmachine_descriptor.to_scrip(netcdf4_filename) + + # writing directly in NETCDF3_64BIT_DATA proved prohibitively slow + # so we will use ncks to convert + args = [ + 'ncks', '-O', '-5', + netcdf4_filename, + out_filename, + ] + check_call(args, logger) + + logger.info(' Done.') def _create_target_scrip_file(self): """ @@ -372,22 +383,21 @@ def _create_target_scrip_file(self): ] check_call(args, logger) - # ConvertMeshToSCRIP doesn't support NETCDF3_64BIT_DATA, so use - # ncks - args = [ - 'ncks', '-O', '-5', - netcdf4_filename, - out_filename, - ] - check_call(args, logger) - # Build lat-lon SCRIP file using pyremap elif self.target_grid == 'lat_lon': descriptor = get_lat_lon_descriptor( dLon=self.resolution, dLat=self.resolution, ) - descriptor.format = 'NETCDF3_64BIT_DATA' - descriptor.to_scrip(out_filename) + descriptor.to_scrip(netcdf4_filename) + + # writing out directly to NETCDF3_64BIT_DATA is either very slow or + # unsupported, so use ncks + args = [ + 'ncks', '-O', '-5', + netcdf4_filename, + out_filename, + ] + check_call(args, logger) logger.info(' Done.')