diff --git a/feedstock/iids_pr.yaml b/feedstock/iids_pr.yaml index f6a0fa6..f41446b 100644 --- a/feedstock/iids_pr.yaml +++ b/feedstock/iids_pr.yaml @@ -1 +1,2 @@ - - "CMIP6.*.*.[CNRM-CM6-1,CanESM5].historical.r1i1p1f1.Omon.[tos, so].*.*" + # - "CMIP6.*.*.[CNRM-CM6-1,CanESM5].historical.r1i1p1f1.Omon.[tos, so].*.*" + - 'CMIP6.RFMIP.*.*.rad-irf.*.Efx.*.*.*' diff --git a/feedstock/recipe.py b/feedstock/recipe.py index c225243..c426832 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -158,6 +158,15 @@ async def get_recipe_inputs(): for iid, data in recipe_data.items(): urls = get_sorted_http_urls_from_iid_dict(data) pattern = pattern_from_file_sequence(urls, concat_dim="time") + + # to accomodate single file we cannot parse target chunks (https://github.com/pangeo-forge/pangeo-forge-recipes/issues/275) + if len(urls) > 1: + chunk_fn = dynamic_chunking_func + combine_dims = pattern.combine_dim_keys + else: + chunk_fn = None + combine_dims = [] + recipes[iid] = ( f"Creating {iid}" >> beam.Create(pattern.items()) # | CheckpointFileTransfer( @@ -176,8 +185,8 @@ async def get_recipe_inputs(): | Preprocessor() | StoreToZarr( store_name=f"{iid}.zarr", - combine_dims=pattern.combine_dim_keys, - dynamic_chunking_fn=dynamic_chunking_func, + combine_dims=combine_dims, + dynamic_chunking_fn=chunk_fn, ) | InjectAttrs({"pangeo_forge_api_responses": data}) | ConsolidateDimensionCoordinates() diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index cef01a8..54a2e2b 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,13 +1,8 @@ -leap-data-management-utils==0.0.12 +git+https://github.com/leap-stc/leap-data-management-utils.git@allow-no-time-in-tests +#leap-data-management-utils==0.0.12 git+https://github.com/jbusecke/pangeo-forge-esgf.git@new-request-scheme dynamic-chunks==0.0.3 -#git+https://github.com/pangeo-forge/pangeo-forge-recipes@feature/concurrency-control -git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1#egg=xarray -git+https://github.com/ranchodeluxe/rioxarray -git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree -git+https://github.com/pangeo-forge/pangeo-forge-recipes@jb/xarray-hack #see @gc/cached_disabled but with cache -#git+https://github.com/moradology/httpfs-sync.git@feature/pool-reuse -#httpfs-sync>=0.0.2 +pangeo-forge-recipes==0.10.8 zarr==2.16.1 gcsfs apache-beam[gcp]