Merge remote-tracking branch 'origin/main' into 31.use-pathlib

ceblanton · Sep 27, 2024 · 88a7d60 · 88a7d60
2 parents 1f1293b + 3d75b57
commit 88a7d60
Show file tree

Hide file tree

Showing 7 changed files with 271 additions and 134 deletions.
diff --git a/Jinja2Filters/get_analysis_info.py b/Jinja2Filters/get_analysis_info.py
@@ -342,9 +342,9 @@ def form_task_definition_string(freq, chunk, pp_dir, comps, item, script_file, s
 
     # ts and av distinction
     if product == "ts":
-        in_data_dir = f"{pp_dir}/{comps[0]}/ts/{bronx_freq}/{bronx_chunk}"
+        in_data_dir = os.path.join(pp_dir,comps[0],"ts",bronx_freq,bronx_chunk,"")
     else:
-        in_data_dir = f"{pp_dir}/{comps[0]}/av/{bronx_freq}_{bronx_chunk}"
+        in_data_dir = os.path.join(pp_dir,comps[0],"av",bronx_freq,bronx_chunk,"")
 
     string = f"""
     [[analysis-{item}]]

diff --git a/flow.cylc b/flow.cylc
@@ -1194,9 +1194,9 @@ REMAP-PP-COMPONENTS-AV-{{ PP_CHUNK_B }}:succeed-all => clean-shards-{{ PP_CHUNK_
 {% include 'site/' ~ SITE ~ '.cylc' %}
 
 {# this is required for portability should probably only include #}
-{# inherit = <some_task_param> if SITE = ppan. #}
+{# inherit = <some_task_param> if SITE = ppan_test #}
 {# but empty task-family inheritence is inconsequential for now. #}
-{% if SITE != 'ppan' %}
+{% if SITE != 'ppan_test' %}
     {% if DO_REGRID %}
     [[<regrid>]]
         {% if DO_STATICS %}

diff --git a/meta/lib/python/macros/analysis_validator.py b/meta/lib/python/macros/analysis_validator.py
@@ -102,7 +102,7 @@ def validate(self, config, meta_config=None):
                else: 
                   #in file/    
                   analysis_file_suffix = os.path.dirname(os.path.abspath(__file__)) + '/../../../../app/analysis/file/'
-                  ascript = analysis_file_suffix+ascript
+                  ascript = analysis_file_suffix + os.path.basename(ascript)
             if os.access(ascript, os.R_OK):
                   pass
             else:

diff --git a/meta/lib/python/macros/chunkcheck.py b/meta/lib/python/macros/chunkcheck.py
@@ -23,9 +23,9 @@ class ChunkChecker(metomi.rose.macro.MacroBase):
 
     def is_multiple_of(self, big_chunk, small_chunk):
         '''Takes in chunk value e.g P1Y and the chunk reference value from HISTORY_SEGMENT, returns True or False based on the validation to check if the former is a multiple of the latter'''
-        big_chunk_days = big_chunk.get_days_and_seconds()[0]
-        small_chunk_days = small_chunk.get_days_and_seconds()[0]
-        if big_chunk_days % small_chunk_days == 0:
+        big_chunk_months = big_chunk.years*12 + big_chunk.months
+        small_chunk_months = small_chunk.years*12 + small_chunk.months
+        if big_chunk_months % small_chunk_months == 0:
             return True
         else:
             return False
@@ -78,7 +78,7 @@ def validate(self, config, meta_config=None):
         # if chunk_b is set, check chunk_a and chunk_b consistency
         pp_chunk_b = config.get_value(['template variables', 'PP_CHUNK_B'])
         if pp_chunk_b:
-            pp_chunk_b = pp_chunk_b.strip('\"') 
+            pp_chunk_b = pp_chunk_b.strip('\"\'') 
             pp_chunk_b_duration = None
             try:
                 pp_chunk_b_duration = parse.DurationParser().parse(pp_chunk_b)

diff --git a/meta/rose-meta.conf b/meta/rose-meta.conf
@@ -97,7 +97,6 @@ type=raw
 [template variables=PP_COMPONENTS]
 compulsory=true
 help=Each component must be defined in the remap-pp-component/rose-app.conf
-    ="all" is also accepted, in which case all components in remap-pp-component/rose-app.conf will be used
 title=Spaced list of pp components to process
 type=spaced_list
 
@@ -185,7 +184,7 @@ type=raw
 [template variables=SITE]
 compulsory=true
 title=Label for the site-specific workflow settings; in site/SITE.cylc
-values="ppan", "generic", "gfdl-ws", "gaea"
+values='ppan', 'generic', 'gfdl-ws', 'gaea'
 type=raw
 
 [template variables=DO_ATMOS_PLEVEL_MASKING]

diff --git a/site/ppan.cylc b/site/ppan.cylc
@@ -1,84 +1,46 @@
-{% set EPMT_PREAMBLE %}
-            eval $(rose task-env)
-            env
-            export PAPIEX_OLD_OUTPUT=$PAPIEX_OUTPUT
-            export PAPIEX_OLD_OPTIONS=$PAPIEX_OPTIONS
-            unset PAPIEX_OUTPUT PAPIEX_OPTIONS LD_PRELOAD
-{% endset %}
-{% set EPMT_POSTAMBLE %}
-            export PAPIEX_OPTIONS=$PAPIEX_OLD_OPTIONS
-            export LD_PRELOAD=$PAPIEX_LD_PRELOAD
-            export PAPIEX_OUTPUT=$PAPIEX_OLD_OUTPUT
-{% endset %}
 
     [[root]]
         init-script = """
             module load ppan_environment
-            module load epmt
             module list
-            epmt check
         """
         env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:PLACE_HOLDER;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
+            eval $(rose task-env)
+            env
         """
         script = rose task-run --verbose
         {# retries don't make sense yet while testing as they will just delay the failures #}
         {# execution retry delays = PT1M, PT5M, PT10M                                      #}
         # Set default time limit to 4 hours
         execution time limit = PT4H
-        platform = ppan_test
+        platform = ppan
         [[[events]]]
             mail events = startup shutdown aborted timeout stalled inactivity
             submission timeout = P1D
         [[[directives]]]
-            --comment=xtmp,epmt,canopy
+            --comment=xtmp,canopy
         [[[environment]]]
             COPY_TOOL=gcp
-            # Uncomment to collect data for the data lineage tool
-            # EPMT_DATA_LINEAGE=1
 
-    [[PP-STARTER]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:pp-starter;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
     [[STAGE-HISTORY]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:stage-history;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
         pre-script = module load gcp/2.3 hsm/test
 
 {% if DO_REFINEDIAG or DO_PREANALYSIS %}
     [[PRE-ANALYSIS]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:analysis;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
         pre-script = """
             env
             set -x
             module load gcp
             module load fre/canopy
             mkdir -p $work $tempCache $refineDiagDir
-            export PAPIEX_TAGS="op:hsmget;op_instance:1;op_is_canopy:1" ; hsmget -v -t -a $histDir -p {{ PTMP_DIR }}/$histDir -w $work $hsmdate/\* ; unset PAPIEX_TAGS
+            hsmget -v -t -a $histDir -p {{ PTMP_DIR }}/$histDir -w $work $hsmdate/\*
             cd $work/$hsmdate
             ls
         """
 {% endif %}
 
 {% if DO_REFINEDIAG %}
     [[REFINE-DIAG]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:refineDiag;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
         post-script = """
             cd $refineDiagDir
             if ls *nc; then
@@ -99,12 +61,12 @@
                 ls {{ PTMP_DIR }}/{{ HISTORY_DIR_REFINED }}
                 echo "the contents of {{ PTMP_DIR }}/{{ HISTORY_DIR_REFINED }}/$oname.nc is..."
                 ls {{ PTMP_DIR }}/{{ HISTORY_DIR_REFINED }}/$oname.nc
-                export PAPIEX_TAGS="op:hsmget;op_instance:2;op_is_canopy:1" ; hsmget -v -t -a {{ HISTORY_DIR_REFINED }} -p {{ PTMP_DIR }}/{{ HISTORY_DIR_REFINED }} -w $TMPDIR/modify_refineDiag $hsmdate/\* ; unset     PAPIEX_TAGS
+                hsmget -v -t -a {{ HISTORY_DIR_REFINED }} -p {{ PTMP_DIR }}/{{ HISTORY_DIR_REFINED }} -w $TMPDIR/modify_refineDiag $hsmdate/\*
                 mv -f * $TMPDIR/modify_refineDiag
                 mv -f $TMPDIR/modify_refineDiag/* .
                 rm -rf $TMPDIR/modify_refineDiag
             fi
-            export PAPIEX_TAGS="op:hsmput;op_instance:1;op_is_canopy:1" ; hsmput -v -t -s tar -a {{ HISTORY_DIR_REFINED }} -p {{ PTMP_DIR }}/{{ HISTORY_DIR_REFINED }} -w $TMPDIR/history_refineDiag $hsmdate ; unset PAPIEX_TAGS
+            hsmput -v -t -s tar -a {{ HISTORY_DIR_REFINED }} -p {{ PTMP_DIR }}/{{ HISTORY_DIR_REFINED }} -w $TMPDIR/history_refineDiag $hsmdate
         """
 {% endif %}
 
@@ -125,11 +87,6 @@
         """
 
     [[data-catalog]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:data-catalog:exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
         pre-script = module load fre/canopy 
 
         [[[directives]]]
@@ -159,94 +116,21 @@
         pre-script = module load python/3.9
 
     [[CLEAN]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:clean;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
         pre-script = "set -x"
 
 
 {% if DO_MDTF %}
     [[mdtf]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:mdtf;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
         pre-script = mkdir -p $MDTF_TMPDIR
         [[[environment]]]
             MDTF_TMPDIR = $CYLC_WORKFLOW_SHARE_DIR/mdtf
 {% endif %}
 
 {% if DO_STATICS %}
     [[combine-statics]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:combine-statics;exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ HISTORY_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
         pre-script = module load cdo && mkdir -p $outputDir
 {% endif %}
 
-
-{% set PP_COMPONENTS_LIST = PP_COMPONENTS.split(' ') %}
-{% for COMPONENT in PP_COMPONENTS_LIST %}
-    [[<component={{ COMPONENT }}>]]
-        env-script = """
-    {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:{{ COMPONENT }};exp_component_source:PLACE_HOLDER;exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ MOS_PER_HIST_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-    {{ EPMT_POSTAMBLE }}
-        """
-{% endfor %}
-
-{% if DO_REGRID %}
-    {% set REGRID_COMPONENTS  =  "regrid-xy" | form_task_parameter_components('temporal', PP_COMPONENTS) %}
-    {% for KEY in REGRID_COMPONENTS %}
-    [[<regrid={{ KEY }}>]]
-        env-script = """
-        {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:{{ REGRID_COMPONENTS[KEY] }};exp_component_source:{{ KEY }};exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ MOS_PER_HIST_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-        {{ EPMT_POSTAMBLE }}
-        """
-    {% endfor %}
-
-    {% if DO_STATICS %}
-        {% set REGRID_STATIC_COMPONENTS  =  "regrid-xy" | form_task_parameter_components('static', PP_COMPONENTS) %}
-        {% for KEY in REGRID_STATIC_COMPONENTS %}
-    [[<regrid_static={{ KEY }}>]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:{{ REGRID_STATIC_COMPONENTS[KEY] }};exp_component_source:{{ KEY }};exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ MOS_PER_HIST_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
-        {% endfor %}
-    {% endif %}
-{% endif %}
-
-{% if DO_NATIVE %}
-    {% set NATIVE_COMPONENTS  =  "native" | form_task_parameter_components('temporal', PP_COMPONENTS) %}
-    {% for KEY in NATIVE_COMPONENTS %}
-    [[<native={{ KEY }}>]]
-        env-script = """
-        {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:{{ NATIVE_COMPONENTS[KEY] }};exp_component_source:{{ KEY }};exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ MOS_PER_HIST_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-        {{ EPMT_POSTAMBLE }}
-        """
-    {% endfor %}
-    {% if DO_STATICS %}
-        {% set NATIVE_STATIC_COMPONENTS  =  "native" | form_task_parameter_components('static', PP_COMPONENTS) %}
-        {% for KEY in NATIVE_STATIC_COMPONENTS %}
-    [[<native_static={{ KEY }}>]]
-        env-script = """
-            {{ EPMT_PREAMBLE }}
-            epmt annotate EPMT_JOB_TAGS="exp_name:{{ EXPERIMENT }};exp_fre_mod:canopy;exp_platform:{{ PLATFORM }};exp_target:{{ TARGET }};exp_component:{{ NATIVE_STATIC_COMPONENTS[KEY] }};exp_component_source:{{ KEY }};exp_time:$CYLC_TASK_CYCLE_POINT;exp_seg_months:{{ MOS_PER_HIST_SEGMENT }};pp_chunk_a_months:{{ MOS_PER_CHUNK_A }};pp_chunk_b_months:{{ MOS_PER_CHUNK_B }};script_name:$CYLC_TASK_NAME;exp_run_uuid:$CYLC_WORKFLOW_UUID"
-            {{ EPMT_POSTAMBLE }}
-        """
-        {% endfor %}
-    {% endif %}
-{% endif %}
-
 # workaround to run Ray's analysis example on the analysis partition as it uses his /net2
 [[analysis-ray_example]]
     [[[directives]]]