Merge pull request #231 from nusbaume/externals_update

Update 'manage_externals' and 'Externals.cfg' to match CAM

.github/scripts/pr_mod_file_tests.py

@@ -197,10 +197,15 @@ def _main_prog():
         #PR itself, so don't check its file type:
         if os.path.exists(file_obj.filename):
 
-            #Check if it is a python file:
-            if _file_is_python(file_obj.filename):
-                #If so, then add to python list:
-                pyfiles.append(file_obj.filename)
+            #Don't analyze 'manage_externals' files,
+            #as they are an external repo and thus
+            #not our responsibility:
+            if 'manage_externals' not in file_obj.filename:
+
+                #Check if it is a python file:
+                if _file_is_python(file_obj.filename):
+                    #If so, then add to python list:
+                    pyfiles.append(file_obj.filename)
 
     #++++++++++++++++++++++++++++++++++++++++++++
     #Check if any python files are being modified:

.github/scripts/pylint_threshold_test.py

@@ -87,7 +87,7 @@ def pylint_check(pyfile_list, rcfile, threshold=10.0):
 
         #Run linter:
         lint_results = lint.Run([rcstr, '--exit-zero', pyfile],
-                                reporter=pylint_report, do_exit=False)
+                                reporter=pylint_report, exit=False)
 
         #Extract linter score:
         lint_score = lint_results.linter.stats.global_note

Externals.cfg

@@ -1,5 +1,5 @@
 [ccs_config]
-tag = ccs_config_cesm0.0.59
+tag = ccs_config_cesm0.0.78
 protocol = git
 repo_url = https://github.com/ESMCI/ccs_config_cesm
 local_path = ccs_config
@@ -13,37 +13,37 @@ local_path = components/cice5
 required = True
 
 [cice6]
-tag = cesm_cice6_4_1_3
+tag = cesm_cice6_4_1_10
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CICE
 local_path = components/cice
 externals = Externals.cfg
 required = True
 
 [cmeps]
-tag = cmeps0.14.34
+tag = cmeps0.14.39
 protocol = git
 repo_url = https://github.com/ESCOMP/CMEPS.git
 local_path = components/cmeps
 required = True
 
 [cdeps]
-tag = cdeps1.0.8
+tag = cdeps1.0.21
 protocol = git
 repo_url = https://github.com/ESCOMP/CDEPS.git
 local_path = components/cdeps
 externals =  Externals_CDEPS.cfg
 required = True
 
 [cpl7]
-tag = cpl7.0.15
+tag = cpl77.0.6
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps
 local_path = components/cpl7
 required = True
 
 [share]
-tag = share1.0.16
+tag = share1.0.17
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_share
 local_path = share
@@ -57,36 +57,37 @@ local_path = libraries/mct
 required = True
 
 [parallelio]
-tag = pio2_5_10
+tag = pio2_6_2
 protocol = git
 repo_url = https://github.com/NCAR/ParallelIO
 local_path = libraries/parallelio
 required = True
 
 [cime]
-tag = cime6.0.94
+tag = cime6.0.156
 protocol = git
 repo_url = https://github.com/ESMCI/cime
 local_path = cime
 required = True
 
 [cism]
-tag = cismwrap_2_1_95
+tag = cismwrap_2_1_96
 protocol = git
 repo_url = https://github.com/ESCOMP/CISM-wrapper
 local_path = components/cism
 externals = Externals_CISM.cfg
 required = True
 
 [clm]
-tag = ctsm5.1.dev120
+tag = ctsm5.1.dev139
 protocol = git
 repo_url = https://github.com/ESCOMP/CTSM
 local_path = components/clm
 externals = Externals_CLM.cfg
 required = True
 
 [fms]
+# Older tag than CESM as there is a compilation error mismatch
 tag = fi_20211011
 protocol = git
 repo_url = https://github.com/ESCOMP/FMS_interface

manage_externals/.github/workflows/bumpversion.yml

@@ -0,0 +1,19 @@
+name: Bump version
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Bump version and push tag
+        id: tag_version
+        uses: mathieudutour/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          create_annotated_tag: true
+          default_bump: patch
+          dry_run: false
+          tag_prefix: manic-

manage_externals/.github/workflows/tests.yml

@@ -0,0 +1,30 @@
+# This is a workflow to compile the cmeps source without cime
+name: Test Manic
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  test-manic:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Test Manic
+        run: |
+          pushd test
+          git config --global user.email "[email protected]"
+          git config --global user.name "GITHUB tester"
+          git config --global protocol.file.allow always
+          make utest
+          make stest
+          popd
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3

manage_externals/README.md

@@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     externals description file pointed 'useful_library/sub-xternals.cfg',
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
+    Note that by default, `checkout_externals` will clone an external's
+    submodules. As a special case, the entry, `externals = None`, will
+    prevent this behavior. For more control over which externals are
+    checked out, create an externals file (and see the `from_submodule`
+    configuration entry below).
 
   * from_submodule (True / False) : used to pull the repo_url, local_path,
     and hash properties for this external from the .gitmodules file in

manage_externals/checkout_externals

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Main driver wrapper around the manic/checkout utility.
 

manage_externals/manic/checkout.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
 Tool to assemble repositories represented in a model-description file.
@@ -227,6 +227,12 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+    Note that by default, checkout_externals will clone an external's
+    submodules. As a special case, the entry, "externals = None", will
+    prevent this behavior. For more control over which externals are
+    checked out, create an externals file (and see the from_submodule
+    configuration entry below).
+
   * from_submodule (True / False) : used to pull the repo_url, local_path,
     and hash properties for this external from the .gitmodules file in
     this repository. Note that the section name (the entry in square
@@ -332,7 +338,34 @@ def commandline_arguments(args=None):
         options = parser.parse_args()
     return options
 
-
+def _dirty_local_repo_msg(program_name, config_file):
+    return """The external repositories labeled with 'M' above are not in a clean state.
+The following are four options for how to proceed:
+(1) Go into each external that is not in a clean state and issue either a 'git status' or
+    an 'svn status' command (depending on whether the external is managed by git or
+    svn). Either revert or commit your changes so that all externals are in a clean
+    state. (To revert changes in git, follow the instructions given when you run 'git
+    status'.) (Note, though, that it is okay to have untracked files in your working
+    directory.) Then rerun {program_name}.
+(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
+    update out-of-sync externals (labeled with 's' above) as described in the
+    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
+    commands to checkout the appropriate tags for each external, as given in
+    {config_file}.)
+(3) You can also use {program_name} to manage most, but not all externals: You can specify
+    one or more externals to ignore using the '-x' or '--exclude' argument to
+    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
+    update the other, non-excluded externals.
+(4) As a last resort, if you are confident that there is no work that needs to be saved
+    from a given external, you can remove that external (via "rm -rf [directory]") and
+    then rerun the {program_name} tool. This option is mainly useful as a workaround for
+    issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157).
+The external repositories labeled with '?' above are not under version
+control using the expected protocol. If you are sure you want to switch
+protocols, and you don't have any work you need to save from this
+directory, then run "rm -rf [directory]" before rerunning the
+{program_name} tool.
+""".format(program_name=program_name, config_file=config_file)
 # ---------------------------------------------------------------------
 #
 # main
@@ -345,9 +378,9 @@ def main(args):
     the --all option is passed.
 
     Returns a tuple (overall_status, tree_status). overall_status is 0
-    on success, non-zero on failure. tree_status gives the full status
-    *before* executing the checkout command - i.e., the status that it
-    used to determine if it's safe to proceed with the checkout.
+    on success, non-zero on failure. tree_status is a dict mapping local path
+    to ExternalStatus -- if no checkout is happening. If checkout is happening, tree_status
+    is None.
     """
     if args.do_logging:
         logging.basicConfig(filename=LOG_FILE_NAME,
@@ -363,19 +396,25 @@ def main(args):
         load_all = True
 
     root_dir = os.path.abspath(os.getcwd())
-    external_data = read_externals_description_file(root_dir, args.externals)
-    external = create_externals_description(
-        external_data, components=args.components, exclude=args.exclude)
+    model_data = read_externals_description_file(root_dir, args.externals)
+    ext_description = create_externals_description(
+        model_data, components=args.components, exclude=args.exclude)
 
     for comp in args.components:
-        if comp not in external.keys():
+        if comp not in ext_description.keys():
+            # Note we can't print out the list of found externals because
+            # they were filtered in create_externals_description above.
             fatal_error(
                 "No component {} found in {}".format(
                     comp, args.externals))
 
-    source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry)
-    printlog('Checking status of externals: ', end='')
-    tree_status = source_tree.status()
+    source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry)
+    if args.components:
+        components_str = 'specified components'
+    else:
+        components_str = 'required & optional components'
+    printlog('Checking local status of ' + components_str + ': ', end='')
+    tree_status = source_tree.status(print_progress=True)
     printlog('')
 
     if args.status:
@@ -390,45 +429,17 @@ def main(args):
             for comp in sorted(tree_status):
                 tree_status[comp].log_status_message(args.verbose)
             # exit gracefully
-            msg = """The external repositories labeled with 'M' above are not in a clean state.
-
-The following are three options for how to proceed:
-
-(1) Go into each external that is not in a clean state and issue either a 'git status' or
-    an 'svn status' command (depending on whether the external is managed by git or
-    svn). Either revert or commit your changes so that all externals are in a clean
-    state. (To revert changes in git, follow the instructions given when you run 'git
-    status'.) (Note, though, that it is okay to have untracked files in your working
-    directory.) Then rerun {program_name}.
-
-(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
-    update out-of-sync externals (labeled with 's' above) as described in the
-    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
-    commands to checkout the appropriate tags for each external, as given in
-    {config_file}.)
-
-(3) You can also use {program_name} to manage most, but not all externals: You can specify
-    one or more externals to ignore using the '-x' or '--exclude' argument to
-    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
-    update the other, non-excluded externals.
-
-
-The external repositories labeled with '?' above are not under version
-control using the expected protocol. If you are sure you want to switch
-protocols, and you don't have any work you need to save from this
-directory, then run "rm -rf [directory]" before re-running the
-checkout_externals tool.
-""".format(program_name=program_name, config_file=args.externals)
-
             printlog('-' * 70)
-            printlog(msg)
+            printlog(_dirty_local_repo_msg(program_name, args.externals))
             printlog('-' * 70)
         else:
             if not args.components:
                 source_tree.checkout(args.verbose, load_all)
             for comp in args.components:
                 source_tree.checkout(args.verbose, load_all, load_comp=comp)
             printlog('')
+            # New tree status is unknown, don't return anything.
+            tree_status = None
 
     logging.info('%s completed without exceptions.', program_name)
     # NOTE(bja, 2017-11) tree status is used by the systems tests