Merge pull request #951 from esm-tools/947-comments-for-generic-masch…

…ine-file

947 comments for generic maschine file

configs/templates/machine_template.yaml

@@ -0,0 +1,155 @@
+#######################################################################################
+# The dummy yaml machine config file for esm-tools
+#
+# This is a dummy machine file. It is intend to be used for creating a new config file 
+# for a new machine setup. It provides a minimal structure to be filled in with
+# the oppropriate values of the new HPC system. 
+#
+# Usage:  
+#     - Please replace the placeholder indicated by <...> 
+#       with your PHC specific values
+#     - Please uncomment sections of this yaml file if appropriate.
+#       Possible sections are indicating this (see below).
+#     - See also files for other machines that are already implemented in esm-tools.
+#     - After editing this file, save it as <machine>.yaml in configs/machines. Add also
+#       an entry in the file configs/machines/all_machines.yaml (in this folder) for the
+#       new machine.
+#######################################################################################
+
+# GENERIC YAML CONFIGURATION FILES
+#
+# Set hostname of the machine 
+name: <hostname>
+
+# General information about operating system
+# Operating system
+#operating_system: "<your-operating-system>"
+# Shell interpreter (e.g. "/usr/bin/bash")
+sh_interpreter: "<path-to-shell>"
+
+# Information about the job scheduler
+# Set batch system (e.g. slurm/pbs)
+batch_system: "<your-job-scheduler>"
+
+# Set whether an account needs to be set for batch system (e.g. true or false). 
+# The actual account that will be used for the experiment will be set in the runscript.
+accounting: true 
+#
+#hetjob_flag: "?" # either hetjob or packjob, this depends on your SLURM version. More info can be found: https://slurm.schedmd.com/heterogeneous_jobs.html
+#
+#jobtype: compute
+#
+# Information about the partitions of the machine
+#
+# Specify if hyperthreading should be used.
+use_hyperthreading: False
+# Set further options depending on hyperthreading variable 'use_hyperthreading' by 
+# uncommenting and editing the following 'choose_use_hyperthreading' section:
+#choose_use_hyperthreading:
+#    True: 
+#        hyperthreading_flag: ""
+#        mt_launcher_flag: ""
+#    False: 
+#       <set_any_variable_here_related_to_hypertherading>: <value>
+
+# Set default partition (e.g. compute)
+partition: <partition_name>
+#
+# Set detail information for available partitions on the machine.
+# These variables will be used in other parts of this yaml file.
+choose_partition:
+    <partition_name>:
+        partition_name: "<partition_name>"
+        partition_cpn: <number_of_cores_per_node>  # integer number 
+#
+# Define available partitions of the machine for different jobtypes (e.g. for computing [compute], post-processing [PP], etc.)
+#
+partitions:
+    compute:
+        name: ${computer.partition_name}     # this uses the variable set in choose_partition section (see above)
+        #name: "compute"                     # or set the partition name as string
+        cores_per_node: ${computer.partition_cpn}
+#
+#logical_cpus_per_core: 2
+#
+#threads_per_core: 1
+#
+# Specify different pool directories for this machine.
+#
+pool_directories:
+#    focipool: "<set_path_to_pool_dir>"
+#    pool: "<set_path_to_pool_dir"
+#
+pool_dir:  "<set_path_to_pool_dir>"
+#
+
+# Setup environment
+# Load necessary modules (e.g. git, cdo, nco, compiler). Edit and extent the following lines.
+module_actions:
+    - "purge"
+    - "load <module>"
+#    - "load <module>"
+#    - "load <module>"
+#
+# Export environment variables. Extend the following lines to export needed environment variables.
+export_vars:
+    # Locale Settings
+    LC_ALL: en_US.UTF-8
+    # Compiler
+    FC: <mpif90>
+    F77: <mpif90>
+    MPICC: <mpicc>
+    MPIFC: <mpif90>
+    CC: <mpicc>
+    CXX: <mpicxx>
+    # Other environemnt variables
+
+## Some other yaml files use a file 'computer.fc', etc to identify the compiler, so we need to add them here.
+fc: "$FC"
+cc: "$CC"
+mpifc: "$MPIFC"
+mpicc: "$MPICC"
+cxx: "$CXX"
+#
+#
+# Launcher flags (i.e. flags for the srun/mpirun/aprun command)
+launcher_flags: "-l"
+
+# Choose another configuration file, that should be evaluated (e.g. slurm.yaml, pbs.yaml)
+further_reading:
+    - batch_system/slurm.yaml
+
+
+####################################################################################
+# Further functionality: 
+# 
+# If there are modules or environment variables that are different for 
+# different e.g. compilers, you can use the funcionality of 
+# declaring choose_ blocks in this yaml file.
+# (For more details about yaml syntax please also see the documentation
+# https://esm-tools.readthedocs.io/en/latest/yaml.html#switches-choose)
+#
+# The following lines show an example of how such a choose_ block can be 
+# set up depending on a given compiler value/key given by the variable compiler_mpi.
+#
+
+#compiler_mpi: <compiler>
+
+#choose_compiler_mpi:
+#    <compiler>:
+#        <variable>: <value>
+#        # load additional modules specific for compiler
+#            add_module_actions:
+#                - "load <module1>"
+#                - "load <module2>"
+#                - "load <module3>"
+#        # add additional variables to exported: change and extend following lines:
+#            add_export_vars:
+#                # Compiler
+#                FC: <mpif90>
+#                F77: <mpif90>
+#                MPICC: <mpicc>
+#                MPIFC: <mpif90>
+#                CC: <mpicc>
+#                CXX: <mpicxx>
+####################################################################################

docs/cookbook.rst

@@ -41,6 +41,7 @@ documentation issue on `our GitHub repository <https://github.com/esm-tools/esm_
 .. include:: recipes/het_par.rst
 .. include:: recipes/how_to_run.rst
 .. include:: recipes/add_model_setup.rst
+.. include:: recipes/add_machine.rst
 .. include:: recipes/add_forcing_input.rst
 .. include:: recipes/exclude_forcing_input.rst
 .. include:: recipes/use_own_namelist.rst

docs/recipes/add_machine.rst

@@ -0,0 +1,50 @@
+Implement a New HPC Machine
+===========================
+
+To implement a new HPC machine to `ESM-Tools`, two files need to be updated and created, respectively:
+
+- ``<PATH>/esm_tools/configs/machines/all_machines.yaml``
+- ``<PATH>/esm_tools/configs/machines/<new_machine>.yaml``
+
+1. Add an additional entry for the new machine.
+
+   Use your favourite text editor and open the file ``<PATH>/esm_tools/configs/machines/all_machines.yaml``::
+
+    $ <your_text_editor> <PATH>/esm_tools/configs/machines/all_machines.yaml
+
+   and add a new entry for the new machine (replace placeholders indicated by <...>)
+
+   .. code-block:: yaml
+
+     <new_machine>:
+        login_nodes: '<hostname>*' # A regex pattern that matches the hostname of login nodes
+        compute_nodes: '<compute_notes>' # A regex pattern that matches the hostname of compute nodes
+
+
+2. Create a new machine file.
+
+   Use your favourite text editor to create and edit a new machine file ``<new_machine>.yaml`` in the
+   ``esm_tools/configs/machines/`` folder::
+
+    $ <your_text_editor> <PATH>/esm_tools/configs/machines/<new_machine>.yaml
+
+   A template file (``machine_template.yaml``) is available in ``configs/templates``, so you can alternatively copy
+   this file into the ``configs/machines`` folder edit the relevant entries::
+
+    $ cp <PATH>/esm_tools/configs/templates/machine_template.yaml  <PATH>/esm_tools/configs/machines/<new_machine>.yaml
+    $ <your_text_editor> <PATH>/esm_tools/configs/machines/<new_machine>.yaml
+
+   You can also reproduce the two steps above simply by running the following ``esm_tools`` command::
+
+    $ esm_tools create-new-config <PATH>/esm_tools/configs/machines/<new_machine>.yaml -t machine
+
+   This will copy the ``machine_template.yaml`` in the target location and open the file in your default editor.
+
+See also
+~~~~~~~~
+
+.. links to relevant parts of the documentation
+
+- :ref:`esm_variables:ESM-Tools Variables`
+- :ref:`yaml:Switches (\`\`choose_\`\`)`
+- :ref:`yaml:What Is YAML?`

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 6.21.7
+current_version = 6.21.8
 commit = True
 tag = True
 

setup.py

@@ -92,6 +92,6 @@
     test_suite="tests",
     tests_require=test_requirements,
     url="https://github.com/esm-tools/esm_tools",
-    version="6.21.7",
+    version="6.21.8",
     zip_safe=False,
 )

src/esm_archiving/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = """Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .esm_archiving import (archive_mistral, check_tar_lists,
                             delete_original_data, determine_datestamp_location,

src/esm_calendar/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .esm_calendar import *

src/esm_cleanup/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"

src/esm_database/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"

src/esm_environment/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .esm_environment import *

src/esm_master/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 
 from . import database

src/esm_motd/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .esm_motd import *

src/esm_parser/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 
 from .esm_parser import *

src/esm_plugin_manager/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi, Paul Gierz, Sebastian Wahl"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .esm_plugin_manager import *

src/esm_profile/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .esm_profile import *

src/esm_runscripts/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .batch_system import *
 from .chunky_parts import *

src/esm_tests/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Miguel Andres-Martinez"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .initialization import *
 from .read_shipped_data import *

src/esm_tools/__init__.py

@@ -23,7 +23,7 @@
 
 __author__ = """Dirk Barbi, Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 import functools
 import inspect

src/esm_tools/cli.py

@@ -33,7 +33,7 @@ def test_state():
 @click.option(
     "-t",
     "--type",
-    type=click.Choice(["component", "setup"], case_sensitive=False),
+    type=click.Choice(["component", "setup", "machine"], case_sensitive=False),
     help="Creates either a new component (default) or a new setup",
     default="component",
     show_default=True,

src/esm_utilities/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.21.7"
+__version__ = "6.21.8"
 
 from .utils import *