diff --git a/configs/templates/machine_template.yaml b/configs/templates/machine_template.yaml new file mode 100644 index 000000000..8f0b91b60 --- /dev/null +++ b/configs/templates/machine_template.yaml @@ -0,0 +1,155 @@ +####################################################################################### +# The dummy yaml machine config file for esm-tools +# +# This is a dummy machine file. It is intend to be used for creating a new config file +# for a new machine setup. It provides a minimal structure to be filled in with +# the oppropriate values of the new HPC system. +# +# Usage: +# - Please replace the placeholder indicated by <...> +# with your PHC specific values +# - Please uncomment sections of this yaml file if appropriate. +# Possible sections are indicating this (see below). +# - See also files for other machines that are already implemented in esm-tools. +# - After editing this file, save it as .yaml in configs/machines. Add also +# an entry in the file configs/machines/all_machines.yaml (in this folder) for the +# new machine. +####################################################################################### + +# GENERIC YAML CONFIGURATION FILES +# +# Set hostname of the machine +name: + +# General information about operating system +# Operating system +#operating_system: "" +# Shell interpreter (e.g. "/usr/bin/bash") +sh_interpreter: "" + +# Information about the job scheduler +# Set batch system (e.g. slurm/pbs) +batch_system: "" + +# Set whether an account needs to be set for batch system (e.g. true or false). +# The actual account that will be used for the experiment will be set in the runscript. +accounting: true +# +#hetjob_flag: "?" # either hetjob or packjob, this depends on your SLURM version. More info can be found: https://slurm.schedmd.com/heterogeneous_jobs.html +# +#jobtype: compute +# +# Information about the partitions of the machine +# +# Specify if hyperthreading should be used. +use_hyperthreading: False +# Set further options depending on hyperthreading variable 'use_hyperthreading' by +# uncommenting and editing the following 'choose_use_hyperthreading' section: +#choose_use_hyperthreading: +# True: +# hyperthreading_flag: "" +# mt_launcher_flag: "" +# False: +# : + +# Set default partition (e.g. compute) +partition: +# +# Set detail information for available partitions on the machine. +# These variables will be used in other parts of this yaml file. +choose_partition: + : + partition_name: "" + partition_cpn: # integer number +# +# Define available partitions of the machine for different jobtypes (e.g. for computing [compute], post-processing [PP], etc.) +# +partitions: + compute: + name: ${computer.partition_name} # this uses the variable set in choose_partition section (see above) + #name: "compute" # or set the partition name as string + cores_per_node: ${computer.partition_cpn} +# +#logical_cpus_per_core: 2 +# +#threads_per_core: 1 +# +# Specify different pool directories for this machine. +# +pool_directories: +# focipool: "" +# pool: "" +# + +# Setup environment +# Load necessary modules (e.g. git, cdo, nco, compiler). Edit and extent the following lines. +module_actions: + - "purge" + - "load " +# - "load " +# - "load " +# +# Export environment variables. Extend the following lines to export needed environment variables. +export_vars: + # Locale Settings + LC_ALL: en_US.UTF-8 + # Compiler + FC: + F77: + MPICC: + MPIFC: + CC: + CXX: + # Other environemnt variables + +## Some other yaml files use a file 'computer.fc', etc to identify the compiler, so we need to add them here. +fc: "$FC" +cc: "$CC" +mpifc: "$MPIFC" +mpicc: "$MPICC" +cxx: "$CXX" +# +# +# Launcher flags (i.e. flags for the srun/mpirun/aprun command) +launcher_flags: "-l" + +# Choose another configuration file, that should be evaluated (e.g. slurm.yaml, pbs.yaml) +further_reading: + - batch_system/slurm.yaml + + +#################################################################################### +# Further functionality: +# +# If there are modules or environment variables that are different for +# different e.g. compilers, you can use the funcionality of +# declaring choose_ blocks in this yaml file. +# (For more details about yaml syntax please also see the documentation +# https://esm-tools.readthedocs.io/en/latest/yaml.html#switches-choose) +# +# The following lines show an example of how such a choose_ block can be +# set up depending on a given compiler value/key given by the variable compiler_mpi. +# + +#compiler_mpi: + +#choose_compiler_mpi: +# : +# : +# # load additional modules specific for compiler +# add_module_actions: +# - "load " +# - "load " +# - "load " +# # add additional variables to exported: change and extend following lines: +# add_export_vars: +# # Compiler +# FC: +# F77: +# MPICC: +# MPIFC: +# CC: +# CXX: +#################################################################################### diff --git a/docs/cookbook.rst b/docs/cookbook.rst index 9f4c6770c..aabc8e4b0 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -41,6 +41,7 @@ documentation issue on `our GitHub repository /esm_tools/configs/machines/all_machines.yaml`` +- ``/esm_tools/configs/machines/.yaml`` + +1. Add an additional entry for the new machine. + + Use your favourite text editor and open the file ``/esm_tools/configs/machines/all_machines.yaml``:: + + $ /esm_tools/configs/machines/all_machines.yaml + + and add a new entry for the new machine (replace placeholders indicated by <...>) + + .. code-block:: yaml + + : + login_nodes: '*' # A regex pattern that matches the hostname of login nodes + compute_nodes: '' # A regex pattern that matches the hostname of compute nodes + + +2. Create a new machine file. + + Use your favourite text editor to create and edit a new machine file ``.yaml`` in the + ``esm_tools/configs/machines/`` folder:: + + $ /esm_tools/configs/machines/.yaml + + A template file (``machine_template.yaml``) is available in ``configs/templates``, so you can alternatively copy + this file into the ``configs/machines`` folder edit the relevant entries:: + + $ cp /esm_tools/configs/templates/machine_template.yaml /esm_tools/configs/machines/.yaml + $ /esm_tools/configs/machines/.yaml + + You can also reproduce the two steps above simply by running the following ``esm_tools`` command:: + + $ esm_tools create-new-config /esm_tools/configs/machines/.yaml -t machine + + This will copy the ``machine_template.yaml`` in the target location and open the file in your default editor. + +See also +~~~~~~~~ + +.. links to relevant parts of the documentation + +- :ref:`esm_variables:ESM-Tools Variables` +- :ref:`yaml:Switches (\`\`choose_\`\`)` +- :ref:`yaml:What Is YAML?` diff --git a/setup.cfg b/setup.cfg index c16bf9247..c5aeda5bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 6.21.7 +current_version = 6.21.8 commit = True tag = True diff --git a/setup.py b/setup.py index 44c511c82..bdea845af 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,6 @@ test_suite="tests", tests_require=test_requirements, url="https://github.com/esm-tools/esm_tools", - version="6.21.7", + version="6.21.8", zip_safe=False, ) diff --git a/src/esm_archiving/__init__.py b/src/esm_archiving/__init__.py index 541e954d9..0bb741013 100644 --- a/src/esm_archiving/__init__.py +++ b/src/esm_archiving/__init__.py @@ -4,7 +4,7 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .esm_archiving import (archive_mistral, check_tar_lists, delete_original_data, determine_datestamp_location, diff --git a/src/esm_calendar/__init__.py b/src/esm_calendar/__init__.py index 61adc9e56..3d6919938 100644 --- a/src/esm_calendar/__init__.py +++ b/src/esm_calendar/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .esm_calendar import * diff --git a/src/esm_cleanup/__init__.py b/src/esm_cleanup/__init__.py index 3343888f6..b163e6042 100644 --- a/src/esm_cleanup/__init__.py +++ b/src/esm_cleanup/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" diff --git a/src/esm_database/__init__.py b/src/esm_database/__init__.py index 3597682c1..3922173d5 100644 --- a/src/esm_database/__init__.py +++ b/src/esm_database/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" diff --git a/src/esm_environment/__init__.py b/src/esm_environment/__init__.py index edc6e7bb2..c14cceff5 100644 --- a/src/esm_environment/__init__.py +++ b/src/esm_environment/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .esm_environment import * diff --git a/src/esm_master/__init__.py b/src/esm_master/__init__.py index a94536ad6..260ce34d8 100644 --- a/src/esm_master/__init__.py +++ b/src/esm_master/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from . import database diff --git a/src/esm_motd/__init__.py b/src/esm_motd/__init__.py index 455dbfe9e..1bcdd60f4 100644 --- a/src/esm_motd/__init__.py +++ b/src/esm_motd/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .esm_motd import * diff --git a/src/esm_parser/__init__.py b/src/esm_parser/__init__.py index ff52c8219..a8f7a65df 100644 --- a/src/esm_parser/__init__.py +++ b/src/esm_parser/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .esm_parser import * diff --git a/src/esm_plugin_manager/__init__.py b/src/esm_plugin_manager/__init__.py index 4174955a3..dc709f003 100644 --- a/src/esm_plugin_manager/__init__.py +++ b/src/esm_plugin_manager/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi, Paul Gierz, Sebastian Wahl""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .esm_plugin_manager import * diff --git a/src/esm_profile/__init__.py b/src/esm_profile/__init__.py index 4c1d2427f..e2b24f30f 100644 --- a/src/esm_profile/__init__.py +++ b/src/esm_profile/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .esm_profile import * diff --git a/src/esm_runscripts/__init__.py b/src/esm_runscripts/__init__.py index 8f6bc1cbe..4c593fad0 100644 --- a/src/esm_runscripts/__init__.py +++ b/src/esm_runscripts/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .batch_system import * from .chunky_parts import * diff --git a/src/esm_tests/__init__.py b/src/esm_tests/__init__.py index 2ef3cc120..1ba8561b4 100644 --- a/src/esm_tests/__init__.py +++ b/src/esm_tests/__init__.py @@ -2,7 +2,7 @@ __author__ = """Miguel Andres-Martinez""" __email__ = "miguel.andres-martinez@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .initialization import * from .read_shipped_data import * diff --git a/src/esm_tools/__init__.py b/src/esm_tools/__init__.py index 8dd04da20..feb18892e 100644 --- a/src/esm_tools/__init__.py +++ b/src/esm_tools/__init__.py @@ -23,7 +23,7 @@ __author__ = """Dirk Barbi, Paul Gierz""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" import functools import inspect diff --git a/src/esm_tools/cli.py b/src/esm_tools/cli.py index d55fcb494..5c52351d8 100644 --- a/src/esm_tools/cli.py +++ b/src/esm_tools/cli.py @@ -33,7 +33,7 @@ def test_state(): @click.option( "-t", "--type", - type=click.Choice(["component", "setup"], case_sensitive=False), + type=click.Choice(["component", "setup", "machine"], case_sensitive=False), help="Creates either a new component (default) or a new setup", default="component", show_default=True, diff --git a/src/esm_utilities/__init__.py b/src/esm_utilities/__init__.py index 349e1b514..d7cb34596 100644 --- a/src/esm_utilities/__init__.py +++ b/src/esm_utilities/__init__.py @@ -2,6 +2,6 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.21.7" +__version__ = "6.21.8" from .utils import *