From efd0a8c38f543122558330c8838159b16e3a5ee4 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 21:50:14 +0100 Subject: [PATCH 1/3] Add regression test for CLOUDSC2 TL AD --- .../tests/test_cloudsc2_tl_ad.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 loki/transformations/tests/test_cloudsc2_tl_ad.py diff --git a/loki/transformations/tests/test_cloudsc2_tl_ad.py b/loki/transformations/tests/test_cloudsc2_tl_ad.py new file mode 100644 index 000000000..a2080d573 --- /dev/null +++ b/loki/transformations/tests/test_cloudsc2_tl_ad.py @@ -0,0 +1,139 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +import os +import io +import resource +from subprocess import CalledProcessError +from pathlib import Path +import pandas as pd +import pytest + +from loki.frontend import FP +from loki.logging import warning +from loki.tools import ( + execute, write_env_launch_script, local_loki_setup, local_loki_cleanup +) + +pytestmark = pytest.mark.skipif('CLOUDSC2_DIR' not in os.environ, reason='CLOUDSC2_DIR not set') + + +@pytest.fixture(scope='module', name='here') +def fixture_here(): + return Path(os.environ['CLOUDSC2_DIR']) + + +@pytest.fixture(scope='module', name='local_loki_bundle') +def fixture_local_loki_bundle(here): + """Call setup utilities for injecting ourselves into the CLOUDSC bundle""" + lokidir, target, backup = local_loki_setup(here) + yield lokidir + local_loki_cleanup(target, backup) + + +@pytest.fixture(scope='module', name='bundle_create') +def fixture_bundle_create(here, local_loki_bundle): + """Inject ourselves into the CLOUDSC bundle""" + env = os.environ.copy() + env['CLOUDSC_BUNDLE_LOKI_DIR'] = local_loki_bundle + + # Run ecbundle to fetch dependencies + execute( + ['./cloudsc-bundle', 'create'], cwd=here, silent=False, env=env + ) + + +@pytest.mark.usefixtures('bundle_create') +@pytest.mark.parametrize('frontend', [FP]) +def test_cloudsc2_tl_ad(here, frontend): + build_cmd = [ + './cloudsc-bundle', 'build', '--retry-verbose', '--clean', + '--with-loki', '--loki-frontend=' + str(frontend), '--without-loki-install', + ] + + if 'CLOUDSC2_ARCH' in os.environ: + build_cmd += [f"--arch={os.environ['CLOUDSC2_ARCH']}"] + else: + # Build without OpenACC support as this makes problems + # with older versions of GNU + build_cmd += ['--cmake=ENABLE_ACC=OFF'] + + execute(build_cmd, cwd=here, silent=False) + + # Raise stack limit + resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) + env = os.environ.copy() + env.update({'OMP_STACKSIZE': '2G', 'NVCOMPILER_ACC_CUDA_HEAPSIZE': '2G'}) + + # Run the produced binaries + nl_binaries = [ + ('dwarf-cloudsc2-nl-loki-idem', '2', '16000', '32'), + ('dwarf-cloudsc2-nl-loki-scc', '1', '16000', '32'), + ('dwarf-cloudsc2-nl-loki-scc-hoist', '1', '16000', '32'), + ] + tl_binaries = [ + ('dwarf-cloudsc2-tl-loki-idem',), + ('dwarf-cloudsc2-tl-loki-scc',), + ('dwarf-cloudsc2-tl-loki-scc-hoist',), + ] + ad_binaries = [ + ('dwarf-cloudsc2-ad-loki-idem',), + ('dwarf-cloudsc2-ad-loki-scc',), + ('dwarf-cloudsc2-ad-loki-scc-hoist',), + ] + + failures, warnings = {}, {} + + for binary, *args in nl_binaries: + # Write a script to source env.sh and launch the binary + script = write_env_launch_script(here, binary, args) + + # Run the script and verify error norms + try: + output = execute([str(script)], cwd=here/'build', capture_output=True, silent=False, env=env) + results = pd.read_fwf(io.StringIO(output.stdout.decode()), index_col='Variable') + no_errors = results['AbsMaxErr'].astype('float') == 0 + if not no_errors.all(axis=None): + only_small_errors = results['MaxRelErr-%'].astype('float') < 1e-12 + if not only_small_errors.all(axis=None): + failures[binary] = results + else: + warnings[binary] = results + except CalledProcessError as err: + failures[binary] = err.stderr.decode() + + for binary, *args in tl_binaries: + # Write a script to source env.sh and launch the binary + script = write_env_launch_script(here, binary, args) + + # Run the script and verify error norms + try: + output = execute([str(script)], cwd=here/'build', capture_output=True, silent=False, env=env) + if 'TEST PASSED' not in output.stdout.decode(): + failures[binary] = output.stdout.decode() + except CalledProcessError as err: + failures[binary] = err.stderr.decode() + + for binary, *args in ad_binaries: + # Write a script to source env.sh and launch the binary + script = write_env_launch_script(here, binary, args) + + # Run the script and verify error norms + try: + output = execute([str(script)], cwd=here/'build', capture_output=True, silent=False, env=env) + if 'TEST OK' not in output.stdout.decode(): + failures[binary] = output.stdout.decode() + except CalledProcessError as err: + failures[binary] = err.stderr.decode() + + if warnings: + msg = '\n'.join([f'{binary}:\n{results}' for binary, results in warnings.items()]) + warning(msg) + + if failures: + msg = '\n'.join([f'{binary}:\n{results}' for binary, results in failures.items()]) + pytest.fail(msg) From ddf8b57de8c7cd6ade952473d98eeb89ded0d6ed Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 21:50:50 +0100 Subject: [PATCH 2/3] Add CLOUDSC2 TL AD to Github regression tests --- .github/workflows/regression_tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 7bac12b9c..acc6ed2d8 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -35,6 +35,13 @@ jobs: path: cloudsc ref: develop + - name: Clone CLOUDSC2 TL AD + uses: actions/checkout@v4 + with: + repository: ecmwf-ifs/dwarf-p-cloudsc2-tl-ad + path: cloudsc2_tl_ad + ref: develop + - name: Clone ECWAM uses: actions/checkout@v4 with: @@ -83,6 +90,7 @@ jobs: - name: Run CLOUDSC and ECWAM regression tests env: CLOUDSC_DIR: ${{ github.workspace }}/cloudsc + CLOUDSC2_DIR: ${{ github.workspace }}/cloudsc2_tl_ad ECWAM_DIR: ${{ github.workspace }}/ecwam OMP_STACKSIZE: 4G run: | From 4d87ec654892071e2468505d28e8fd5df598713b Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 11 Jun 2024 11:35:21 +0200 Subject: [PATCH 3/3] CLOUDSC2: Increase problem size for TL test to achieve convergence --- loki/transformations/tests/test_cloudsc2_tl_ad.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/loki/transformations/tests/test_cloudsc2_tl_ad.py b/loki/transformations/tests/test_cloudsc2_tl_ad.py index a2080d573..6cfab9887 100644 --- a/loki/transformations/tests/test_cloudsc2_tl_ad.py +++ b/loki/transformations/tests/test_cloudsc2_tl_ad.py @@ -71,14 +71,14 @@ def test_cloudsc2_tl_ad(here, frontend): # Run the produced binaries nl_binaries = [ - ('dwarf-cloudsc2-nl-loki-idem', '2', '16000', '32'), - ('dwarf-cloudsc2-nl-loki-scc', '1', '16000', '32'), - ('dwarf-cloudsc2-nl-loki-scc-hoist', '1', '16000', '32'), + ('dwarf-cloudsc2-nl-loki-idem', '2', '16384', '32'), + ('dwarf-cloudsc2-nl-loki-scc', '1', '16384', '32'), + ('dwarf-cloudsc2-nl-loki-scc-hoist', '1', '16384', '32'), ] tl_binaries = [ - ('dwarf-cloudsc2-tl-loki-idem',), - ('dwarf-cloudsc2-tl-loki-scc',), - ('dwarf-cloudsc2-tl-loki-scc-hoist',), + ('dwarf-cloudsc2-tl-loki-idem', '1', '1024', '32'), + ('dwarf-cloudsc2-tl-loki-scc', '1', '1024', '32'), + ('dwarf-cloudsc2-tl-loki-scc-hoist', '1', '1024', '32'), ] ad_binaries = [ ('dwarf-cloudsc2-ad-loki-idem',),