Skip to content

Commit

Permalink
Merge pull request #2355 from vkarak/feat/human-readable-test-names
Browse files Browse the repository at this point in the history 
[feat] Introduce new naming scheme for tests
  • Loading branch information
Vasileios Karakasis authored Jan 24, 2022
2 parents bc280c5 + 5a6e35b commit 11493d2
Show file tree
Hide file tree
Showing 87 changed files with 3,520 additions and 2,118 deletions.
6 changes: 3 additions & 3 deletions cscs-checks/system/io/ior_check.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import getpass
import os
import re

import reframe as rfm
import reframe.utility.sanity as sn
Expand Down Expand Up @@ -173,5 +172,6 @@ def set_perf_patterns(self):
}

@run_after('init')
def set_dependency(self):
self.depends_on(re.sub(r'IorReadCheck', 'IorWriteCheck', self.name))
def set_deps(self):
variant = IorWriteCheck.get_variant_nums(base_dir=self.base_dir)[0]
self.depends_on(IorWriteCheck.variant_name(variant))
1 change: 0 additions & 1 deletion cscs-checks/tools/profiling_and_debugging/notool.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ class JacobiNoToolHybrid(rfm.RegressionTest):
@run_after('init')
def set_descr_name(self):
self.descr = f'Jacobi (without tool) {self.lang} check'
self.name = f'{type(self).__name__}_{self.lang.replace("+", "p")}'

@run_after('init')
def remove_buggy_prgenv(self):
Expand Down
130 changes: 130 additions & 0 deletions docs/listings/alltests_daint.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
[ReFrame Setup]
version: 3.10.0-dev.3+605af31a
command: './bin/reframe -c tutorials/basics/ -R -n HelloMultiLangTest|HelloThreadedExtended2Test|StreamWithRefTest --performance-report -r'
launched by: user@host
working directory: '/home/user/Devel/reframe'
settings file: '/home/user/Devel/reframe/tutorials/config/settings.py'
check search path: (R) '/home/user/Devel/reframe/tutorials/basics'
stage directory: '/home/user/Devel/reframe/stage'
output directory: '/home/user/Devel/reframe/output'

[==========] Running 4 check(s)
[==========] Started on Sat Jan 22 22:43:38 2022

[----------] start processing checks
[ RUN ] HelloMultiLangTest %lang=cpp @daint:login+builtin
[ RUN ] HelloMultiLangTest %lang=cpp @daint:login+gnu
[ RUN ] HelloMultiLangTest %lang=cpp @daint:login+intel
[ RUN ] HelloMultiLangTest %lang=cpp @daint:login+pgi
[ RUN ] HelloMultiLangTest %lang=cpp @daint:login+cray
[ RUN ] HelloMultiLangTest %lang=cpp @daint:gpu+gnu
[ RUN ] HelloMultiLangTest %lang=cpp @daint:gpu+intel
[ RUN ] HelloMultiLangTest %lang=cpp @daint:gpu+pgi
[ RUN ] HelloMultiLangTest %lang=cpp @daint:gpu+cray
[ RUN ] HelloMultiLangTest %lang=cpp @daint:mc+gnu
[ RUN ] HelloMultiLangTest %lang=cpp @daint:mc+intel
[ RUN ] HelloMultiLangTest %lang=cpp @daint:mc+pgi
[ RUN ] HelloMultiLangTest %lang=cpp @daint:mc+cray
[ RUN ] HelloMultiLangTest %lang=c @daint:login+builtin
[ RUN ] HelloMultiLangTest %lang=c @daint:login+gnu
[ RUN ] HelloMultiLangTest %lang=c @daint:login+intel
[ RUN ] HelloMultiLangTest %lang=c @daint:login+pgi
[ RUN ] HelloMultiLangTest %lang=c @daint:login+cray
[ RUN ] HelloMultiLangTest %lang=c @daint:gpu+gnu
[ RUN ] HelloMultiLangTest %lang=c @daint:gpu+intel
[ RUN ] HelloMultiLangTest %lang=c @daint:gpu+pgi
[ RUN ] HelloMultiLangTest %lang=c @daint:gpu+cray
[ RUN ] HelloMultiLangTest %lang=c @daint:mc+gnu
[ RUN ] HelloMultiLangTest %lang=c @daint:mc+intel
[ RUN ] HelloMultiLangTest %lang=c @daint:mc+pgi
[ RUN ] HelloMultiLangTest %lang=c @daint:mc+cray
[ RUN ] HelloThreadedExtended2Test @daint:login+builtin
[ RUN ] HelloThreadedExtended2Test @daint:login+gnu
[ RUN ] HelloThreadedExtended2Test @daint:login+intel
[ RUN ] HelloThreadedExtended2Test @daint:login+pgi
[ RUN ] HelloThreadedExtended2Test @daint:login+cray
[ RUN ] HelloThreadedExtended2Test @daint:gpu+gnu
[ RUN ] HelloThreadedExtended2Test @daint:gpu+intel
[ RUN ] HelloThreadedExtended2Test @daint:gpu+pgi
[ RUN ] HelloThreadedExtended2Test @daint:gpu+cray
[ RUN ] HelloThreadedExtended2Test @daint:mc+gnu
[ RUN ] HelloThreadedExtended2Test @daint:mc+intel
[ RUN ] HelloThreadedExtended2Test @daint:mc+pgi
[ RUN ] HelloThreadedExtended2Test @daint:mc+cray
[ RUN ] StreamWithRefTest @daint:login+gnu
[ RUN ] StreamWithRefTest @daint:gpu+gnu
[ RUN ] StreamWithRefTest @daint:mc+gnu
[ OK ] ( 1/42) HelloMultiLangTest %lang=cpp @daint:login+builtin [compile: 4.053s run: 36.016s total: 43.208s]
[ OK ] ( 2/42) HelloMultiLangTest %lang=cpp @daint:login+gnu [compile: 4.047s run: 36.009s total: 43.203s]
[ OK ] ( 3/42) HelloMultiLangTest %lang=cpp @daint:login+intel [compile: 3.431s run: 35.376s total: 43.206s]
[ OK ] ( 4/42) HelloMultiLangTest %lang=cpp @daint:login+pgi [compile: 2.758s run: 34.675s total: 43.208s]
[ OK ] ( 5/42) HelloMultiLangTest %lang=cpp @daint:login+cray [compile: 2.149s run: 34.052s total: 43.211s]
[ OK ] ( 6/42) HelloMultiLangTest %lang=cpp @daint:gpu+gnu [compile: 2.139s run: 60.830s total: 69.995s]
[ OK ] ( 7/42) HelloMultiLangTest %lang=cpp @daint:gpu+intel [compile: 8.863s run: 55.184s total: 70.004s]
[ OK ] ( 8/42) HelloMultiLangTest %lang=c @daint:login+builtin [compile: 32.460s run: 18.053s total: 69.949s]
[ OK ] ( 9/42) HelloMultiLangTest %lang=c @daint:login+gnu [compile: 27.081s run: 18.051s total: 69.954s]
[ OK ] (10/42) HelloMultiLangTest %lang=c @daint:login+intel [compile: 39.615s run: 32.065s total: 87.922s]
[ OK ] (11/42) HelloMultiLangTest %lang=c @daint:login+pgi [compile: 38.873s run: 31.356s total: 87.926s]
[ OK ] (12/42) HelloMultiLangTest %lang=c @daint:login+cray [compile: 38.265s run: 30.731s total: 87.931s]
[ OK ] (13/42) HelloThreadedExtended2Test @daint:login+builtin [compile: 12.837s run: 7.254s total: 92.404s]
[ OK ] (14/42) HelloThreadedExtended2Test @daint:login+gnu [compile: 31.377s run: 31.894s total: 119.747s]
[ OK ] (15/42) HelloThreadedExtended2Test @daint:login+intel [compile: 30.708s run: 31.252s total: 119.749s]
[ OK ] (16/42) HelloThreadedExtended2Test @daint:login+pgi [compile: 18.581s run: 30.571s total: 119.753s]
[ OK ] (17/42) HelloThreadedExtended2Test @daint:login+cray [compile: 17.981s run: 29.963s total: 119.756s]
[ OK ] (18/42) HelloMultiLangTest %lang=cpp @daint:mc+intel [compile: 33.792s run: 87.427s total: 130.572s]
[ OK ] (19/42) HelloMultiLangTest %lang=cpp @daint:mc+pgi [compile: 33.120s run: 84.192s total: 130.591s]
[ OK ] (20/42) HelloMultiLangTest %lang=cpp @daint:mc+cray [compile: 32.474s run: 81.119s total: 130.609s]
[ OK ] (21/42) HelloMultiLangTest %lang=c @daint:mc+pgi [compile: 13.468s run: 51.389s total: 130.540s]
[ OK ] (22/42) HelloMultiLangTest %lang=c @daint:mc+cray [compile: 12.847s run: 48.146s total: 130.559s]
[ OK ] (23/42) HelloMultiLangTest %lang=cpp @daint:gpu+pgi [compile: 8.167s run: 120.870s total: 138.874s]
[ OK ] (24/42) HelloMultiLangTest %lang=cpp @daint:gpu+cray [compile: 7.412s run: 109.470s total: 138.883s]
[ OK ] (25/42) HelloMultiLangTest %lang=c @daint:gpu+gnu [compile: 13.293s run: 81.519s total: 138.729s]
[ OK ] (26/42) HelloMultiLangTest %lang=c @daint:gpu+cray [compile: 11.378s run: 74.651s total: 138.736s]
[ OK ] (27/42) HelloMultiLangTest %lang=c @daint:mc+gnu [compile: 25.399s run: 65.789s total: 138.749s]
[ OK ] (28/42) HelloMultiLangTest %lang=c @daint:gpu+intel [compile: 12.677s run: 79.097s total: 139.421s]
[ OK ] (29/42) HelloMultiLangTest %lang=c @daint:gpu+pgi [compile: 23.579s run: 69.505s total: 139.432s]
[ OK ] (30/42) HelloThreadedExtended2Test @daint:gpu+gnu [compile: 22.616s run: 46.878s total: 139.268s]
[ OK ] (31/42) HelloThreadedExtended2Test @daint:gpu+pgi [compile: 21.265s run: 40.181s total: 139.267s]
[ OK ] (32/42) HelloThreadedExtended2Test @daint:gpu+cray [compile: 20.642s run: 37.158s total: 139.275s]
[ OK ] (33/42) HelloThreadedExtended2Test @daint:mc+gnu [compile: 4.691s run: 30.273s total: 139.280s]
[ OK ] (34/42) HelloThreadedExtended2Test @daint:mc+intel [compile: 28.304s run: 19.597s total: 139.281s]
[ OK ] (35/42) StreamWithRefTest @daint:login+gnu [compile: 24.257s run: 10.594s total: 139.286s]
[ OK ] (36/42) HelloMultiLangTest %lang=c @daint:mc+intel [compile: 14.135s run: 70.976s total: 146.961s]
[ OK ] (37/42) HelloMultiLangTest %lang=cpp @daint:mc+gnu [compile: 7.397s run: 194.065s total: 229.737s]
[ OK ] (38/42) HelloThreadedExtended2Test @daint:gpu+intel [compile: 21.956s run: 133.885s total: 229.342s]
[ OK ] (39/42) HelloThreadedExtended2Test @daint:mc+pgi [compile: 27.596s run: 106.403s total: 229.264s]
[ OK ] (40/42) HelloThreadedExtended2Test @daint:mc+cray [compile: 26.958s run: 103.318s total: 229.274s]
[ OK ] (41/42) StreamWithRefTest @daint:gpu+gnu [compile: 38.940s run: 98.873s total: 229.279s]
[ OK ] (42/42) StreamWithRefTest @daint:mc+gnu [compile: 38.304s run: 94.811s total: 229.299s]
[----------] all spawned checks have finished

[ PASSED ] Ran 42/42 test case(s) from 4 check(s) (0 failure(s), 0 skipped)
[==========] Finished on Sat Jan 22 22:47:28 2022
==============================================================================
PERFORMANCE REPORT
------------------------------------------------------------------------------
StreamWithRefTest
- daint:login
- gnu
* num_tasks: 1
* Copy: 67915.3 MB/s
* Scale: 37485.6 MB/s
* Add: 39545.5 MB/s
* Triad: 39906.2 MB/s
- daint:gpu
- gnu
* num_tasks: 1
* Copy: 50553.4 MB/s
* Scale: 34780.1 MB/s
* Add: 38043.6 MB/s
* Triad: 38522.2 MB/s
- daint:mc
- gnu
* num_tasks: 1
* Copy: 48200.9 MB/s
* Scale: 31370.4 MB/s
* Add: 33000.2 MB/s
* Triad: 33205.5 MB/s
------------------------------------------------------------------------------
Run report saved in '/home/user/.reframe/reports/run-report.json'
Log file(s) saved in '/tmp/rfm-n3d18lq9.log'
117 changes: 117 additions & 0 deletions docs/listings/deps_complex_run.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
[ReFrame Setup]
version: 3.10.0-dev.3+149af549
command: './bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -r'
launched by: user@host
working directory: '/home/user/Repositories/reframe'
settings file: '<builtin>'
check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py'
stage directory: '/home/user/Repositories/reframe/stage'
output directory: '/home/user/Repositories/reframe/output'

[==========] Running 10 check(s)
[==========] Started on Sat Jan 22 23:44:18 2022

[----------] start processing checks
[ RUN ] T0 @generic:default+builtin
[ OK ] ( 1/10) T0 @generic:default+builtin [compile: 0.018s run: 0.292s total: 0.336s]
[ RUN ] T4 @generic:default+builtin
[ OK ] ( 2/10) T4 @generic:default+builtin [compile: 0.016s run: 0.336s total: 0.380s]
[ RUN ] T5 @generic:default+builtin
[ OK ] ( 3/10) T5 @generic:default+builtin [compile: 0.016s run: 0.389s total: 0.446s]
[ RUN ] T1 @generic:default+builtin
[ OK ] ( 4/10) T1 @generic:default+builtin [compile: 0.016s run: 0.459s total: 0.501s]
[ RUN ] T8 @generic:default+builtin
[ FAIL ] ( 5/10) T8 @generic:default+builtin [compile: n/a run: n/a total: 0.006s]
==> test failed during 'setup': test staged in '/home/user/Repositories/reframe/stage/generic/default/builtin/T8'
[ FAIL ] ( 6/10) T9 @generic:default+builtin [compile: n/a run: n/a total: n/a]
==> test failed during 'startup': test staged in None
[ RUN ] T6 @generic:default+builtin
[ OK ] ( 7/10) T6 @generic:default+builtin [compile: 0.016s run: 0.530s total: 0.584s]
[ RUN ] T2 @generic:default+builtin
[ RUN ] T3 @generic:default+builtin
[ FAIL ] ( 8/10) T2 @generic:default+builtin [compile: 0.019s run: 0.324s total: 0.424s]
==> test failed during 'sanity': test staged in '/home/user/Repositories/reframe/stage/generic/default/builtin/T2'
[ FAIL ] ( 9/10) T7 @generic:default+builtin [compile: n/a run: n/a total: n/a]
==> test failed during 'startup': test staged in None
[ OK ] (10/10) T3 @generic:default+builtin [compile: 0.017s run: 0.328s total: 0.403s]
[----------] all spawned checks have finished

[ FAILED ] Ran 10/10 test case(s) from 10 check(s) (4 failure(s), 0 skipped)
[==========] Finished on Sat Jan 22 23:44:21 2022

==============================================================================
SUMMARY OF FAILURES
------------------------------------------------------------------------------
FAILURE INFO for T8
* Expanded name: T8
* Description: T8
* System partition: generic:default
* Environment: builtin
* Stage directory: /home/user/Repositories/reframe/stage/generic/default/builtin/T8
* Node list:
* Job type: local (id=None)
* Dependencies (conceptual): ['T1']
* Dependencies (actual): [('T1', 'generic:default', 'builtin')]
* Maintainers: []
* Failing phase: setup
* Rerun with '-n T8 -p builtin --system generic:default -r'
* Reason: exception
Traceback (most recent call last):
File "/home/user/Repositories/reframe/reframe/frontend/executors/__init__.py", line 291, in _safe_call
return fn(*args, **kwargs)
File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 82, in _fn
getattr(obj, h.__name__)()
File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 32, in _fn
func(*args, **kwargs)
File "/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py", line 180, in fail
raise Exception
Exception

------------------------------------------------------------------------------
FAILURE INFO for T9
* Expanded name: T9
* Description: T9
* System partition: generic:default
* Environment: builtin
* Stage directory: None
* Node list:
* Job type: local (id=None)
* Dependencies (conceptual): ['T8']
* Dependencies (actual): [('T8', 'generic:default', 'builtin')]
* Maintainers: []
* Failing phase: startup
* Rerun with '-n T9 -p builtin --system generic:default -r'
* Reason: task dependency error: dependencies failed
------------------------------------------------------------------------------
FAILURE INFO for T2
* Expanded name: T2
* Description: T2
* System partition: generic:default
* Environment: builtin
* Stage directory: /home/user/Repositories/reframe/stage/generic/default/builtin/T2
* Node list: tresa.localNone
* Job type: local (id=49427)
* Dependencies (conceptual): ['T6']
* Dependencies (actual): [('T6', 'generic:default', 'builtin')]
* Maintainers: []
* Failing phase: sanity
* Rerun with '-n T2 -p builtin --system generic:default -r'
* Reason: sanity error: 31 != 30
------------------------------------------------------------------------------
FAILURE INFO for T7
* Expanded name: T7
* Description: T7
* System partition: generic:default
* Environment: builtin
* Stage directory: None
* Node list:
* Job type: local (id=None)
* Dependencies (conceptual): ['T2']
* Dependencies (actual): [('T2', 'generic:default', 'builtin')]
* Maintainers: []
* Failing phase: startup
* Rerun with '-n T7 -p builtin --system generic:default -r'
* Reason: task dependency error: dependencies failed
------------------------------------------------------------------------------
Run report saved in '/home/user/.reframe/reports/run-report.json'
Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-92y3fr5s.log'
22 changes: 22 additions & 0 deletions docs/listings/deps_rerun_t6.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[ReFrame Setup]
version: 3.10.0-dev.3+149af549
command: './bin/reframe --restore-session --keep-stage-files -n T6 -r'
launched by: user@host
working directory: '/home/user/Repositories/reframe'
settings file: '<builtin>'
check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py'
stage directory: '/home/user/Repositories/reframe/stage'
output directory: '/home/user/Repositories/reframe/output'

[==========] Running 1 check(s)
[==========] Started on Sat Jan 22 23:44:25 2022

[----------] start processing checks
[ RUN ] T6 @generic:default+builtin
[ OK ] (1/1) T6 @generic:default+builtin [compile: 0.017s run: 0.286s total: 0.330s]
[----------] all spawned checks have finished

[ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped)
[==========] Finished on Sat Jan 22 23:44:25 2022
Run report saved in '/home/user/.reframe/reports/run-report.json'
Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-mug0a4cb.log'
30 changes: 30 additions & 0 deletions docs/listings/deps_run_t6.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
[ReFrame Setup]
version: 3.10.0-dev.3+149af549
command: './bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -n T6 -r'
launched by: user@host
working directory: '/home/user/Repositories/reframe'
settings file: '<builtin>'
check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py'
stage directory: '/home/user/Repositories/reframe/stage'
output directory: '/home/user/Repositories/reframe/output'

[==========] Running 5 check(s)
[==========] Started on Sat Jan 22 23:44:25 2022

[----------] start processing checks
[ RUN ] T0 @generic:default+builtin
[ OK ] (1/5) T0 @generic:default+builtin [compile: 0.017s run: 0.289s total: 0.331s]
[ RUN ] T4 @generic:default+builtin
[ OK ] (2/5) T4 @generic:default+builtin [compile: 0.018s run: 0.330s total: 0.374s]
[ RUN ] T5 @generic:default+builtin
[ OK ] (3/5) T5 @generic:default+builtin [compile: 0.018s run: 0.384s total: 0.442s]
[ RUN ] T1 @generic:default+builtin
[ OK ] (4/5) T1 @generic:default+builtin [compile: 0.018s run: 0.452s total: 0.494s]
[ RUN ] T6 @generic:default+builtin
[ OK ] (5/5) T6 @generic:default+builtin [compile: 0.018s run: 0.525s total: 0.582s]
[----------] all spawned checks have finished

[ PASSED ] Ran 5/5 test case(s) from 5 check(s) (0 failure(s), 0 skipped)
[==========] Finished on Sat Jan 22 23:44:28 2022
Run report saved in '/home/user/.reframe/reports/run-report.json'
Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-ktylyaqk.log'
22 changes: 22 additions & 0 deletions docs/listings/hello1.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[ReFrame Setup]
version: 3.10.0-dev.3+c22440c1
command: './bin/reframe -c tutorials/basics/hello/hello1.py -r'
launched by: user@host
working directory: '/path/to/reframe'
settings file: '<builtin>'
check search path: '/path/to/reframe/tutorials/basics/hello/hello1.py'
stage directory: '/path/to/reframe/stage'
output directory: '/path/to/reframe/output'

[==========] Running 1 check(s)
[==========] Started on Sat Jan 22 13:21:50 2022

[----------] start processing checks
[ RUN ] HelloTest @generic:default+builtin
[ OK ] (1/1) HelloTest @generic:default+builtin [compile: 0.272s run: 0.359s total: 0.784s]
[----------] all spawned checks have finished

[ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped)
[==========] Finished on Sat Jan 22 13:21:51 2022
Run report saved in '/home/user/.reframe/reports/run-report.json'
Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-8c6ybdvg.log'
Loading

0 comments on commit 11493d2

Please sign in to comment.