From 76df75e2e7dd6d05b68a7c899204f82ef8fe8448 Mon Sep 17 00:00:00 2001 From: Andrea Cervesato Date: Thu, 21 Mar 2024 14:38:46 +0100 Subject: [PATCH] Add LTP statistics documentation The LTP statistics documentation is generated at build time. For now, we are generating syscalls statistics, comparing kernel syscalls_n64.tbl file and the tests we have in the testcases/kernel/syscalls folder. --- doc_new/.gitignore | 2 + doc_new/conf.py | 132 ++++++++++++++++++++++++++++++++++++++++ doc_new/index.rst | 4 ++ doc_new/users/stats.rst | 9 +++ 4 files changed, 147 insertions(+) create mode 100644 doc_new/users/stats.rst diff --git a/doc_new/.gitignore b/doc_new/.gitignore index 5ccff1a6bea2..7b6f0a4ab45c 100644 --- a/doc_new/.gitignore +++ b/doc_new/.gitignore @@ -1 +1,3 @@ html/ +_static/syscalls.rst +syscalls.tbl diff --git a/doc_new/conf.py b/doc_new/conf.py index 44122b761495..42d4ec7abba1 100644 --- a/doc_new/conf.py +++ b/doc_new/conf.py @@ -3,6 +3,12 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +import os +import re +import sphinx +import socket +import urllib.request + # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information @@ -22,5 +28,131 @@ html_theme = 'sphinx_rtd_theme' html_static_path = ['_static'] + +def generate_syscalls_stats(_): + """ + Generate statistics for syscalls. We fetch the syscalls list from the kernel + sources, then we compare it with testcases/kernel/syscalls folder and + generate a file that is included in the statistics documentation section. + """ + output = '_static/syscalls.rst' + + # sometimes checking testcases/kernel/syscalls file names are not enough, + # because in some cases (i.e. io_ring) syscalls are tested, but they are + # part of a more complex scenario. In the following list, we define syscalls + # which we know they are 100% tested already. + white_list = [ + 'rt_sigpending', + 'sethostname', + 'lsetxattr', + 'inotify_add_watch', + 'inotify_rm_watch', + 'newfstatat', + 'pselect6', + 'fanotify_init', + 'fanotify_mark', + 'prlimit64', + 'getdents64', + 'pkey_mprotect', + 'pkey_alloc', + 'pkey_free', + 'io_uring_setup', + 'io_uring_enter', + 'io_uring_register', + 'epoll_pwait2', + 'quotactl_fd', + ] + + # populate with non-syscalls which are present in the kernel sources + # syscalls file + black_list = [ + 'reserved177', + 'reserved193', + 'rseq' + ] + + text = [ + 'Syscalls\n', + '--------\n\n', + ] + + error = False + try: + socket.setdefaulttimeout(3) + urllib.request.urlretrieve( + "https://raw.githubusercontent.com/torvalds/linux/master/arch/mips/kernel/syscalls/syscall_n64.tbl", + "syscalls.tbl") + except Exception as err: + error = True + logger = sphinx.util.logging.getLogger(__name__) + msg = "Can't download syscall_n64.tbl from kernel sources" + logger.warning(msg) + + with open(output, 'w+') as stats: + stats.write(f".. warning::\n\n\t{msg}") + + if error: + return + + # collect all available kernel syscalls + regexp = re.compile(r'\d+\s+n64\s+(?P\w+)\s+\w+') + ker_syscalls = [] + with open("syscalls.tbl", 'r') as data: + for line in data: + match = regexp.search(line) + if match: + ker_syscalls.append(match.group('syscall')) + + # collect all LTP tested syscalls + ltp_syscalls = [] + for root, _, files in os.walk('../testcases/kernel/syscalls'): + for myfile in files: + if myfile.endswith('.c'): + ltp_syscalls.append(myfile) + + # compare kernel syscalls with LTP tested syscalls + syscalls = {} + for kersc in ker_syscalls: + if kersc in black_list: + continue + + if kersc not in syscalls: + if kersc in white_list: + syscalls[kersc] = True + continue + + syscalls[kersc] = False + + for ltpsc in ltp_syscalls: + if ltpsc.startswith(kersc): + syscalls[kersc] = True + + # generate the statistics file + tested_syscalls = [key for key, val in syscalls.items() if val] + text.append('syscalls which are tested under **testcases/kernel/syscalls**:\n\n') + text.append(f'* kernel syscalls: {len(ker_syscalls)}\n') + text.append(f'* tested syscalls: {len(tested_syscalls)}\n\n') + + # tested syscalls + text.append('.. list-table::\n') + text.append(' :header-rows: 1\n\n') + text.append(' * - Tested syscalls\n') + for sysname, tested in syscalls.items(): + if tested: + text.append(f' * - {sysname}\n') + + # untested syscalls + text.append('\n.. list-table::\n') + text.append(' :header-rows: 1\n\n') + text.append(' * - Untested syscalls\n') + for sysname, tested in syscalls.items(): + if not tested: + text.append(f' * - {sysname}\n') + + with open(output, 'w+') as stats: + stats.writelines(text) + + def setup(app): app.add_css_file('custom.css') + app.connect('builder-inited', generate_syscalls_stats) diff --git a/doc_new/index.rst b/doc_new/index.rst index a8778b3d8201..9a4995d5f6b5 100644 --- a/doc_new/index.rst +++ b/doc_new/index.rst @@ -35,6 +35,7 @@ Some references: users/quick_start users/setup_tests users/supported_systems + users/stats .. toctree:: :maxdepth: 3 @@ -74,6 +75,9 @@ For users :doc:`users/supported_systems` A list of supported technologies by the LTP framework +:doc:`users/stats` + Some LTP statistics + For developers -------------- diff --git a/doc_new/users/stats.rst b/doc_new/users/stats.rst new file mode 100644 index 000000000000..7073442aaacd --- /dev/null +++ b/doc_new/users/stats.rst @@ -0,0 +1,9 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Statistics +========== + +In this section we collect some statistics related to the current state of +LTP tests. + +.. include:: ../_static/syscalls.rst