diff --git a/scripts/classify-failures b/scripts/classify-failures new file mode 100755 index 00000000..5a90942f --- /dev/null +++ b/scripts/classify-failures @@ -0,0 +1,254 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. + +import argparse +import subprocess +import sys + +ISSUES = [ + ( + "", + "", + ), + ( + # The detection may be improved, not sure if just with grep. + "[675] ([1]) https://github.com/rhinstaller/kickstart-tests/issues/675 The string is also in successful installation, but there is high chance that for test which times out it will be this issue.", + "Failed to start man-db-cache-update.service: Unit man-db-cache-update.service not found.", + # Context of this line can be useful as well + # "INFO anaconda:dnf: Running transaction check", + ), + ( + "[6] - for rebooting tests it can be rather [675] or something else, so check. Look at kstest.log, libvirt log.", + "RESULT.*Problem starting virtual install", + ), + ( + "[694] ([4]) https://github.com/rhinstaller/kickstart-tests/issues/694", + "Nothing useful found for Hard drive ISO", + ), + ( + "[992] https://github.com/rhinstaller/kickstart-tests/issues/992 - or it can some more recent issue, the string is pretty generic.", + "Process Core Dump", + ), + ( + "[846] on rhel8 ([26]) https://github.com/rhinstaller/kickstart-tests/issues/846", + "raise.*Failed to activate service 'org.freedesktop.hostname1'", + ), + ( + "[767] ([758]) https://github.com/rhinstaller/kickstart-tests/issues/767", + "Payload error.*Failed to download metadata for repo", + ), + ( + "[846] on rhel8 ([26]) https://github.com/rhinstaller/kickstart-tests/issues/846", + "Network.*Failed to activate service 'org.freedesktop.hostname1'", + ), + ( + "[786] https://github.com/rhinstaller/kickstart-tests/issues/786", + "Traceback.*Failed to activate swap on /dev/md/test-raid-ddf_0p2: No such file or directory", + ), + ( + "[859] https://github.com/rhinstaller/kickstart-tests/issues/859", + "Failed to activate filesystems: invalid device specification", + ), + ( + "[845] https://github.com/rhinstaller/kickstart-tests/issues/845", + "INFO lvmdbusd:KeyError: 'pv_uuid'", + ), + ( + "[889] raid-ddf https://github.com/rhinstaller/kickstart-tests/issues/889", + "SwapError: Failed to open the device '/dev/md/test-raid-ddf_0p2'", + ), + ( + "[890] default-systemd-target-vnc-graphical https://github.com/rhinstaller/kickstart-tests/issues/890", + "gnome-kiosk exited on signal 11", + ), + ( + "[857] resource to create this format lvmpv is unavailable https://github.com/rhinstaller/kickstart-tests/issues/857", + "ERROR.*resource to create this format lvmpv is unavailable", + ), + ( + "[894] rpm-ostree https://github.com/rhinstaller/kickstart-tests/issues/894", + "PayloadInstallationError: Failed to pull from repository.*Timeout was reached", + ), + ( + "[TODO1] The DNF payload failed https://github.com/rhinstaller/kickstart-tests/issues/", + "The DNF payload failed", + ), + ( + "[949] infrastructure https://github.com/rhinstaller/kickstart-tests/issues/949", + "Failed to add the 'anaconda' repository", + ), + ( + "[962] https://github.com/rhinstaller/kickstart-tests/issues/962", + "BUG: soft lockup", + ), + ( + "[964] Validation failed and no RESULT https://github.com/rhinstaller/kickstart-tests/issues/964", + "FAILED:Validation failed with return code 1", + ), + ( + "[964] Validation failed and no RESULT - another string https://github.com/rhinstaller/kickstart-tests/issues/964", + "/root/RESULT does not exist in VM image.", + ), + ( + "[930] [939] [794] [782] https://github.com/rhinstaller/kickstart-tests/issues/930", + "lvmdbusd:json.decoder.JSONDecodeError:", + ), + ( + "[985] https://github.com/rhinstaller/kickstart-tests/issues/985", + "WARNING.*Problem 1.*python3-dnf", + ), + ( + "[984] https://github.com/rhinstaller/kickstart-tests/issues/984", + "RESULT.*CRIT.*Anaconda crashed on signal 11", + ), + ( + "[983] https://github.com/rhinstaller/kickstart-tests/issues/983", + "CRIT.*argument of type 'NoneType' is not iterable", + ), + ( + "[993] https://github.com/rhinstaller/kickstart-tests/issues/993", + "WARNING.*nothing provides.*getent", + ), + ( + "[997] https://github.com/rhinstaller/kickstart-tests/issues/997", + "CRIT.*gnome-kiosk exited with status 1", + ), +] + +CLOSED_ISSUES = [ + ( + "[11] https://github.com/rhinstaller/kickstart-tests/issues/795", + "ERR.*Timeout trying to start Xorg", + ), + ( + "[9] RHSM https://github.com/rhinstaller/kickstart-tests/issues/707", + "DBusError: {\"exception\": \"NoSectionError\", \"severity\": \"error\", \"message\": \"No section: 'logging'\"}", + ), + ( + "[780] RHSM https://github.com/rhinstaller/kickstart-tests/issues/780", + "rhsm-service:ERROR.*argument of type 'Undefined' is not iterable", + ), + ( + "[779] RHSM https://github.com/rhinstaller/kickstart-tests/issues/779", + "rhsm-service:ERROR.*'Undefined' object is not iterable", + ), + ( + "[5] https://bugzilla.redhat.com/show_bug.cgi?id=1931389", + "Network:.*Fatal.*Segmentation fault", + ), + ( + "[24] package download failure, may be infra hiccup?", + "Failed to download.*Curl error", + ), + ( + "[759] https://github.com/rhinstaller/kickstart-tests/issues/759", + "Validation.*Your BIOS-based system", + ), + ( + "[879] https://github.com/rhinstaller/kickstart-tests/issues/879", + "INFO:program:/usr/sbin/grub2-probe: error: ../grub-core/kern/disk.c:236", + ), + ( + "[882] blivet DMTech https://github.com/rhinstaller/kickstart-tests/issues/882", + "AttributeError: type object 'DMTech' has no attribute 'MAP'", + ), + ( + "[980] https://github.com/rhinstaller/kickstart-tests/issues/980", + "Payloads: - nothing provides libperl", + ), +] + + +FILTER_FILENAMES = ["kstest.log", "virt-install.log"] + + +def _count_matches(output): + # using grep's default group separator (see grep --group-separator) + return output.count('--\n') + 1 + + +def cmd_cli(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" +Classify kickstart test failures looked up in logs. + +Runs grep in the working directory recursively trying to match log files +to the issues tracked in kickstart-test repository. + +Ideally should be run on log files tree produced by weekly_summary script +with --archive-logs option, which is also run by kickstart-test repository +Weekly summary workflow. + +Can be used also on any kickstart test logs (the names of files to be +scanned are configurable by option) but examining logs of tests that +passed can produce noise in some cases.""", + epilog=""" +EXAMPLES: + + Look only for issue "[1]" and show context of 3 lines for each match: + {filename} --show-line-context 3 --filter-issue-contains [1] + + Look only for failures (ignore flakes subdir in log path): + (Assumes log directory structure produced by weekly_report.) + {filename} --exclude-dir flakes + + Look only to results of daily-iso scenario: + (Assumes log directory structure produced by weekly_report.) + {filename} --exclude-dir "logs-rhel*" +""".format(filename=sys.argv[0]) + ) + parser.add_argument("-i", "--filter-issue-contains", + help="only look for issues containing this substring", + metavar="STRING", default="") + parser.add_argument("-f", "--filter-filenames", + help="only look at specified log files", + metavar="FILENAME", nargs="?", action='append', default=[]) + parser.add_argument("-d", "--exclude-dir", + help="exclude directories with suffix (grep API)", + metavar="GLOB", nargs="?", action='append', default=[]) + parser.add_argument("-c", "--show-line-context", + help="show context of the found string", + metavar="N", default=0, type=int) + args = parser.parse_args() + + filenames = args.filter_filenames or FILTER_FILENAMES + include_args = [option for value in zip(["--include"] * len(filenames), filenames) for option in value] or FILTER_FILENAMES + exclude_dir_args = [option for value in zip(["--exclude-dir"] * len(args.exclude_dir), args.exclude_dir) for option in value] + + for issue, grep_re in ISSUES: + if args.filter_issue_contains not in issue: + continue + grep = subprocess.run( + [ + "grep", + "-R", grep_re, + "-C", str(args.show_line_context), + ] + include_args + exclude_dir_args, + capture_output=True, + encoding="utf8" + ) + if grep.stdout: + print("-"*80) + print(issue) + print("#: {}".format(_count_matches(grep.stdout))) + print(grep.stdout.strip()) + + +if __name__ == "__main__": + cmd_cli()