Add script scanning weekly report logs for existing issues

scripts/classify-failures

@@ -0,0 +1,254 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2023  Red Hat, Inc.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# the GNU General Public License v.2, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY expressed or implied, including the implied warranties of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.  You should have received a copy of the
+# GNU General Public License along with this program; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
+# source code or documentation are not subject to the GNU General Public
+# License and may only be used or replicated with the express permission of
+# Red Hat, Inc.
+
+import argparse
+import subprocess
+import sys
+
+ISSUES = [
+    (
+        "<ISSUE DESCRIPTION PREFERABLY WITH GITHUB ISSUE REFERENCE>",
+        "<REGULAR EXPRESSION FOR GREP TO BE LOOKED FOR>",
+    ),
+    (
+        # The detection may be improved, not sure if just with grep.
+        "[675] ([1]) https://github.com/rhinstaller/kickstart-tests/issues/675 The string is also in successful installation, but there is high chance that for test which times out it will be this issue.",
+        "Failed to start man-db-cache-update.service: Unit man-db-cache-update.service not found.",
+        # Context of this line can be useful as well
+        # "INFO anaconda:dnf: Running transaction check",
+    ),
+    (
+        "[6] - for rebooting tests it can be rather [675] or something else, so check. Look at kstest.log, libvirt log.",
+        "RESULT.*Problem starting virtual install",
+    ),
+    (
+        "[694] ([4]) https://github.com/rhinstaller/kickstart-tests/issues/694",
+        "Nothing useful found for Hard drive ISO",
+    ),
+    (
+        "[992] https://github.com/rhinstaller/kickstart-tests/issues/992 - or it can some more recent issue, the string is pretty generic.",
+        "Process Core Dump",
+    ),
+    (
+        "[846] on rhel8 ([26]) https://github.com/rhinstaller/kickstart-tests/issues/846",
+        "raise.*Failed to activate service 'org.freedesktop.hostname1'",
+    ),
+    (
+        "[767] ([758]) https://github.com/rhinstaller/kickstart-tests/issues/767",
+        "Payload error.*Failed to download metadata for repo",
+    ),
+    (
+        "[846] on rhel8 ([26]) https://github.com/rhinstaller/kickstart-tests/issues/846",
+        "Network.*Failed to activate service 'org.freedesktop.hostname1'",
+    ),
+    (
+        "[786] https://github.com/rhinstaller/kickstart-tests/issues/786",
+        "Traceback.*Failed to activate swap on /dev/md/test-raid-ddf_0p2: No such file or directory",
+    ),
+    (
+        "[859] https://github.com/rhinstaller/kickstart-tests/issues/859",
+        "Failed to activate filesystems: invalid device specification",
+    ),
+    (
+        "[845] https://github.com/rhinstaller/kickstart-tests/issues/845",
+        "INFO lvmdbusd:KeyError: 'pv_uuid'",
+    ),
+    (
+        "[889] raid-ddf https://github.com/rhinstaller/kickstart-tests/issues/889",
+        "SwapError: Failed to open the device '/dev/md/test-raid-ddf_0p2'",
+    ),
+    (
+        "[890] default-systemd-target-vnc-graphical https://github.com/rhinstaller/kickstart-tests/issues/890",
+        "gnome-kiosk exited on signal 11",
+    ),
+    (
+        "[857] resource to create this format lvmpv is unavailable https://github.com/rhinstaller/kickstart-tests/issues/857",
+        "ERROR.*resource to create this format lvmpv is unavailable",
+    ),
+    (
+        "[894] rpm-ostree https://github.com/rhinstaller/kickstart-tests/issues/894",
+        "PayloadInstallationError: Failed to pull from repository.*Timeout was reached",
+    ),
+    (
+        "[TODO1] The DNF payload failed https://github.com/rhinstaller/kickstart-tests/issues/",
+        "The DNF payload failed",
+    ),
+    (
+        "[949] infrastructure https://github.com/rhinstaller/kickstart-tests/issues/949",
+        "Failed to add the 'anaconda' repository",
+    ),
+    (
+        "[962] https://github.com/rhinstaller/kickstart-tests/issues/962",
+        "BUG: soft lockup",
+    ),
+    (
+        "[964] Validation failed and no RESULT https://github.com/rhinstaller/kickstart-tests/issues/964",
+        "FAILED:Validation failed with return code 1",
+    ),
+    (
+        "[964] Validation failed and no RESULT - another string https://github.com/rhinstaller/kickstart-tests/issues/964",
+        "/root/RESULT does not exist in VM image.",
+    ),
+    (
+        "[930] [939] [794] [782] https://github.com/rhinstaller/kickstart-tests/issues/930",
+        "lvmdbusd:json.decoder.JSONDecodeError:",
+    ),
+    (
+        "[985] https://github.com/rhinstaller/kickstart-tests/issues/985",
+        "WARNING.*Problem 1.*python3-dnf",
+    ),
+    (
+        "[984] https://github.com/rhinstaller/kickstart-tests/issues/984",
+        "RESULT.*CRIT.*Anaconda crashed on signal 11",
+    ),
+    (
+        "[983] https://github.com/rhinstaller/kickstart-tests/issues/983",
+        "CRIT.*argument of type 'NoneType' is not iterable",
+    ),
+    (
+        "[993] https://github.com/rhinstaller/kickstart-tests/issues/993",
+        "WARNING.*nothing provides.*getent",
+    ),
+    (
+        "[997] https://github.com/rhinstaller/kickstart-tests/issues/997",
+        "CRIT.*gnome-kiosk exited with status 1",
+    ),
+]
+
+CLOSED_ISSUES = [
+    (
+        "[11] https://github.com/rhinstaller/kickstart-tests/issues/795",
+        "ERR.*Timeout trying to start Xorg",
+    ),
+    (
+        "[9] RHSM https://github.com/rhinstaller/kickstart-tests/issues/707",
+        "DBusError: {\"exception\": \"NoSectionError\", \"severity\": \"error\", \"message\": \"No section: 'logging'\"}",
+    ),
+    (
+        "[780] RHSM https://github.com/rhinstaller/kickstart-tests/issues/780",
+        "rhsm-service:ERROR.*argument of type 'Undefined' is not iterable",
+    ),
+    (
+        "[779] RHSM https://github.com/rhinstaller/kickstart-tests/issues/779",
+        "rhsm-service:ERROR.*'Undefined' object is not iterable",
+    ),
+    (
+        "[5] https://bugzilla.redhat.com/show_bug.cgi?id=1931389",
+        "Network:.*Fatal.*Segmentation fault",
+    ),
+    (
+        "[24] package download failure, may be infra hiccup?",
+        "Failed to download.*Curl error",
+    ),
+    (
+        "[759] https://github.com/rhinstaller/kickstart-tests/issues/759",
+        "Validation.*Your BIOS-based system",
+    ),
+    (
+        "[879] https://github.com/rhinstaller/kickstart-tests/issues/879",
+        "INFO:program:/usr/sbin/grub2-probe: error: ../grub-core/kern/disk.c:236",
+    ),
+    (
+        "[882] blivet DMTech https://github.com/rhinstaller/kickstart-tests/issues/882",
+        "AttributeError: type object 'DMTech' has no attribute 'MAP'",
+    ),
+    (
+        "[980] https://github.com/rhinstaller/kickstart-tests/issues/980",
+        "Payloads:  - nothing provides libperl",
+    ),
+]
+
+
+FILTER_FILENAMES = ["kstest.log", "virt-install.log"]
+
+
+def _count_matches(output):
+    # using grep's default group separator (see grep --group-separator)
+    return output.count('--\n') + 1
+
+
+def cmd_cli():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="""
+Classify kickstart test failures looked up in logs.
+
+Runs grep in the working directory recursively trying to match log files
+to the issues tracked in kickstart-test repository.
+
+Ideally should be run on log files tree produced by weekly_summary script
+with --archive-logs option, which is also run by kickstart-test repository
+Weekly summary workflow.
+
+Can be used also on any kickstart test logs (the names of files to be
+scanned are configurable by option) but examining logs of tests that
+passed can produce noise in some cases.""",
+        epilog="""
+EXAMPLES:
+
+  Look only for issue "[1]" and show context of 3 lines for each match:
+  {filename} --show-line-context 3 --filter-issue-contains [1]
+
+  Look only for failures (ignore flakes subdir in log path):
+  (Assumes log directory structure produced by weekly_report.)
+  {filename} --exclude-dir flakes
+
+  Look only to results of daily-iso scenario:
+  (Assumes log directory structure produced by weekly_report.)
+  {filename} --exclude-dir "logs-rhel*"
+""".format(filename=sys.argv[0])
+    )
+    parser.add_argument("-i", "--filter-issue-contains",
+                        help="only look for issues containing this substring",
+                        metavar="STRING", default="")
+    parser.add_argument("-f", "--filter-filenames",
+                        help="only look at specified log files",
+                        metavar="FILENAME", nargs="?", action='append', default=[])
+    parser.add_argument("-d", "--exclude-dir",
+                        help="exclude directories with suffix (grep API)",
+                        metavar="GLOB", nargs="?", action='append', default=[])
+    parser.add_argument("-c", "--show-line-context",
+                        help="show context of the found string",
+                        metavar="N", default=0, type=int)
+    args = parser.parse_args()
+
+    filenames = args.filter_filenames or FILTER_FILENAMES
+    include_args = [option for value in zip(["--include"] * len(filenames), filenames) for option in value] or FILTER_FILENAMES
+    exclude_dir_args = [option for value in zip(["--exclude-dir"] * len(args.exclude_dir), args.exclude_dir) for option in value]
+
+    for issue, grep_re in ISSUES:
+        if args.filter_issue_contains not in issue:
+            continue
+        grep = subprocess.run(
+            [
+                "grep",
+                "-R", grep_re,
+                "-C", str(args.show_line_context),
+            ] + include_args + exclude_dir_args,
+            capture_output=True,
+            encoding="utf8"
+        )
+        if grep.stdout:
+            print("-"*80)
+            print(issue)
+            print("#: {}".format(_count_matches(grep.stdout)))
+            print(grep.stdout.strip())
+
+
+if __name__ == "__main__":
+    cmd_cli()