[core] Fix decode errors from searchkit

When UTF-8 decoding fails, searchkit throws an exception.

Passing decode_errors='backslashreplace' to cope with that.

Fixes #860.

Signed-off-by: Ponnuvel Palaniyappan <[email protected]>

hotsos/core/host_helpers/apparmor.py

@@ -27,7 +27,7 @@ def profiles(self):
 
         @return: dictionary of {<mode>: {'profiles': <list>, 'count': <int>}}
         """
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         seqdef = SequenceSearchDef(
                      start=SearchDef(r"(\d+) profiles are in (\S+) mode."),
                      body=SearchDef(r"\s+(\S+)"),

hotsos/core/host_helpers/network.py

@@ -102,7 +102,7 @@ def stats(self):
         if counters:
             return counters
 
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         seqdef = SequenceSearchDef(
                     # match start of interface
                     start=SearchDef(IP_IFACE_NAME_TEMPLATE.format(self.name)),

hotsos/core/host_helpers/systemd.py

@@ -145,7 +145,7 @@ def memory_current_kb(self):
                                 format(self.name), 'memory.current')
         if os.path.exists(cgroupv1):
             total_usage = {}
-            fs = FileSearcher()
+            fs = FileSearcher(decode_errors='backslashreplace')
             fs.add(SearchDef(r'(cache|rss|swap) (\d+)'), path=cgroupv1)
             for result in fs.run().get(cgroupv1, {}):
                 total_usage[result.get(1)] = int(result.get(2))

hotsos/core/plugins/kernel/kernlog/common.py

@@ -84,7 +84,8 @@ class KernLogBase(object):
 
     def __init__(self):
         c = SearchConstraintSearchSince(ts_matcher_cls=CommonTimestampMatcher)
-        self.searcher = FileSearcher(constraint=c)
+        self.searcher = FileSearcher(constraint=c,
+                                     decode_errors='backslashreplace')
         self.hostnet_helper = HostNetworkingHelper()
         self.cli_helper = CLIHelper()
 

hotsos/core/plugins/kernel/net.py

@@ -466,7 +466,7 @@ class Lsof(STOVParserBase):
     """
 
     def _load(self):
-        search = FileSearcher()
+        search = FileSearcher(decode_errors='backslashreplace')
         with CLIHelperFile() as cli:
             fout = cli.lsof_Mnlc()
             search.add(SearchDef(self._header_matcher, tag='header'), fout)

hotsos/core/plugins/lxd/common.py

@@ -24,7 +24,7 @@ class LXD(object):
     def instances(self):
         """ Return a list of instance names. """
         _instances = []
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         seq = SequenceSearchDef(start=SearchDef(r'^## Instances$'),
                                 body=SearchDef(r'^\|\s+(\S+)\s+\|'),
                                 end=SearchDef(r'##.*'),

hotsos/core/plugins/openstack/nova.py

@@ -127,7 +127,7 @@ def cpu_models(self):
 
         guests = []
         seqs = {}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         for i in self.instances.values():
             guests.append(i.name)
             start = SearchDef(r"\s+<cpu .+>")

hotsos/core/plugins/openvswitch/ovn.py

@@ -54,7 +54,7 @@ class OVNDBBase(object):
 
     def __init__(self):
         contents = mktemp_dump(''.join(self.db_show))
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         s.add(self.resources_sd, contents)
         self.results = s.run()
 

hotsos/core/plugins/openvswitch/ovs.py

@@ -148,7 +148,7 @@ def tunnels(self):
 
         nethelp = HostNetworkingHelper()
         with CLIHelperFile() as cli:
-            s = FileSearcher()
+            s = FileSearcher(decode_errors='backslashreplace')
             expr = r'.+ \(([a-z]+): ([a-f\d\.:]+)->([a-f\d\.:]+), .+'
             s.add(SearchDef(expr, tag='all'),
                   cli.ovs_appctl(command='ofproto/list-tunnels'))

hotsos/core/plugins/rabbitmq/report.py

@@ -26,7 +26,7 @@ class RabbitMQReport(object):
     def __init__(self):
         # save to file so we can search it later
         with CLIHelperFile() as cli:
-            searcher = FileSearcher()
+            searcher = FileSearcher(decode_errors='backslashreplace')
             fout = cli.rabbitmqctl_report()
             searcher.add(self.connections_searchdef, fout)
             searcher.add(self.memory_searchdef, fout)

hotsos/core/plugins/sosreport.py

@@ -49,7 +49,7 @@ def timed_out_plugins(self):
                                            'sos_logs')):
             return timeouts
 
-        searcher = FileSearcher()
+        searcher = FileSearcher(decode_errors='backslashreplace')
         path = os.path.join(HotSOSConfig.data_root, 'sos_logs/ui.log')
         searcher.add(SearchDef(r".* Plugin (\S+) timed out.*", tag="timeouts"),
                      path=path)

hotsos/core/plugins/storage/bcache.py

@@ -115,7 +115,7 @@ def udev_bcache_devs(self):
         """ If bcache devices exist fetch information and return as a list. """
         devs = []
         with CLIHelperFile() as cli:
-            s = FileSearcher()
+            s = FileSearcher(decode_errors='backslashreplace')
             sdef = SequenceSearchDef(start=SearchDef(r"^P: .+/(bcache\S+)"),
                                      body=SearchDef(r"^S: disk/by-uuid/(\S+)"),
                                      tag="bcacheinfo")

hotsos/core/plugins/storage/ceph.py

@@ -427,7 +427,7 @@ def _get_version_info(self, daemon_type=None):
         version (and only versions for that daemon type.)
         """
         versions = {}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         body = SearchDef(r"\s+\"ceph version (\S+) .+ (\S+) "
                          r"\(\S+\)\":\s+(\d)+,?$")
         if daemon_type is None:
@@ -800,7 +800,7 @@ def rss(self):
 
         NOTE: this assumes we have ps auxwwwm format.
         """
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         # columns: USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
         if self.id is not None:
             ceph_id = r"--id\s+{}".format(self.id)
@@ -1030,7 +1030,7 @@ def local_osds(self):
         """
         osds = []
 
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         sd = SequenceSearchDef(start=SearchDef(r"^=+\s+osd\.(\d+)\s+=+.*"),
                                body=SearchDef([r"\s+osd\s+(fsid)\s+(\S+)\s*",
                                                r"\s+(devices)\s+([\S]+)\s*"]),

hotsos/core/plugins/system/system.py

@@ -167,7 +167,7 @@ def ubuntu_pro_status(self):
         # human-readable output for now. This function should ideally
         # rely on the json output when the upstream sos starts to
         # include it.
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         service_status_seqdef = SequenceSearchDef(
             start=SearchDef(r"^SERVICE +ENTITLED +STATUS +DESCRIPTION"),
             body=SearchDef(r"^(\S+) +(\S+) +(\S+) +(.+)\n"),

hotsos/core/ycheck/scenarios.py

@@ -43,7 +43,8 @@ def __init__(self, *args, **kwargs):
         constraint = SearchConstraintSearchSince(
                                         ts_matcher_cls=CommonTimestampMatcher,
                                         hours=hours)
-        self._searcher = FileSearcher(constraint=constraint)
+        self._searcher = FileSearcher(constraint=constraint,
+                                      decode_errors='backslashreplace')
 
     @property
     def searcher(self):

hotsos/plugin_extensions/juju/summary.py

@@ -23,7 +23,8 @@ class UnitLogInfo(object):
     def error_and_warnings(self):
         log.debug("searching unit logs for errors and warnings")
         c = SearchConstraintSearchSince(ts_matcher_cls=CommonTimestampMatcher)
-        searchobj = FileSearcher(constraint=c)
+        searchobj = FileSearcher(constraint=c,
+                                 decode_errors='backslashreplace')
         path = os.path.join(HotSOSConfig.data_root, 'var/log/juju/unit-*.log')
         ts_expr = r"^([\d-]+)\s+([\d:]+)"
         for msg in ['ERROR', 'WARNING']:

hotsos/plugin_extensions/openstack/agent/events.py

@@ -370,6 +370,8 @@ def _run_checks(self, checks):
             return
 
         check_objs = [c() for c in checks]
+        check_objs = [c(searcher=searcher) for c in checks]
+        results = searcher.run()
         _final_results = {}
         for check in check_objs:
             check.run()

hotsos/plugin_extensions/openstack/agent/exceptions.py

@@ -116,7 +116,8 @@ def __init__(self):
         self._agent_results = None
         c = SearchConstraintSearchSince(
                                       ts_matcher_cls=CommonTimestampMatcher)
-        self.searchobj = FileSearcher(constraint=c)
+        self.searchobj = FileSearcher(constraint=c,
+                                      decode_errors='backslashreplace')
         # The following are expected to be logged using WARNING log level.
         self._agent_warnings = {
             'nova': ['MessagingTimeout',

hotsos/plugin_extensions/openstack/nova_external_events.py

@@ -33,7 +33,7 @@ def __call__(self, event):
         events_found = {}
 
         c = SearchConstraintSearchSince(ts_matcher_cls=CommonTimestampMatcher)
-        s = FileSearcher(constraint=c)
+        s = FileSearcher(constraint=c, decode_errors='backslashreplace')
         for result in event.results:
             instance_id = result.get(1)
             event_id = result.get(2)

tests/unit/test_analytics.py

@@ -66,7 +66,7 @@ def test_ordered_complete(self):
         expected = {'0': {'duration': 60.0,
                           'start': start0, 'end': end0},
                     '1': {'duration': 60.0, 'start': start1, 'end': end1}}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start'
         s.add(SearchDef(expr, tag="eventX-start"), path=fname)
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end'
@@ -90,7 +90,7 @@ def test_unordered_complete(self):
         expected = {'0': {'duration': 60.0,
                           'start': start0, 'end': end0},
                     '1': {'duration': 60.0, 'start': start1, 'end': end1}}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start'
         s.add(SearchDef(expr, tag="eventX-start"), path=fname)
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end'
@@ -114,7 +114,7 @@ def test_ordered_complete_clobbered(self):
         expected = {'0': {'duration': 60.0,
                           'start': start0, 'end': end0},
                     '1': {'duration': 60.0, 'start': start1, 'end': end1}}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start'
         s.add(SearchDef(expr, tag="eventX-start"), path=fname)
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end'
@@ -141,7 +141,7 @@ def test_ordered_incomplete_clobbered(self):
         expected = {'0': {'duration': 60.0,
                           'start': start0, 'end': end0},
                     '1': {'duration': 60.0, 'start': start1, 'end': end1}}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start'
         s.add(SearchDef(expr, tag="eventX-start"), path=fname)
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end'
@@ -169,7 +169,7 @@ def test_ordered_incomplete_clobbered2(self):
         expected = {'0': {'duration': 120.0,
                           'start': start0, 'end': end0},
                     '1': {'duration': 60.0, 'start': start1, 'end': end1}}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start'
         s.add(SearchDef(expr, tag="eventX-start"), path=fname)
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end'
@@ -193,7 +193,7 @@ def test_ordered_multiple(self):
         expected = {'0': {'duration': 60.0,
                           'start': start0, 'end': end0},
                     '1': {'duration': 60.0, 'start': start1, 'end': end1}}
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start'
         s.add(SearchDef(expr, tag="eventX-start"), path=fname)
         expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end'

tests/unit/test_ycheck_scenarios.py

@@ -739,7 +739,7 @@ def _create_search_results(self, path, contents=None):
                 for line in contents:
                     fd.write(line)
 
-        s = FileSearcher()
+        s = FileSearcher(decode_errors='backslashreplace')
         s.add(SearchDef(r'^(\S+) (\S+) .+', tag='all'), path)
         return s.run().find_by_tag('all')