diff --git a/hotsos/core/host_helpers/apparmor.py b/hotsos/core/host_helpers/apparmor.py index 95368d225..4d32cacb9 100644 --- a/hotsos/core/host_helpers/apparmor.py +++ b/hotsos/core/host_helpers/apparmor.py @@ -27,7 +27,7 @@ def profiles(self): @return: dictionary of {: {'profiles': , 'count': }} """ - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') seqdef = SequenceSearchDef( start=SearchDef(r"(\d+) profiles are in (\S+) mode."), body=SearchDef(r"\s+(\S+)"), diff --git a/hotsos/core/host_helpers/network.py b/hotsos/core/host_helpers/network.py index f46d98ce8..766b212c6 100644 --- a/hotsos/core/host_helpers/network.py +++ b/hotsos/core/host_helpers/network.py @@ -102,7 +102,7 @@ def stats(self): if counters: return counters - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') seqdef = SequenceSearchDef( # match start of interface start=SearchDef(IP_IFACE_NAME_TEMPLATE.format(self.name)), diff --git a/hotsos/core/host_helpers/systemd.py b/hotsos/core/host_helpers/systemd.py index 650153900..2d9359d63 100644 --- a/hotsos/core/host_helpers/systemd.py +++ b/hotsos/core/host_helpers/systemd.py @@ -145,7 +145,7 @@ def memory_current_kb(self): format(self.name), 'memory.current') if os.path.exists(cgroupv1): total_usage = {} - fs = FileSearcher() + fs = FileSearcher(decode_errors='backslashreplace') fs.add(SearchDef(r'(cache|rss|swap) (\d+)'), path=cgroupv1) for result in fs.run().get(cgroupv1, {}): total_usage[result.get(1)] = int(result.get(2)) diff --git a/hotsos/core/plugins/kernel/kernlog/common.py b/hotsos/core/plugins/kernel/kernlog/common.py index 9b6ce7462..8745b090c 100644 --- a/hotsos/core/plugins/kernel/kernlog/common.py +++ b/hotsos/core/plugins/kernel/kernlog/common.py @@ -84,7 +84,8 @@ class KernLogBase(object): def __init__(self): c = SearchConstraintSearchSince(ts_matcher_cls=CommonTimestampMatcher) - self.searcher = FileSearcher(constraint=c) + self.searcher = FileSearcher(constraint=c, + decode_errors='backslashreplace') self.hostnet_helper = HostNetworkingHelper() self.cli_helper = CLIHelper() diff --git a/hotsos/core/plugins/kernel/net.py b/hotsos/core/plugins/kernel/net.py index 8c8e3022c..d45508e83 100644 --- a/hotsos/core/plugins/kernel/net.py +++ b/hotsos/core/plugins/kernel/net.py @@ -466,7 +466,7 @@ class Lsof(STOVParserBase): """ def _load(self): - search = FileSearcher() + search = FileSearcher(decode_errors='backslashreplace') with CLIHelperFile() as cli: fout = cli.lsof_Mnlc() search.add(SearchDef(self._header_matcher, tag='header'), fout) diff --git a/hotsos/core/plugins/lxd/common.py b/hotsos/core/plugins/lxd/common.py index 8b0944467..7c495819c 100644 --- a/hotsos/core/plugins/lxd/common.py +++ b/hotsos/core/plugins/lxd/common.py @@ -24,7 +24,7 @@ class LXD(object): def instances(self): """ Return a list of instance names. """ _instances = [] - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') seq = SequenceSearchDef(start=SearchDef(r'^## Instances$'), body=SearchDef(r'^\|\s+(\S+)\s+\|'), end=SearchDef(r'##.*'), diff --git a/hotsos/core/plugins/openstack/nova.py b/hotsos/core/plugins/openstack/nova.py index 3b2aae0fe..58d7c796a 100644 --- a/hotsos/core/plugins/openstack/nova.py +++ b/hotsos/core/plugins/openstack/nova.py @@ -127,7 +127,7 @@ def cpu_models(self): guests = [] seqs = {} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') for i in self.instances.values(): guests.append(i.name) start = SearchDef(r"\s+") diff --git a/hotsos/core/plugins/openvswitch/ovn.py b/hotsos/core/plugins/openvswitch/ovn.py index ae93f15d6..21db4dd11 100644 --- a/hotsos/core/plugins/openvswitch/ovn.py +++ b/hotsos/core/plugins/openvswitch/ovn.py @@ -54,7 +54,7 @@ class OVNDBBase(object): def __init__(self): contents = mktemp_dump(''.join(self.db_show)) - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') s.add(self.resources_sd, contents) self.results = s.run() diff --git a/hotsos/core/plugins/openvswitch/ovs.py b/hotsos/core/plugins/openvswitch/ovs.py index ddcd4952d..0525243e7 100644 --- a/hotsos/core/plugins/openvswitch/ovs.py +++ b/hotsos/core/plugins/openvswitch/ovs.py @@ -148,7 +148,7 @@ def tunnels(self): nethelp = HostNetworkingHelper() with CLIHelperFile() as cli: - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') expr = r'.+ \(([a-z]+): ([a-f\d\.:]+)->([a-f\d\.:]+), .+' s.add(SearchDef(expr, tag='all'), cli.ovs_appctl(command='ofproto/list-tunnels')) diff --git a/hotsos/core/plugins/rabbitmq/report.py b/hotsos/core/plugins/rabbitmq/report.py index 22085e625..82911a937 100644 --- a/hotsos/core/plugins/rabbitmq/report.py +++ b/hotsos/core/plugins/rabbitmq/report.py @@ -26,7 +26,7 @@ class RabbitMQReport(object): def __init__(self): # save to file so we can search it later with CLIHelperFile() as cli: - searcher = FileSearcher() + searcher = FileSearcher(decode_errors='backslashreplace') fout = cli.rabbitmqctl_report() searcher.add(self.connections_searchdef, fout) searcher.add(self.memory_searchdef, fout) diff --git a/hotsos/core/plugins/sosreport.py b/hotsos/core/plugins/sosreport.py index c6511c01b..fe533533d 100644 --- a/hotsos/core/plugins/sosreport.py +++ b/hotsos/core/plugins/sosreport.py @@ -49,7 +49,7 @@ def timed_out_plugins(self): 'sos_logs')): return timeouts - searcher = FileSearcher() + searcher = FileSearcher(decode_errors='backslashreplace') path = os.path.join(HotSOSConfig.data_root, 'sos_logs/ui.log') searcher.add(SearchDef(r".* Plugin (\S+) timed out.*", tag="timeouts"), path=path) diff --git a/hotsos/core/plugins/storage/bcache.py b/hotsos/core/plugins/storage/bcache.py index a5e0bef60..a7e6b5df1 100644 --- a/hotsos/core/plugins/storage/bcache.py +++ b/hotsos/core/plugins/storage/bcache.py @@ -115,7 +115,7 @@ def udev_bcache_devs(self): """ If bcache devices exist fetch information and return as a list. """ devs = [] with CLIHelperFile() as cli: - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') sdef = SequenceSearchDef(start=SearchDef(r"^P: .+/(bcache\S+)"), body=SearchDef(r"^S: disk/by-uuid/(\S+)"), tag="bcacheinfo") diff --git a/hotsos/core/plugins/storage/ceph.py b/hotsos/core/plugins/storage/ceph.py index 61728194f..59cb1c266 100644 --- a/hotsos/core/plugins/storage/ceph.py +++ b/hotsos/core/plugins/storage/ceph.py @@ -427,7 +427,7 @@ def _get_version_info(self, daemon_type=None): version (and only versions for that daemon type.) """ versions = {} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') body = SearchDef(r"\s+\"ceph version (\S+) .+ (\S+) " r"\(\S+\)\":\s+(\d)+,?$") if daemon_type is None: @@ -800,7 +800,7 @@ def rss(self): NOTE: this assumes we have ps auxwwwm format. """ - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') # columns: USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND if self.id is not None: ceph_id = r"--id\s+{}".format(self.id) @@ -1030,7 +1030,7 @@ def local_osds(self): """ osds = [] - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') sd = SequenceSearchDef(start=SearchDef(r"^=+\s+osd\.(\d+)\s+=+.*"), body=SearchDef([r"\s+osd\s+(fsid)\s+(\S+)\s*", r"\s+(devices)\s+([\S]+)\s*"]), diff --git a/hotsos/core/plugins/system/system.py b/hotsos/core/plugins/system/system.py index 4090ae9be..45978a1a3 100644 --- a/hotsos/core/plugins/system/system.py +++ b/hotsos/core/plugins/system/system.py @@ -167,7 +167,7 @@ def ubuntu_pro_status(self): # human-readable output for now. This function should ideally # rely on the json output when the upstream sos starts to # include it. - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') service_status_seqdef = SequenceSearchDef( start=SearchDef(r"^SERVICE +ENTITLED +STATUS +DESCRIPTION"), body=SearchDef(r"^(\S+) +(\S+) +(\S+) +(.+)\n"), diff --git a/hotsos/core/ycheck/events.py b/hotsos/core/ycheck/events.py index 95c19a3d4..762ffc0a8 100644 --- a/hotsos/core/ycheck/events.py +++ b/hotsos/core/ycheck/events.py @@ -317,7 +317,8 @@ def get_global_searcher(self, allow_create=False): self._custom_searcher = None constraint = SearchConstraintSearchSince( ts_matcher_cls=CommonTimestampMatcher) - searcher = FileSearcher(constraint=constraint) + searcher = FileSearcher(constraint=constraint, + decode_errors='backslashreplace') self._global_searcher = searcher self._global_searcher_results = None log.debug("creating new global searcher (%s)", searcher) diff --git a/hotsos/core/ycheck/scenarios.py b/hotsos/core/ycheck/scenarios.py index 3dc8c5499..c1ccf7098 100644 --- a/hotsos/core/ycheck/scenarios.py +++ b/hotsos/core/ycheck/scenarios.py @@ -43,7 +43,8 @@ def __init__(self, *args, **kwargs): constraint = SearchConstraintSearchSince( ts_matcher_cls=CommonTimestampMatcher, hours=hours) - self._searcher = FileSearcher(constraint=constraint) + self._searcher = FileSearcher(constraint=constraint, + decode_errors='backslashreplace') @property def searcher(self): diff --git a/hotsos/plugin_extensions/juju/summary.py b/hotsos/plugin_extensions/juju/summary.py index 1dabc15ee..a4b11acbf 100644 --- a/hotsos/plugin_extensions/juju/summary.py +++ b/hotsos/plugin_extensions/juju/summary.py @@ -23,7 +23,8 @@ class UnitLogInfo(object): def error_and_warnings(self): log.debug("searching unit logs for errors and warnings") c = SearchConstraintSearchSince(ts_matcher_cls=CommonTimestampMatcher) - searchobj = FileSearcher(constraint=c) + searchobj = FileSearcher(constraint=c, + decode_errors='backslashreplace') path = os.path.join(HotSOSConfig.data_root, 'var/log/juju/unit-*.log') ts_expr = r"^([\d-]+)\s+([\d:]+)" for msg in ['ERROR', 'WARNING']: diff --git a/hotsos/plugin_extensions/openstack/agent/events.py b/hotsos/plugin_extensions/openstack/agent/events.py index 53eccb51a..052b9ef8f 100644 --- a/hotsos/plugin_extensions/openstack/agent/events.py +++ b/hotsos/plugin_extensions/openstack/agent/events.py @@ -375,7 +375,8 @@ def _run_checks(self, checks): return searcher = FileSearcher(constraint=SearchConstraintSearchSince( - ts_matcher_cls=CommonTimestampMatcher)) + ts_matcher_cls=CommonTimestampMatcher), + decode_errors='backslashreplace') check_objs = [c(searcher=searcher) for c in checks] results = searcher.run() _final_results = {} diff --git a/hotsos/plugin_extensions/openstack/agent/exceptions.py b/hotsos/plugin_extensions/openstack/agent/exceptions.py index 2ebc153a5..3f54c6457 100644 --- a/hotsos/plugin_extensions/openstack/agent/exceptions.py +++ b/hotsos/plugin_extensions/openstack/agent/exceptions.py @@ -116,7 +116,8 @@ def __init__(self): self._agent_results = None c = SearchConstraintSearchSince( ts_matcher_cls=CommonTimestampMatcher) - self.searchobj = FileSearcher(constraint=c) + self.searchobj = FileSearcher(constraint=c, + decode_errors='backslashreplace') # The following are expected to be logged using WARNING log level. self._agent_warnings = { 'nova': ['MessagingTimeout', diff --git a/hotsos/plugin_extensions/openstack/nova_external_events.py b/hotsos/plugin_extensions/openstack/nova_external_events.py index a21685c7b..7ee98b2e7 100644 --- a/hotsos/plugin_extensions/openstack/nova_external_events.py +++ b/hotsos/plugin_extensions/openstack/nova_external_events.py @@ -33,7 +33,7 @@ def __call__(self, event): events_found = {} c = SearchConstraintSearchSince(ts_matcher_cls=CommonTimestampMatcher) - s = FileSearcher(constraint=c) + s = FileSearcher(constraint=c, decode_errors='backslashreplace') for result in event.results: instance_id = result.get(1) event_id = result.get(2) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index e37b1673d..0faded7df 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -66,7 +66,7 @@ def test_ordered_complete(self): expected = {'0': {'duration': 60.0, 'start': start0, 'end': end0}, '1': {'duration': 60.0, 'start': start1, 'end': end1}} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start' s.add(SearchDef(expr, tag="eventX-start"), path=fname) expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end' @@ -90,7 +90,7 @@ def test_unordered_complete(self): expected = {'0': {'duration': 60.0, 'start': start0, 'end': end0}, '1': {'duration': 60.0, 'start': start1, 'end': end1}} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start' s.add(SearchDef(expr, tag="eventX-start"), path=fname) expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end' @@ -114,7 +114,7 @@ def test_ordered_complete_clobbered(self): expected = {'0': {'duration': 60.0, 'start': start0, 'end': end0}, '1': {'duration': 60.0, 'start': start1, 'end': end1}} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start' s.add(SearchDef(expr, tag="eventX-start"), path=fname) expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end' @@ -141,7 +141,7 @@ def test_ordered_incomplete_clobbered(self): expected = {'0': {'duration': 60.0, 'start': start0, 'end': end0}, '1': {'duration': 60.0, 'start': start1, 'end': end1}} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start' s.add(SearchDef(expr, tag="eventX-start"), path=fname) expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end' @@ -169,7 +169,7 @@ def test_ordered_incomplete_clobbered2(self): expected = {'0': {'duration': 120.0, 'start': start0, 'end': end0}, '1': {'duration': 60.0, 'start': start1, 'end': end1}} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start' s.add(SearchDef(expr, tag="eventX-start"), path=fname) expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end' @@ -193,7 +193,7 @@ def test_ordered_multiple(self): expected = {'0': {'duration': 60.0, 'start': start0, 'end': end0}, '1': {'duration': 60.0, 'start': start1, 'end': end1}} - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start' s.add(SearchDef(expr, tag="eventX-start"), path=fname) expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end' diff --git a/tests/unit/test_ycheck_scenarios.py b/tests/unit/test_ycheck_scenarios.py index b85f02185..677aa5e45 100644 --- a/tests/unit/test_ycheck_scenarios.py +++ b/tests/unit/test_ycheck_scenarios.py @@ -739,7 +739,7 @@ def _create_search_results(self, path, contents=None): for line in contents: fd.write(line) - s = FileSearcher() + s = FileSearcher(decode_errors='backslashreplace') s.add(SearchDef(r'^(\S+) (\S+) .+', tag='all'), path) return s.run().find_by_tag('all')