From fe13a81412dbf977a4f0354ff70f6e68af71c56c Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sat, 7 Oct 2023 01:26:38 -0700 Subject: [PATCH] prepare journald testing --- e2e_tests/e2e_test.go | 43 +++++++-- e2e_tests/testdata/test_values.yaml.tmpl | 3 + test/k8s_logging_tests/test_config_logging.py | 91 ------------------- 3 files changed, 40 insertions(+), 97 deletions(-) diff --git a/e2e_tests/e2e_test.go b/e2e_tests/e2e_test.go index 8da2b3673d..dfd04cf39a 100644 --- a/e2e_tests/e2e_test.go +++ b/e2e_tests/e2e_test.go @@ -321,11 +321,18 @@ func testAgentLogs(t *testing.T, ) { var podAnnoLogRecord *plog.LogRecord var nsAnnoResource pcommon.Resource var nsAnnoLogRecord *plog.LogRecord + //var journaldResource pcommon.Resource + //var journaldLogRecord *plog.LogRecord + var sourcetypes []string for i := 0; i < len(logsConsumer.AllLogs()); i++ { l := logsConsumer.AllLogs()[i] for j := 0; j < l.ResourceLogs().Len(); j++ { rl := l.ResourceLogs().At(j) + if value, ok := rl.Resource().Attributes().Get("com.splunk.sourcetype"); ok { + sourcetypes = append(sourcetypes, value.AsString()) + } + for k := 0; k < rl.ScopeLogs().Len(); k++ { sl := rl.ScopeLogs().At(k) for m := 0; m < sl.LogRecords().Len(); m++ { @@ -344,6 +351,12 @@ func testAgentLogs(t *testing.T, ) { nsAnnoResource = rl.Resource() } } + //if value, ok := rl.Resource().Attributes().Get("com.splunk.sourcetype"); ok { + // if strings.Contains(value.AsString(), "journald") { + // journaldLogRecord = &logRecord + // journaldResource = rl.Resource() + // } + //} } } } @@ -368,18 +381,36 @@ func testAgentLogs(t *testing.T, ) { sourceType, ok := podAnnoResource.Attributes().Get("com.splunk.sourcetype") assert.True(t, ok) assert.Equal(t, "kube:container:pod-w-index-wo-ns-index", sourceType.AsString()) - tag, ok := podAnnoLogRecord.Attributes().Get("k8s.pod.labels.app") - assert.True(t, ok) - assert.Equal(t, "pod-w-index-wo-ns-index", tag.AsString()) } { assert.NotNil(t, nsAnnoLogRecord) sourceType, ok := nsAnnoResource.Attributes().Get("com.splunk.sourcetype") assert.True(t, ok) assert.Equal(t, "kube:container:pod-wo-index-w-ns-index", sourceType.AsString()) - tag, ok := nsAnnoLogRecord.Attributes().Get("k8s.pod.labels.app") - assert.True(t, ok) - assert.Equal(t, "pod-w-index-wo-ns-index", tag.AsString()) + } + { + // journald testing fails with kind. + //assert.NotNil(t, journaldLogRecord) + //_, ok := journaldResource.Attributes().Get("com.splunk.host") + //assert.True(t, ok) + + //foundContainerdService := false + //foundDockerService := false + //foundKubeletService := false + // + //for _, sourcetype := range sourcetypes { + // switch sourcetype { + // case "kube:journald:containerd.service": + // foundContainerdService = true + // case "kube:journald:docker.service": + // foundDockerService = true + // case "kube:journald:kubelet.service": + // foundKubeletService = true + // } + //} + //assert.True(t, foundContainerdService) + //assert.True(t, foundDockerService) + //assert.True(t, foundKubeletService) } } diff --git a/e2e_tests/testdata/test_values.yaml.tmpl b/e2e_tests/testdata/test_values.yaml.tmpl index 8341d1cc45..f23728148e 100644 --- a/e2e_tests/testdata/test_values.yaml.tmpl +++ b/e2e_tests/testdata/test_values.yaml.tmpl @@ -13,6 +13,9 @@ splunkPlatform: endpoint: {{ .LogHecEndpoint }} metricsEnabled: true metricsIndex: myMetricsIndex +logsCollection: + journald: + enabled: true agent: config: exporters: diff --git a/test/k8s_logging_tests/test_config_logging.py b/test/k8s_logging_tests/test_config_logging.py index fe5ce5b091..dd2be01f98 100644 --- a/test/k8s_logging_tests/test_config_logging.py +++ b/test/k8s_logging_tests/test_config_logging.py @@ -108,52 +108,6 @@ def test_annotation_excluding(setup, container_name, expected): len(events)) assert len(events) == expected -@pytest.mark.parametrize("test_input,expected", [ - ("kube:container:kube-apiserver", 1), - ("kube:container:etcd", 1), - ("kube:container:kube-controller-manager", 1), - ("empty_sourcetype", 0) -]) -def test_sourcetype(setup, test_input, expected): - ''' - Test that known sourcetypes are present in target index - ''' - logger.info("testing for presence of sourcetype={0} expected={1} event(s)".format( - test_input, expected)) - index_logging = os.environ["CI_INDEX_EVENTS"] if os.environ["CI_INDEX_EVENTS"] else "ci_events" - source_type = ' sourcetype=""' if test_input == "empty_sourcetype" else ' sourcetype=' + test_input - search_query = "index=" + index_logging + source_type - events = check_events_from_splunk(start_time="-24h@h", - url=setup["splunkd_url"], - user=setup["splunk_user"], - query=["search {0}".format( - search_query)], - password=setup["splunk_password"]) - logger.info("Splunk received %s events in the last minute", - len(events)) - assert len(events) >= expected if test_input != "empty_sourcetype" else len( - events) == expected - -@pytest.mark.parametrize("sourcetype,index,expected", [ - ("sourcetype-anno", "pod-anno", 1) -]) -def test_annotation_sourcetype(setup, sourcetype, index, expected): - ''' - Test annotation for sourcetype properly overwrites it when set - ''' - logger.info("testing for annotation sourcetype of {0} index={1} expected={2} event(s)".format( - sourcetype, index, expected)) - search_query = "index=" + index + ' sourcetype=' + sourcetype - events = check_events_from_splunk(start_time="-1h@h", - url=setup["splunkd_url"], - user=setup["splunk_user"], - query=["search {0}".format( - search_query)], - password=setup["splunk_password"]) - logger.info("Splunk received %s events in the last minute", - len(events)) - assert len(events) >= expected - @pytest.mark.skipif(True, reason="Jira: ADDON-36296") @pytest.mark.parametrize("test_input,expected", [ ("/var/log/pods/*_kube-apiserver*", 1), @@ -182,27 +136,6 @@ def test_source(setup, test_input, expected): assert len(events) >= expected if test_input != "empty_source" else len( events) == expected -@pytest.mark.parametrize("test_input,host_name,expected", [ - ("valid_host", "minikube", 1), - ("empty_host", "", 0) -]) -def test_host(setup, test_input, host_name, expected): - ''' - Test that known hosts are present in target index - ''' - logger.info("testing for presence of host={0} expected={1} event(s)".format( - test_input, expected)) - index_logging = os.environ["CI_INDEX_EVENTS"] if os.environ["CI_INDEX_EVENTS"] else "ci_events" - search_query = "index={0} host=\"{1}\"".format(index_logging, host_name) - events = check_events_from_splunk(start_time="-24h@h", - url=setup["splunkd_url"], - user=setup["splunk_user"], - query=["search {0}".format(search_query)], - password=setup["splunk_password"]) - logger.info("Splunk received %s events in the last minute", - len(events)) - assert len(events) >= expected - @pytest.mark.parametrize("test_input,expected", [ ("k8s.pod.name", 1), ("k8s.namespace.name", 1), @@ -273,30 +206,6 @@ def test_custom_metadata_fields_annotations(setup, label, index, value, expected len(events)) assert len(events) >= expected -@pytest.mark.parametrize("test_input,expected", [ - ("test_journald_data", 1) -]) -def test_journald_logs(setup, test_input, expected): - ''' - Test that user specified index can successfully index the - journald log stream from k8s. If no index is specified, default - index "ci_events" will be used. - ''' - logger.info("testing test_journald_logs input={0} expected={1} event(s)".format( - test_input, expected)) - index_logging = os.environ["CI_INDEX_EVENTS"] if os.environ["CI_INDEX_EVENTS"] else "ci_events" - search_query = "index=" + index_logging + " sourcetype=kube:journald*" - - events = check_events_from_splunk(start_time="-1h@h", - url=setup["splunkd_url"], - user=setup["splunk_user"], - query=["search {0}".format( - search_query)], - password=setup["splunk_password"]) - logger.info("Splunk received %s events in the last hour", - len(events)) - assert len(events) >= expected - @pytest.mark.parametrize("test_input,expected", [ ("containerd.service", 1), ("docker.service", 1),