From 2f0a97f65e236ef3aa66a36ec40d2fc171652ac4 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 02:56:13 -0400 Subject: [PATCH 01/31] datadog: add rest client --- .../integrations/datadogWrapper/__init__.py | 2 ++ .../datadogWrapper/api/__init__.py | 2 ++ automon/integrations/datadogWrapper/api/v1.py | 14 +++++++++++ automon/integrations/datadogWrapper/api/v2.py | 9 +++++++ automon/integrations/datadogWrapper/client.py | 25 +++++++++++++++++++ automon/integrations/datadogWrapper/config.py | 20 +++++++++++++++ .../datadogWrapper/tests/__init__.py | 0 .../datadogWrapper/tests/test_auth.py | 20 +++++++++++++++ env-example.sh | 4 +++ 9 files changed, 96 insertions(+) create mode 100644 automon/integrations/datadogWrapper/__init__.py create mode 100644 automon/integrations/datadogWrapper/api/__init__.py create mode 100644 automon/integrations/datadogWrapper/api/v1.py create mode 100644 automon/integrations/datadogWrapper/api/v2.py create mode 100644 automon/integrations/datadogWrapper/client.py create mode 100644 automon/integrations/datadogWrapper/config.py create mode 100644 automon/integrations/datadogWrapper/tests/__init__.py create mode 100644 automon/integrations/datadogWrapper/tests/test_auth.py diff --git a/automon/integrations/datadogWrapper/__init__.py b/automon/integrations/datadogWrapper/__init__.py new file mode 100644 index 00000000..e0eec9fe --- /dev/null +++ b/automon/integrations/datadogWrapper/__init__.py @@ -0,0 +1,2 @@ +from .client import DatadogClientRest +from .config import DatadogConfigRest diff --git a/automon/integrations/datadogWrapper/api/__init__.py b/automon/integrations/datadogWrapper/api/__init__.py new file mode 100644 index 00000000..04c187c2 --- /dev/null +++ b/automon/integrations/datadogWrapper/api/__init__.py @@ -0,0 +1,2 @@ +from .v1 import V1 +from .v2 import V2 diff --git a/automon/integrations/datadogWrapper/api/v1.py b/automon/integrations/datadogWrapper/api/v1.py new file mode 100644 index 00000000..bf61327b --- /dev/null +++ b/automon/integrations/datadogWrapper/api/v1.py @@ -0,0 +1,14 @@ +class V1(object): + + def __init__(self, host): + self.endpoint = host + + @property + def api(self): + self.endpoint += '/api/v1' + return self + + @property + def validate(self): + self.endpoint += '/validate' + return self diff --git a/automon/integrations/datadogWrapper/api/v2.py b/automon/integrations/datadogWrapper/api/v2.py new file mode 100644 index 00000000..65cff586 --- /dev/null +++ b/automon/integrations/datadogWrapper/api/v2.py @@ -0,0 +1,9 @@ +class V2(object): + + def __init__(self, host): + self.endpoint = host + + @property + def api(self): + self.endpoint += '/api/v2' + return self diff --git a/automon/integrations/datadogWrapper/client.py b/automon/integrations/datadogWrapper/client.py new file mode 100644 index 00000000..071b0781 --- /dev/null +++ b/automon/integrations/datadogWrapper/client.py @@ -0,0 +1,25 @@ +from automon.integrations.requestsWrapper import RequestsClient + +from .config import DatadogConfigRest +from .api import V1, V2 + + +class DatadogClientRest(object): + + def __init__(self, host: str = None, api_key: str = None): + self.config = DatadogConfigRest(host=host, api_key=api_key) + self.requests = RequestsClient() + + async def is_ready(self): + if await self.config.is_ready(): + if await self.validate(): + return True + + async def validate(self): + url = V1(self.config.host).api.validate.endpoint + + self.requests.session.headers.update(await self.config.headers()) + response = await self.requests.get(url=url, headers=await self.config.headers()) + validate = await self.requests.to_dict() + + return validate diff --git a/automon/integrations/datadogWrapper/config.py b/automon/integrations/datadogWrapper/config.py new file mode 100644 index 00000000..5ead4234 --- /dev/null +++ b/automon/integrations/datadogWrapper/config.py @@ -0,0 +1,20 @@ +from automon import environ + + +class DatadogConfigRest(object): + api_key: str + + def __init__(self, host: str = 'https://api.datadoghq.com', api_key: str = None): + self.host = host or environ('DD_SITE') + self.api_key = api_key or environ('DD_API_KEY') + + async def is_ready(self): + if self.host and self.api_key: + return True + + async def headers(self): + if await self.is_ready(): + return { + 'DD-API-KEY': f'{self.api_key}', + 'Accept': 'application/json', + } diff --git a/automon/integrations/datadogWrapper/tests/__init__.py b/automon/integrations/datadogWrapper/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/automon/integrations/datadogWrapper/tests/test_auth.py b/automon/integrations/datadogWrapper/tests/test_auth.py new file mode 100644 index 00000000..6f0b53ea --- /dev/null +++ b/automon/integrations/datadogWrapper/tests/test_auth.py @@ -0,0 +1,20 @@ +import asyncio +import unittest + +from automon.integrations.datadogWrapper import DatadogClientRest + + +class MyTestCase(unittest.TestCase): + client = DatadogClientRest() + + if asyncio.run(client.is_ready()): + def test_auth(self): + self.assertTrue(asyncio.run( + self.client.validate() + )) + + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/env-example.sh b/env-example.sh index b5ea64cb..c0d5620f 100644 --- a/env-example.sh +++ b/env-example.sh @@ -1,6 +1,10 @@ # Codecov CODECOV_TOKEN= +# Datadog +DD_SITE=https://us5.datadoghq.com +DD_API_KEY= + # Elasticsearch ELASTICSEARCH_HOST=https://elastic-cloud.com:9200 ELASTICSEARCH_USER=elastic From b19433a71666f1a6d54fb79c459d9f7e0e65d00a Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 16:28:31 -0400 Subject: [PATCH 02/31] helpers: import networking --- automon/helpers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/automon/helpers/__init__.py b/automon/helpers/__init__.py index 4225d6a1..dc883576 100755 --- a/automon/helpers/__init__.py +++ b/automon/helpers/__init__.py @@ -1,5 +1,6 @@ from .dates import Dates from .markdown import Chat, Format +from .networking import Networking from .osWrapper import environ from .sleeper import Sleeper from .subprocessWrapper import Run From 868930d0cbae0542cce4ef5878a0b0292920b6c4 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 16:29:11 -0400 Subject: [PATCH 03/31] opentelemetry: add working in-memory tracer --- .../openTelemetryWrapper/__init__.py | 3 +- .../openTelemetryWrapper/client.py | 28 +++++++++++++ .../openTelemetryWrapper/config.py | 39 +++++++++++++++++++ .../openTelemetryWrapper/test/__init__.py | 0 .../test/test_client_ready.py | 17 ++++++++ .../test/test_memory_trace.py | 17 ++++++++ env-example.sh | 1 + 7 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 automon/integrations/openTelemetryWrapper/client.py create mode 100644 automon/integrations/openTelemetryWrapper/test/__init__.py create mode 100644 automon/integrations/openTelemetryWrapper/test/test_client_ready.py create mode 100644 automon/integrations/openTelemetryWrapper/test/test_memory_trace.py diff --git a/automon/integrations/openTelemetryWrapper/__init__.py b/automon/integrations/openTelemetryWrapper/__init__.py index 27c9ec62..be75a734 100644 --- a/automon/integrations/openTelemetryWrapper/__init__.py +++ b/automon/integrations/openTelemetryWrapper/__init__.py @@ -1 +1,2 @@ -from .config import * +from .client import OpenTelemetryClient +from .config import OpenTelemetryConfig diff --git a/automon/integrations/openTelemetryWrapper/client.py b/automon/integrations/openTelemetryWrapper/client.py new file mode 100644 index 00000000..81171f23 --- /dev/null +++ b/automon/integrations/openTelemetryWrapper/client.py @@ -0,0 +1,28 @@ +import json + +from automon.log import logging + +from .config import OpenTelemetryConfig + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class OpenTelemetryClient(object): + + def __init__(self): + self.config = OpenTelemetryConfig() + + async def clear(self): + return await self.config.clear() + + async def is_ready(self): + if await self.config.is_ready(): + return True + + async def get_finished_spans(self): + return await self.config.get_finished_spans() + + async def to_dict(self): + for span in await self.get_finished_spans(): + yield json.loads(span.to_json()) diff --git a/automon/integrations/openTelemetryWrapper/config.py b/automon/integrations/openTelemetryWrapper/config.py index e69de29b..85e940f7 100644 --- a/automon/integrations/openTelemetryWrapper/config.py +++ b/automon/integrations/openTelemetryWrapper/config.py @@ -0,0 +1,39 @@ +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from automon.log import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class OpenTelemetryConfig(object): + def __init__(self): + self.provider = TracerProvider() + self.memory_processor = InMemorySpanExporter() + self.processor = SimpleSpanProcessor(self.memory_processor) + self.provider.add_span_processor(self.processor) + + trace.set_tracer_provider(self.provider) + + self.tracer = trace.get_tracer(__name__) + + async def clear(self): + return self.memory_processor.clear() + + async def is_ready(self): + if self.provider and self.memory_processor and self.processor: + return True + + async def get_finished_spans(self): + return self.memory_processor.get_finished_spans() + + async def test(self): + with self.tracer.start_as_current_span("rootSpan"): + with self.tracer.start_as_current_span("childSpan"): + print("Hello world!") + + assert len(await self.get_finished_spans()) == 2 + return True diff --git a/automon/integrations/openTelemetryWrapper/test/__init__.py b/automon/integrations/openTelemetryWrapper/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/automon/integrations/openTelemetryWrapper/test/test_client_ready.py b/automon/integrations/openTelemetryWrapper/test/test_client_ready.py new file mode 100644 index 00000000..71f9ec65 --- /dev/null +++ b/automon/integrations/openTelemetryWrapper/test/test_client_ready.py @@ -0,0 +1,17 @@ +import unittest +import asyncio + +from automon.integrations.openTelemetryWrapper import OpenTelemetryClient + + +class MyTestCase(unittest.TestCase): + client = OpenTelemetryClient() + + def test_client(self): + self.assertTrue(asyncio.run( + self.client.is_ready() + )) + + +if __name__ == '__main__': + unittest.main() diff --git a/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py b/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py new file mode 100644 index 00000000..f8c28c73 --- /dev/null +++ b/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py @@ -0,0 +1,17 @@ +import unittest +import asyncio + +from automon.integrations.openTelemetryWrapper import OpenTelemetryConfig + + +class MyTestCase(unittest.TestCase): + config = OpenTelemetryConfig() + + def test_something(self): + self.assertTrue(asyncio.run( + self.config.test() + )) + + +if __name__ == '__main__': + unittest.main() diff --git a/env-example.sh b/env-example.sh index c0d5620f..392ee8c6 100644 --- a/env-example.sh +++ b/env-example.sh @@ -4,6 +4,7 @@ CODECOV_TOKEN= # Datadog DD_SITE=https://us5.datadoghq.com DD_API_KEY= +DD_APP_KEY= # Elasticsearch ELASTICSEARCH_HOST=https://elastic-cloud.com:9200 From 3720b8f200f3148e40e38199202025074892aac4 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 16:30:17 -0400 Subject: [PATCH 04/31] datadog: add client --- automon/integrations/datadogWrapper/api/v2.py | 8 ++++++++ automon/integrations/datadogWrapper/client.py | 16 +++++++++++++--- automon/integrations/datadogWrapper/config.py | 12 ++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/automon/integrations/datadogWrapper/api/v2.py b/automon/integrations/datadogWrapper/api/v2.py index 65cff586..4ed0109e 100644 --- a/automon/integrations/datadogWrapper/api/v2.py +++ b/automon/integrations/datadogWrapper/api/v2.py @@ -1,3 +1,6 @@ +from automon import Networking + + class V2(object): def __init__(self, host): @@ -7,3 +10,8 @@ def __init__(self, host): def api(self): self.endpoint += '/api/v2' return self + + @property + def logs(self): + self.endpoint += '/logs' + return self diff --git a/automon/integrations/datadogWrapper/client.py b/automon/integrations/datadogWrapper/client.py index 071b0781..af7377a7 100644 --- a/automon/integrations/datadogWrapper/client.py +++ b/automon/integrations/datadogWrapper/client.py @@ -1,3 +1,5 @@ +import json + from automon.integrations.requestsWrapper import RequestsClient from .config import DatadogConfigRest @@ -15,11 +17,19 @@ async def is_ready(self): if await self.validate(): return True + async def log(self, log: dict): + url = V2(self.config.host_log).api.logs.endpoint + + response = await self.requests.post(url=url, json=log) + response_log = await self.requests.to_dict() + + return response_log + async def validate(self): url = V1(self.config.host).api.validate.endpoint self.requests.session.headers.update(await self.config.headers()) - response = await self.requests.get(url=url, headers=await self.config.headers()) - validate = await self.requests.to_dict() + response = await self.requests.get(url=url) + response_validate = await self.requests.to_dict() - return validate + return response_validate diff --git a/automon/integrations/datadogWrapper/config.py b/automon/integrations/datadogWrapper/config.py index 5ead4234..121e9d89 100644 --- a/automon/integrations/datadogWrapper/config.py +++ b/automon/integrations/datadogWrapper/config.py @@ -1,20 +1,28 @@ from automon import environ +from automon.log import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class DatadogConfigRest(object): api_key: str - def __init__(self, host: str = 'https://api.datadoghq.com', api_key: str = None): + def __init__(self, host: str = 'https://api.datadoghq.com', api_key: str = None, app_key: str = None): self.host = host or environ('DD_SITE') + self.host_log = 'https://http-intake.logs.us5.datadoghq.com' self.api_key = api_key or environ('DD_API_KEY') + self.app_key = app_key or environ('DD_APP_KEY') async def is_ready(self): - if self.host and self.api_key: + if self.host and self.api_key and self.app_key: return True async def headers(self): if await self.is_ready(): return { 'DD-API-KEY': f'{self.api_key}', + 'DD-APPLICATION-KEY': f'{self.app_key}', 'Accept': 'application/json', + 'Content-Type': 'application/json', } From 5d1924e32dac44ff78bfe9a413e5f63fb56e2fc0 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 17:53:50 -0400 Subject: [PATCH 05/31] opentelemetry: fix to_dict. add to_datadog. --- .../openTelemetryWrapper/client.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/automon/integrations/openTelemetryWrapper/client.py b/automon/integrations/openTelemetryWrapper/client.py index 81171f23..5be560be 100644 --- a/automon/integrations/openTelemetryWrapper/client.py +++ b/automon/integrations/openTelemetryWrapper/client.py @@ -24,5 +24,26 @@ async def get_finished_spans(self): return await self.config.get_finished_spans() async def to_dict(self): - for span in await self.get_finished_spans(): - yield json.loads(span.to_json()) + return [ + json.loads(span.to_json()) + for span in await self.get_finished_spans() + ] + + async def to_datadog(self): + log = [] + for span in await self.to_dict(): + message = dict(span).copy() + ddsource = None + ddtags = ','.join([f"{x[0]}:{x[1]}" for x in span.get("resource").get("attributes").items()]) + hostname = span['context']['trace_id'] + service = span['context']['span_id'] + + span['datadog'] = dict( + ddsource=ddsource, + ddtags=ddtags, + hostname=hostname, + service=service, + message=message, + ) + log.append(span) + return log From 61a6d398435ffa970e95b976dce43a2e29eef939 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 17:54:28 -0400 Subject: [PATCH 06/31] datadog: fix log --- automon/integrations/datadogWrapper/client.py | 10 +++++- .../datadogWrapper/tests/test_log.py | 31 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 automon/integrations/datadogWrapper/tests/test_log.py diff --git a/automon/integrations/datadogWrapper/client.py b/automon/integrations/datadogWrapper/client.py index af7377a7..27e0a787 100644 --- a/automon/integrations/datadogWrapper/client.py +++ b/automon/integrations/datadogWrapper/client.py @@ -17,9 +17,17 @@ async def is_ready(self): if await self.validate(): return True - async def log(self, log: dict): + async def log(self, ddsource: str, hostname: str, service: str, message: str, ddtags: str = 'env:test,version:0.1'): url = V2(self.config.host_log).api.logs.endpoint + log = { + "ddsource": ddsource, + "ddtags": ddtags, + "hostname": hostname, + "service": service, + 'message': message + } + response = await self.requests.post(url=url, json=log) response_log = await self.requests.to_dict() diff --git a/automon/integrations/datadogWrapper/tests/test_log.py b/automon/integrations/datadogWrapper/tests/test_log.py new file mode 100644 index 00000000..4cd0d6e7 --- /dev/null +++ b/automon/integrations/datadogWrapper/tests/test_log.py @@ -0,0 +1,31 @@ +import asyncio +import unittest + +from automon.integrations.datadogWrapper import DatadogClientRest +from automon.integrations.openTelemetryWrapper import OpenTelemetryClient + + +class MyTestCase(unittest.TestCase): + client = DatadogClientRest() + client_tracer = OpenTelemetryClient() + + def test_log(self): + if asyncio.run(self.client.is_ready()): + + asyncio.run(self.client_tracer.config.test()) + + spans = asyncio.run(self.client_tracer.to_datadog()) + for span in spans: + asyncio.run( + self.client.log( + ddsource=span['datadog']['ddsource'], + ddtags=span['datadog']['ddtags'], + hostname=span['datadog']['hostname'], + service=span['datadog']['service'], + message=span['datadog']['message'], + )) + + pass + + if __name__ == '__main__': + unittest.main() From 43ffffaf6532d40904479e4c5cd31d1cb40b811a Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 17:57:19 -0400 Subject: [PATCH 07/31] datadog: update logging --- automon/integrations/datadogWrapper/client.py | 9 +++++++-- automon/integrations/datadogWrapper/config.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/automon/integrations/datadogWrapper/client.py b/automon/integrations/datadogWrapper/client.py index 27e0a787..3594825e 100644 --- a/automon/integrations/datadogWrapper/client.py +++ b/automon/integrations/datadogWrapper/client.py @@ -1,10 +1,12 @@ -import json - from automon.integrations.requestsWrapper import RequestsClient +from automon.log import logging from .config import DatadogConfigRest from .api import V1, V2 +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + class DatadogClientRest(object): @@ -16,6 +18,7 @@ async def is_ready(self): if await self.config.is_ready(): if await self.validate(): return True + logger.error(f'client not ready') async def log(self, ddsource: str, hostname: str, service: str, message: str, ddtags: str = 'env:test,version:0.1'): url = V2(self.config.host_log).api.logs.endpoint @@ -28,6 +31,8 @@ async def log(self, ddsource: str, hostname: str, service: str, message: str, dd 'message': message } + logger.debug(log) + response = await self.requests.post(url=url, json=log) response_log = await self.requests.to_dict() diff --git a/automon/integrations/datadogWrapper/config.py b/automon/integrations/datadogWrapper/config.py index 121e9d89..1111f6d8 100644 --- a/automon/integrations/datadogWrapper/config.py +++ b/automon/integrations/datadogWrapper/config.py @@ -17,6 +17,7 @@ def __init__(self, host: str = 'https://api.datadoghq.com', api_key: str = None, async def is_ready(self): if self.host and self.api_key and self.app_key: return True + logger.error(f'missing DD_SITE DD_API_KEY DD_APP_KEY') async def headers(self): if await self.is_ready(): From 80f5bb896e019cbc010a071359590f875f134d66 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 17:59:09 -0400 Subject: [PATCH 08/31] automon: fix test.sh --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index faa7ad6f..fef7de9a 100755 --- a/test.sh +++ b/test.sh @@ -4,7 +4,7 @@ cd $(dirname $0) && set -e -if [ "$@" == "" ]; then +if [ ! -z "$@" ]; then set -x python3 -m pytest automon else From 68f1f89dce01c7a29eb1125f432ff764108ada95 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 26 Mar 2024 18:05:53 -0400 Subject: [PATCH 09/31] opentelemetry: fix test --- automon/integrations/openTelemetryWrapper/config.py | 1 - .../openTelemetryWrapper/test/test_memory_trace.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/automon/integrations/openTelemetryWrapper/config.py b/automon/integrations/openTelemetryWrapper/config.py index 85e940f7..07a33e30 100644 --- a/automon/integrations/openTelemetryWrapper/config.py +++ b/automon/integrations/openTelemetryWrapper/config.py @@ -35,5 +35,4 @@ async def test(self): with self.tracer.start_as_current_span("childSpan"): print("Hello world!") - assert len(await self.get_finished_spans()) == 2 return True diff --git a/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py b/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py index f8c28c73..ea70c542 100644 --- a/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py +++ b/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py @@ -9,8 +9,8 @@ class MyTestCase(unittest.TestCase): def test_something(self): self.assertTrue(asyncio.run( - self.config.test() - )) + self.config.test()) + ) if __name__ == '__main__': From 7e5dadb6c2971060c1e0053aa1ec314114183636 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:13:09 -0400 Subject: [PATCH 10/31] opentelemetry: add pop_finished_spans --- .../openTelemetryWrapper/config.py | 6 +++++ .../test/test_pop_finished_spans.py | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 automon/integrations/openTelemetryWrapper/test/test_pop_finished_spans.py diff --git a/automon/integrations/openTelemetryWrapper/config.py b/automon/integrations/openTelemetryWrapper/config.py index 07a33e30..92e11bff 100644 --- a/automon/integrations/openTelemetryWrapper/config.py +++ b/automon/integrations/openTelemetryWrapper/config.py @@ -30,6 +30,12 @@ async def is_ready(self): async def get_finished_spans(self): return self.memory_processor.get_finished_spans() + async def pop_finished_spans(self): + """ideal is to lock, pop spans, and clear""" + spans = await self.get_finished_spans() + clear = await self.clear() + return spans + async def test(self): with self.tracer.start_as_current_span("rootSpan"): with self.tracer.start_as_current_span("childSpan"): diff --git a/automon/integrations/openTelemetryWrapper/test/test_pop_finished_spans.py b/automon/integrations/openTelemetryWrapper/test/test_pop_finished_spans.py new file mode 100644 index 00000000..6a6d8642 --- /dev/null +++ b/automon/integrations/openTelemetryWrapper/test/test_pop_finished_spans.py @@ -0,0 +1,23 @@ +import unittest +import asyncio + +from automon.integrations.openTelemetryWrapper import OpenTelemetryConfig + + +class MyTestCase(unittest.TestCase): + config = OpenTelemetryConfig() + + if asyncio.run(config.is_ready()): + test = asyncio.run(config.test()) + + def test_pop_finished_spans(self): + spans = asyncio.run(self.config.pop_finished_spans()) + self.assertIsNotNone(spans) + + pass + + pass + + +if __name__ == '__main__': + unittest.main() From 2200bf6be6d2c3175092c8203abb0808aa3fa001 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:13:41 -0400 Subject: [PATCH 11/31] opentelemetry: add consumer and producer --- automon/integrations/openTelemetryWrapper/client.py | 12 ++++++++++++ automon/integrations/openTelemetryWrapper/config.py | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/automon/integrations/openTelemetryWrapper/client.py b/automon/integrations/openTelemetryWrapper/client.py index 5be560be..8528ceda 100644 --- a/automon/integrations/openTelemetryWrapper/client.py +++ b/automon/integrations/openTelemetryWrapper/client.py @@ -23,6 +23,18 @@ async def is_ready(self): async def get_finished_spans(self): return await self.config.get_finished_spans() + async def start_consumer(self): + """adds spans from memory to queue""" + while True: + pass + return + + async def start_producer(self): + """""" + while True: + pass + return + async def to_dict(self): return [ json.loads(span.to_json()) diff --git a/automon/integrations/openTelemetryWrapper/config.py b/automon/integrations/openTelemetryWrapper/config.py index 92e11bff..1126e16e 100644 --- a/automon/integrations/openTelemetryWrapper/config.py +++ b/automon/integrations/openTelemetryWrapper/config.py @@ -1,3 +1,5 @@ +import asyncio + from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor @@ -20,6 +22,9 @@ def __init__(self): self.tracer = trace.get_tracer(__name__) + self.queue_consumer = asyncio.Queue() + self.queue_producer = asyncio.Queue() + async def clear(self): return self.memory_processor.clear() From 245a3706d0074217b5416682a4cec0aa4f1b9948 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:22:15 -0400 Subject: [PATCH 12/31] automon: fix readme --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index ad133e0f..3357b1ab 100644 --- a/README.md +++ b/README.md @@ -51,20 +51,6 @@ Github issues and feature requests welcomed. | SOAR | swimlane
splunk soar | | Recon | nmap | | Test Automation | selenium | -| Category | Library | -|-------------------|-------------------------------------------------------------| -| API | flask | -| Chat | slack | -| Data Scraping | beautifulsoup
facebook groups
instagram
scrapy | -| Databases | elasticsearch
neo4j
splunk | -| Data Store | minio
swift | -| Devices | snmp | -| Google Cloud | google auth api
google people api
google sheets api | -| Tracing / Logging | openTelemetry
sentryio | -| macOS | airport
macchanger | -| Python | logging
requests | -| Recon | nmap | -| Test Automation | selenium | #### Requires From 69c3b00ac26318837fbd7f18f2a903a626402e90 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:39:14 -0400 Subject: [PATCH 13/31] automon: update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3357b1ab..fea2bf23 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Github issues and feature requests welcomed. | Data Store | minio
swift | | Devices | snmp | | Google Cloud | google auth api
google people api
google sheets api | +| Helpers | os
subprocess
threading
socket
datetime | | Logging | sentryio | | MacOS | airport
macchanger | | Python | logging
requests | From e7dfa8130a3126a122dc61c6cdbadf1d47deb485 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:39:46 -0400 Subject: [PATCH 14/31] grok: move to integrations --- automon/{helpers => integrations}/grok/__init__.py | 0 .../grok/logstash-patterns-core/LICENSE | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/aws | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/bacula | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/bind | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/bro | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/exim | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/firewalls | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/grok-patterns | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/haproxy | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/httpd | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/java | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/junos | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/linux-syslog | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/maven | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/mcollective | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/mongodb | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/nagios | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/postgresql | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/rails | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/redis | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/ruby | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/squid | 0 .../grok/logstash-patterns-core/patterns/ecs-v1/zeek | 0 .../grok/logstash-patterns-core/patterns/legacy/aws | 0 .../grok/logstash-patterns-core/patterns/legacy/bacula | 0 .../grok/logstash-patterns-core/patterns/legacy/bind | 0 .../grok/logstash-patterns-core/patterns/legacy/bro | 0 .../grok/logstash-patterns-core/patterns/legacy/exim | 0 .../grok/logstash-patterns-core/patterns/legacy/firewalls | 0 .../grok/logstash-patterns-core/patterns/legacy/grok-patterns | 0 .../grok/logstash-patterns-core/patterns/legacy/haproxy | 0 .../grok/logstash-patterns-core/patterns/legacy/httpd | 0 .../grok/logstash-patterns-core/patterns/legacy/java | 0 .../grok/logstash-patterns-core/patterns/legacy/junos | 0 .../grok/logstash-patterns-core/patterns/legacy/linux-syslog | 0 .../grok/logstash-patterns-core/patterns/legacy/maven | 0 .../grok/logstash-patterns-core/patterns/legacy/mcollective | 0 .../logstash-patterns-core/patterns/legacy/mcollective-patterns | 0 .../grok/logstash-patterns-core/patterns/legacy/mongodb | 0 .../grok/logstash-patterns-core/patterns/legacy/nagios | 0 .../grok/logstash-patterns-core/patterns/legacy/postgresql | 0 .../grok/logstash-patterns-core/patterns/legacy/rails | 0 .../grok/logstash-patterns-core/patterns/legacy/redis | 0 .../grok/logstash-patterns-core/patterns/legacy/ruby | 0 .../grok/logstash-patterns-core/patterns/legacy/squid | 0 automon/integrations/grok/tests/__init__.py | 0 automon/{helpers => integrations/grok}/tests/test_grok.py | 2 +- 48 files changed, 1 insertion(+), 1 deletion(-) rename automon/{helpers => integrations}/grok/__init__.py (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/LICENSE (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/aws (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/bacula (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/bind (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/bro (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/exim (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/firewalls (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/grok-patterns (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/haproxy (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/httpd (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/java (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/junos (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/linux-syslog (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/maven (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/mcollective (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/mongodb (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/nagios (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/postgresql (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/rails (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/redis (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/ruby (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/squid (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/ecs-v1/zeek (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/aws (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/bacula (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/bind (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/bro (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/exim (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/firewalls (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/grok-patterns (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/haproxy (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/httpd (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/java (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/junos (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/linux-syslog (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/maven (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/mcollective (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/mcollective-patterns (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/mongodb (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/nagios (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/postgresql (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/rails (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/redis (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/ruby (100%) rename automon/{helpers => integrations}/grok/logstash-patterns-core/patterns/legacy/squid (100%) create mode 100644 automon/integrations/grok/tests/__init__.py rename automon/{helpers => integrations/grok}/tests/test_grok.py (78%) diff --git a/automon/helpers/grok/__init__.py b/automon/integrations/grok/__init__.py similarity index 100% rename from automon/helpers/grok/__init__.py rename to automon/integrations/grok/__init__.py diff --git a/automon/helpers/grok/logstash-patterns-core/LICENSE b/automon/integrations/grok/logstash-patterns-core/LICENSE similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/LICENSE rename to automon/integrations/grok/logstash-patterns-core/LICENSE diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/aws b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/aws similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/aws rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/aws diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/bacula b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/bacula similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/bacula rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/bacula diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/bind b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/bind similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/bind rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/bind diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/bro b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/bro similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/bro rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/bro diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/exim b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/exim similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/exim rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/exim diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/firewalls b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/firewalls similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/firewalls rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/firewalls diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/grok-patterns b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/grok-patterns similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/grok-patterns rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/grok-patterns diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/haproxy b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/haproxy similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/haproxy rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/haproxy diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/httpd b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/httpd similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/httpd rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/httpd diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/java b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/java similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/java rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/java diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/junos b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/junos similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/junos rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/junos diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/linux-syslog b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/linux-syslog similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/linux-syslog rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/linux-syslog diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/maven b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/maven similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/maven rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/maven diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/mcollective b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/mcollective similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/mcollective rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/mcollective diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/mongodb b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/mongodb similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/mongodb rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/mongodb diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/nagios b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/nagios similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/nagios rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/nagios diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/postgresql b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/postgresql similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/postgresql rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/postgresql diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/rails b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/rails similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/rails rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/rails diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/redis b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/redis similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/redis rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/redis diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/ruby b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/ruby similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/ruby rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/ruby diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/squid b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/squid similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/squid rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/squid diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/zeek b/automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/zeek similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/ecs-v1/zeek rename to automon/integrations/grok/logstash-patterns-core/patterns/ecs-v1/zeek diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/aws b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/aws similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/aws rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/aws diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/bacula b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/bacula similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/bacula rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/bacula diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/bind b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/bind similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/bind rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/bind diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/bro b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/bro similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/bro rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/bro diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/exim b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/exim similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/exim rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/exim diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/firewalls b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/firewalls similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/firewalls rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/firewalls diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/grok-patterns b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/grok-patterns similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/grok-patterns rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/grok-patterns diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/haproxy b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/haproxy similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/haproxy rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/haproxy diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/httpd b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/httpd similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/httpd rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/httpd diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/java b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/java similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/java rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/java diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/junos b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/junos similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/junos rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/junos diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/linux-syslog b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/linux-syslog similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/linux-syslog rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/linux-syslog diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/maven b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/maven similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/maven rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/maven diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/mcollective b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/mcollective similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/mcollective rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/mcollective diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/mcollective-patterns b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/mcollective-patterns similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/mcollective-patterns rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/mcollective-patterns diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/mongodb b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/mongodb similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/mongodb rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/mongodb diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/nagios b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/nagios similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/nagios rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/nagios diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/postgresql b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/postgresql similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/postgresql rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/postgresql diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/rails b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/rails similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/rails rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/rails diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/redis b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/redis similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/redis rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/redis diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/ruby b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/ruby similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/ruby rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/ruby diff --git a/automon/helpers/grok/logstash-patterns-core/patterns/legacy/squid b/automon/integrations/grok/logstash-patterns-core/patterns/legacy/squid similarity index 100% rename from automon/helpers/grok/logstash-patterns-core/patterns/legacy/squid rename to automon/integrations/grok/logstash-patterns-core/patterns/legacy/squid diff --git a/automon/integrations/grok/tests/__init__.py b/automon/integrations/grok/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/automon/helpers/tests/test_grok.py b/automon/integrations/grok/tests/test_grok.py similarity index 78% rename from automon/helpers/tests/test_grok.py rename to automon/integrations/grok/tests/test_grok.py index 7e54dab7..2186638a 100644 --- a/automon/helpers/tests/test_grok.py +++ b/automon/integrations/grok/tests/test_grok.py @@ -1,6 +1,6 @@ import unittest -from automon.helpers.grok import Grok +from automon.integrations.grok import Grok class GrokTest(unittest.TestCase): From 9bfd7bc7ef56b81ae0bb0f7519215ef923e8f114 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:39:56 -0400 Subject: [PATCH 15/31] nestasyncio: move to integrations --- .../{helpers => integrations}/nest_asyncioWrapper/__init__.py | 0 automon/{helpers => integrations}/nest_asyncioWrapper/client.py | 0 automon/integrations/slackWrapper/clientAsync.py | 2 +- automon/integrations/slackWrapper/slack_logger.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename automon/{helpers => integrations}/nest_asyncioWrapper/__init__.py (100%) rename automon/{helpers => integrations}/nest_asyncioWrapper/client.py (100%) diff --git a/automon/helpers/nest_asyncioWrapper/__init__.py b/automon/integrations/nest_asyncioWrapper/__init__.py similarity index 100% rename from automon/helpers/nest_asyncioWrapper/__init__.py rename to automon/integrations/nest_asyncioWrapper/__init__.py diff --git a/automon/helpers/nest_asyncioWrapper/client.py b/automon/integrations/nest_asyncioWrapper/client.py similarity index 100% rename from automon/helpers/nest_asyncioWrapper/client.py rename to automon/integrations/nest_asyncioWrapper/client.py diff --git a/automon/integrations/slackWrapper/clientAsync.py b/automon/integrations/slackWrapper/clientAsync.py index cdc89668..2ebf98bb 100644 --- a/automon/integrations/slackWrapper/clientAsync.py +++ b/automon/integrations/slackWrapper/clientAsync.py @@ -4,7 +4,7 @@ import asyncio from automon import log -from automon.helpers.nest_asyncioWrapper import AsyncStarter +from automon.integrations.nest_asyncioWrapper import AsyncStarter from .config import ConfigSlack from .bots import BotInfo diff --git a/automon/integrations/slackWrapper/slack_logger.py b/automon/integrations/slackWrapper/slack_logger.py index 91b5c5ff..11daf360 100644 --- a/automon/integrations/slackWrapper/slack_logger.py +++ b/automon/integrations/slackWrapper/slack_logger.py @@ -4,7 +4,7 @@ from json import dumps from asyncio import sleep -from automon.helpers.nest_asyncioWrapper import AsyncStarter +from automon.integrations.nest_asyncioWrapper import AsyncStarter from automon.integrations.slackWrapper.client import SlackClient from automon.integrations.slackWrapper.slack_formatting import Emoji, Chat, Format From 9c6d42b1a6bab9adde8c1b9d754f6715347fb3cf Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:40:10 -0400 Subject: [PATCH 16/31] psutil: move to integrations --- automon/integrations/psutilWrapper/__init__.py | 0 automon/{helpers => integrations/psutilWrapper}/cpu.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 automon/integrations/psutilWrapper/__init__.py rename automon/{helpers => integrations/psutilWrapper}/cpu.py (100%) diff --git a/automon/integrations/psutilWrapper/__init__.py b/automon/integrations/psutilWrapper/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/automon/helpers/cpu.py b/automon/integrations/psutilWrapper/cpu.py similarity index 100% rename from automon/helpers/cpu.py rename to automon/integrations/psutilWrapper/cpu.py From fbb27783e9802d7528efdfae2b0ab2e79f71e979 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 27 Mar 2024 05:42:40 -0400 Subject: [PATCH 17/31] github actions: disable python36.yml on all branches --- .github/workflows/python36.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python36.yml b/.github/workflows/python36.yml index a5a19d23..0259f4e9 100644 --- a/.github/workflows/python36.yml +++ b/.github/workflows/python36.yml @@ -2,9 +2,9 @@ name: 3.6 EOL on: push: - branches: [ '*' ] + branches: [ ] pull_request: - branches: [ '*' ] + branches: [ ] env: # Use docker.io for Docker Hub if empty From 5e017972b45dd48b851a3d1aa8350ef9292f128d Mon Sep 17 00:00:00 2001 From: laerfulaolun Date: Mon, 1 Apr 2024 16:42:51 -0700 Subject: [PATCH 18/31] requests: fix result processor --- automon/integrations/requestsWrapper/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automon/integrations/requestsWrapper/client.py b/automon/integrations/requestsWrapper/client.py index b77aa50d..f6aab814 100644 --- a/automon/integrations/requestsWrapper/client.py +++ b/automon/integrations/requestsWrapper/client.py @@ -36,7 +36,7 @@ async def _log_result(self): self.response.request.method, self.response.url, f'{round(len(self.content) / 1024, 2)} KB', - self.status_code, + f'{self.status_code}', ] msg = ' '.join(msg) return logger.debug(msg) @@ -45,8 +45,8 @@ async def _log_result(self): self.response.request.method, self.response.url, f'{round(len(self.content) / 1024, 2)} KB', - self.status_code, - self.content + f'{self.status_code}', + f'{self.content}' ] msg = ' '.join(msg) From f96f7bced4d2628a540b79b4698aa50c2767baf7 Mon Sep 17 00:00:00 2001 From: laerfulaolun Date: Mon, 1 Apr 2024 16:46:02 -0700 Subject: [PATCH 19/31] swimlane: update test --- .../swimlaneWrapper/tests/test_rest_record_delete_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automon/integrations/swimlaneWrapper/tests/test_rest_record_delete_all.py b/automon/integrations/swimlaneWrapper/tests/test_rest_record_delete_all.py index 64586ddf..ed57f209 100644 --- a/automon/integrations/swimlaneWrapper/tests/test_rest_record_delete_all.py +++ b/automon/integrations/swimlaneWrapper/tests/test_rest_record_delete_all.py @@ -7,7 +7,7 @@ class MyTestCase(unittest.TestCase): - def test_login(self): + def test_record_delete_all(self): if asyncio.run(client.is_ready()): if asyncio.run(client.login()): app_id = client.config.appId From a1affc147b002f8c40e54e36823fd46d40699a14 Mon Sep 17 00:00:00 2001 From: laerfulaolun Date: Mon, 1 Apr 2024 16:48:43 -0700 Subject: [PATCH 20/31] swimlane: fix record_delete_all --- automon/integrations/swimlaneWrapper/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/automon/integrations/swimlaneWrapper/client.py b/automon/integrations/swimlaneWrapper/client.py index 36a47d62..ec6639da 100644 --- a/automon/integrations/swimlaneWrapper/client.py +++ b/automon/integrations/swimlaneWrapper/client.py @@ -260,6 +260,9 @@ async def record_delete_all(self, appId: str): url=url ) + if self.requests.status_code == 204 or self.requests.status_code == 404: + return True + return response async def record_get(self, appId: str, id: str): From c35bad145f44e6b20ccda18a2a64067cc7432300 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 2 Apr 2024 17:30:27 -0400 Subject: [PATCH 21/31] 0.5.0 Change log: add opentelemetry add datadog add swimlane fix bugs fix imports update requests to async opentelemetry: add consumer and producer opentelemetry: add pop_finished_spans opentelemetry: fix test opentelemetry: fix to_dict. add to_datadog. opentelemetry: add working in-memory tracer opentelemetry: update requirements.txt opentelemetry: fix for 0.44b0 opentelemetry: update docker opentelemetry: update readme opentelemetry: fix python packages datadog: update logging datadog: fix log datadog: add client datadog: add rest client selenium: fix test_browser_headless.py selenium: update driver to 122.0.6261.111 selenium: fix tests selenium: fix error message selenium: raise exception if webdriver is missing before using it swimlane: add logging_by_id. add logging_recent. swimlane: add app_by_id. add app_export swimlane: add tests swimlane: add tests swimlane: fix tests swimlane: add tests swimlane: add method record_resolve_fields. rename method record_schema. fix method record_create_hard swimlane: update test swimlane: disable library test. its dependencies are so old it breaks python > 3.10 swimlane: add python package swimlane: fix record hardcoded key hash swimlane: fix test_library_record_create.py swimlane: update test_rest_record.py swimlane: update test_rest_app.py swimlane: return record after created swimlane: add test_rest_record_create.py swimlane: add test_rest_auth_token.py swimlane: fix credentials swimlane: update record_create_easy swimlane: fix login_token swimlane: v2 add user authorize endpoint swimlane: add appId to config swimlane: add get record swimlane: fix record methods swimlane: rename tests swimlane: add test_library_record_create.py swimlane: update client swimlane: add headers_jwt_token swimlane: update record methods swimlane: support api "record" swimlane: support apis "app", "workspace" swimlane: update client and config swimlane: add client and config requests: fix tests for async requests: fix rest to async requests: fix config requests: add content_to_dict requests: fix to_dict, fix to_json requests: rename property results to response requests: default to requests.Session requests: minor support for request.Session requests: add content prop, add to_json method requests: add reason method requests: update to async requests: add content_to_dict requests: fix to_dict, fix to_json requests: rename property results to response requests: default to requests.Session requests: minor support for request.Session requests: add content prop, add to_json method requests: add reason method requests: update to async automon: update readme automon: fix readme automon: fix test.sh automon: fix missing deprecation package pypi: quotes around vars pypi: quotes around vars psutil: move to integrations nestasyncio: move to integrations helpers: import networking robinhood: add status api endpoints grok: move to integrations grok: removing depreciated code --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e525a421..c3cadeed 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.4.2", + version="0.5.0", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From 62488854d4bd6286aab0fc228ac35e9a5b1cdc06 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 2 Apr 2024 21:50:50 -0400 Subject: [PATCH 22/31] opentelemetry: fix packages --- .../integrations/openTelemetryWrapper/requirements.txt | 9 ++++----- requirements.txt | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/automon/integrations/openTelemetryWrapper/requirements.txt b/automon/integrations/openTelemetryWrapper/requirements.txt index fde6ed71..a87bd892 100644 --- a/automon/integrations/openTelemetryWrapper/requirements.txt +++ b/automon/integrations/openTelemetryWrapper/requirements.txt @@ -1,6 +1,5 @@ -# opentelemetry -opentelemetry-distro>=0.44b0 -opentelemetry-exporter-otlp>=1.22.0 -opentelemetry-instrumentation-logging>=0.44b0 +opentelemetry-distro==0.43b0 +opentelemetry-exporter-otlp==1.22.0 +opentelemetry-instrumentation-logging==0.43b0 # workaround -git+https://github.com/TheShellLand/opentelemetry-python-contrib.git@0.44b0#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server +git+https://github.com/open-telemetry/opentelemetry-python-contrib.git@0.43b0hotfix#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server diff --git a/requirements.txt b/requirements.txt index 496db56b..7491225a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,11 +42,11 @@ python-keystoneclient>=4.2.0 python-swiftclient>=3.12.0 # opentelemetry -opentelemetry-distro>=0.44b0 -opentelemetry-exporter-otlp>=1.22.0 -opentelemetry-instrumentation-logging>=0.44b0 +opentelemetry-distro==0.43b0 +opentelemetry-exporter-otlp==1.22.0 +opentelemetry-instrumentation-logging==0.43b0 # workaround -git+https://github.com/TheShellLand/opentelemetry-python-contrib.git@0.44b0#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server +git+https://github.com/open-telemetry/opentelemetry-python-contrib.git@0.43b0hotfix#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server # splunk soar pytz>=2021.1 From 90b065c73636a66097e787b9c4a43a19de543bcf Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 2 Apr 2024 21:51:25 -0400 Subject: [PATCH 23/31] 0.5.1 Change log: opentelemetry: fix packages --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c3cadeed..8b61e0c5 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.5.0", + version="0.5.1", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From f2427aec2373132142fcbbe432c063cbb03e24d4 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 2 Apr 2024 21:54:30 -0400 Subject: [PATCH 24/31] github actions: disable python36.yml --- .github/workflows/python36.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python36.yml b/.github/workflows/python36.yml index 0259f4e9..f621ca70 100644 --- a/.github/workflows/python36.yml +++ b/.github/workflows/python36.yml @@ -2,9 +2,9 @@ name: 3.6 EOL on: push: - branches: [ ] + branches: [ 'none' ] pull_request: - branches: [ ] + branches: [ 'none' ] env: # Use docker.io for Docker Hub if empty From de8ca3d298daac9c2ca7237146e54a020e84b73e Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 2 Apr 2024 21:57:38 -0400 Subject: [PATCH 25/31] github actions: disable python37.yml --- .github/workflows/python37.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python37.yml b/.github/workflows/python37.yml index d7692d46..207540ec 100644 --- a/.github/workflows/python37.yml +++ b/.github/workflows/python37.yml @@ -2,9 +2,9 @@ name: 3.7 EOL on: push: - branches: [ '*' ] + branches: [ 'none' ] pull_request: - branches: [ '*' ] + branches: [ 'none' ] env: # Use docker.io for Docker Hub if empty From 57bb53c5103604ded7236476af6a4dcbe3894f95 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 2 Apr 2024 22:11:20 -0400 Subject: [PATCH 26/31] automon: require python '>=3.10' --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fea2bf23..0c9a4460 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Github issues and feature requests welcomed. #### Requires -- python >= 3.8 +- python >= 3.10 _Note: install requirements.txt to use all integrations_ diff --git a/setup.py b/setup.py index 8b61e0c5..13749c0f 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,6 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - python_requires='>=3.7', + python_requires='>=3.10', install_requires=[] ) From 2ff85696a107250c318a57984bcdc9dff86c2c08 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 3 Apr 2024 03:02:47 -0400 Subject: [PATCH 27/31] opentelemetry: add event. add exception. add contextmanager from client. update datadog --- .../datadogWrapper/tests/test_log.py | 10 +++---- .../openTelemetryWrapper/client.py | 27 ++++++++++++++++--- .../openTelemetryWrapper/config.py | 11 ++++++-- .../test/test_memory_trace_client.py | 17 ++++++++++++ ...y_trace.py => test_memory_trace_config.py} | 0 5 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py rename automon/integrations/openTelemetryWrapper/test/{test_memory_trace.py => test_memory_trace_config.py} (100%) diff --git a/automon/integrations/datadogWrapper/tests/test_log.py b/automon/integrations/datadogWrapper/tests/test_log.py index 4cd0d6e7..242a0d2e 100644 --- a/automon/integrations/datadogWrapper/tests/test_log.py +++ b/automon/integrations/datadogWrapper/tests/test_log.py @@ -18,11 +18,11 @@ def test_log(self): for span in spans: asyncio.run( self.client.log( - ddsource=span['datadog']['ddsource'], - ddtags=span['datadog']['ddtags'], - hostname=span['datadog']['hostname'], - service=span['datadog']['service'], - message=span['datadog']['message'], + ddsource=span['ddsource'], + ddtags=span['ddtags'], + hostname=span['hostname'], + service=span['service'], + message=span['message'], )) pass diff --git a/automon/integrations/openTelemetryWrapper/client.py b/automon/integrations/openTelemetryWrapper/client.py index 8528ceda..479b2bfc 100644 --- a/automon/integrations/openTelemetryWrapper/client.py +++ b/automon/integrations/openTelemetryWrapper/client.py @@ -1,5 +1,6 @@ import json +from opentelemetry.util import types from automon.log import logging from .config import OpenTelemetryConfig @@ -13,6 +14,9 @@ class OpenTelemetryClient(object): def __init__(self): self.config = OpenTelemetryConfig() + def add_event(self, name: str, attributes: types.Attributes = None, **kwargs): + return self.config.current_span.add_event(name=name, attributes=attributes, **kwargs) + async def clear(self): return await self.config.clear() @@ -23,6 +27,16 @@ async def is_ready(self): async def get_finished_spans(self): return await self.config.get_finished_spans() + async def pop_finished_spans(self): + return await self.config.pop_finished_spans() + + def record_exception(self, exception: Exception): + return self.config.current_span.record_exception(exception=exception) + + def start_as_current_span(self, name: str, attributes: types.Attributes = None, **kwargs): + return self.config.tracer.start_as_current_span( + name=name, attributes=attributes, **kwargs) + async def start_consumer(self): """adds spans from memory to queue""" while True: @@ -35,6 +49,14 @@ async def start_producer(self): pass return + async def test(self): + with self.start_as_current_span(name='rootSpan') as trace_root: + with self.start_as_current_span(name='childSpan') as trace_child: + self.add_event('AAAAAAAA') + self.add_event('BBBBBBBB') + + return True + async def to_dict(self): return [ json.loads(span.to_json()) @@ -50,12 +72,11 @@ async def to_datadog(self): hostname = span['context']['trace_id'] service = span['context']['span_id'] - span['datadog'] = dict( + log.append(dict( ddsource=ddsource, ddtags=ddtags, hostname=hostname, service=service, message=message, - ) - log.append(span) + )) return log diff --git a/automon/integrations/openTelemetryWrapper/config.py b/automon/integrations/openTelemetryWrapper/config.py index 1126e16e..99462265 100644 --- a/automon/integrations/openTelemetryWrapper/config.py +++ b/automon/integrations/openTelemetryWrapper/config.py @@ -1,6 +1,7 @@ import asyncio from opentelemetry import trace +from opentelemetry.trace import Status, StatusCode from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -28,6 +29,10 @@ def __init__(self): async def clear(self): return self.memory_processor.clear() + @property + def current_span(self): + return trace.get_current_span() + async def is_ready(self): if self.provider and self.memory_processor and self.processor: return True @@ -42,8 +47,10 @@ async def pop_finished_spans(self): return spans async def test(self): - with self.tracer.start_as_current_span("rootSpan"): - with self.tracer.start_as_current_span("childSpan"): + with self.tracer.start_as_current_span(name="rootSpan") as trace_root: + with self.tracer.start_as_current_span(name="childSpan") as trace_child: + trace_child.add_event('AAAAAAAA') + trace_child.add_event('BBBBBBBB') print("Hello world!") return True diff --git a/automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py b/automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py new file mode 100644 index 00000000..6de34bce --- /dev/null +++ b/automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py @@ -0,0 +1,17 @@ +import unittest +import asyncio + +from automon.integrations.openTelemetryWrapper import OpenTelemetryClient + + +class MyTestCase(unittest.TestCase): + client = OpenTelemetryClient() + + def test_something(self): + self.assertTrue(asyncio.run( + self.client.test()) + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/automon/integrations/openTelemetryWrapper/test/test_memory_trace.py b/automon/integrations/openTelemetryWrapper/test/test_memory_trace_config.py similarity index 100% rename from automon/integrations/openTelemetryWrapper/test/test_memory_trace.py rename to automon/integrations/openTelemetryWrapper/test/test_memory_trace_config.py From 0a8a8c958532740254e16a9f527d3998afe43201 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 3 Apr 2024 04:56:37 -0400 Subject: [PATCH 28/31] opentelemetry: add instrumentation log format --- automon/log/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/automon/log/__init__.py b/automon/log/__init__.py index 196f4921..67d2e22a 100755 --- a/automon/log/__init__.py +++ b/automon/log/__init__.py @@ -3,4 +3,12 @@ from .logger import logging log_format = f'{LogRecordAttribute(timestamp=True).levelname().name_and_lineno().funcName().message()}' -logging.basicConfig(level=DEBUG, format=log_format) +log_format_opentelemetry = f'%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [trace_id=%(otelTraceID)s span_id=%(otelSpanID)s resource.service.name=%(otelServiceName)s trace_sampled=%(otelTraceSampled)s] %(message)s'.replace( + ' ', '\t') + +try: + import opentelemetry + + logging.basicConfig(level=DEBUG, format=log_format_opentelemetry) +except: + logging.basicConfig(level=DEBUG, format=log_format) From de3a352f0e0b1b71e3990d7fd6ec10b36e8f363f Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 3 Apr 2024 05:14:15 -0400 Subject: [PATCH 29/31] opentelemetry: update to async --- .../openTelemetryWrapper/client.py | 47 ++++++++++++++----- .../openTelemetryWrapper/config.py | 15 +++--- .../test/test_memory_trace_client.py | 8 +++- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/automon/integrations/openTelemetryWrapper/client.py b/automon/integrations/openTelemetryWrapper/client.py index 479b2bfc..fb553acd 100644 --- a/automon/integrations/openTelemetryWrapper/client.py +++ b/automon/integrations/openTelemetryWrapper/client.py @@ -1,6 +1,8 @@ import json from opentelemetry.util import types +from opentelemetry.trace import Status, StatusCode + from automon.log import logging from .config import OpenTelemetryConfig @@ -14,28 +16,47 @@ class OpenTelemetryClient(object): def __init__(self): self.config = OpenTelemetryConfig() - def add_event(self, name: str, attributes: types.Attributes = None, **kwargs): - return self.config.current_span.add_event(name=name, attributes=attributes, **kwargs) + async def add_event(self, name: str, attributes: types.Attributes = None, **kwargs): + logger.debug(dict(name=name, attributes=attributes, kwargs=kwargs)) + span = await self.current_span() + return span.add_event(name=name, attributes=attributes, **kwargs) async def clear(self): + logger.debug('clear') return await self.config.clear() + async def current_span(self): + logger.debug('current_span') + return await self.config.current_span() + async def is_ready(self): if await self.config.is_ready(): return True async def get_finished_spans(self): + logger.debug('get_finished_spans') return await self.config.get_finished_spans() async def pop_finished_spans(self): + logger.debug('pop_finished_spans') return await self.config.pop_finished_spans() - def record_exception(self, exception: Exception): - return self.config.current_span.record_exception(exception=exception) - - def start_as_current_span(self, name: str, attributes: types.Attributes = None, **kwargs): + async def record_exception(self, exception: Exception): + logger.error(f'{exception}') + span = await self.current_span() + span.set_status(Status(StatusCode.ERROR)) + return span.record_exception(exception=exception) + + async def start_as_current_span( + self, name: str, + attributes: types.Attributes = None, + **kwargs + ): + logger.debug(dict(name=name, attributes=attributes, kwargs=kwargs)) return self.config.tracer.start_as_current_span( - name=name, attributes=attributes, **kwargs) + name=name, + attributes=attributes, + **kwargs) async def start_consumer(self): """adds spans from memory to queue""" @@ -50,10 +71,14 @@ async def start_producer(self): return async def test(self): - with self.start_as_current_span(name='rootSpan') as trace_root: - with self.start_as_current_span(name='childSpan') as trace_child: - self.add_event('AAAAAAAA') - self.add_event('BBBBBBBB') + with await self.start_as_current_span(name='rootSpan') as trace_root: + await self.add_event('AAAAAAAA') + + with await self.start_as_current_span(name='childSpan') as trace_child: + await self.add_event('AAAAAAAA') + await self.add_event('BBBBBBBB') + + await self.add_event('BBBBBBBB') return True diff --git a/automon/integrations/openTelemetryWrapper/config.py b/automon/integrations/openTelemetryWrapper/config.py index 99462265..e496b5ba 100644 --- a/automon/integrations/openTelemetryWrapper/config.py +++ b/automon/integrations/openTelemetryWrapper/config.py @@ -1,7 +1,6 @@ import asyncio +import opentelemetry -from opentelemetry import trace -from opentelemetry.trace import Status, StatusCode from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -19,25 +18,27 @@ def __init__(self): self.processor = SimpleSpanProcessor(self.memory_processor) self.provider.add_span_processor(self.processor) - trace.set_tracer_provider(self.provider) + opentelemetry.trace.set_tracer_provider(self.provider) - self.tracer = trace.get_tracer(__name__) + self.tracer = opentelemetry.trace.get_tracer(__name__) self.queue_consumer = asyncio.Queue() self.queue_producer = asyncio.Queue() async def clear(self): + logger.debug('clear') return self.memory_processor.clear() - @property - def current_span(self): - return trace.get_current_span() + async def current_span(self): + logger.debug('get_current_span') + return opentelemetry.trace.get_current_span() async def is_ready(self): if self.provider and self.memory_processor and self.processor: return True async def get_finished_spans(self): + logger.debug('get_finished_spans') return self.memory_processor.get_finished_spans() async def pop_finished_spans(self): diff --git a/automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py b/automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py index 6de34bce..6fa4b062 100644 --- a/automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py +++ b/automon/integrations/openTelemetryWrapper/test/test_memory_trace_client.py @@ -9,8 +9,12 @@ class MyTestCase(unittest.TestCase): def test_something(self): self.assertTrue(asyncio.run( - self.client.test()) - ) + self.client.test())) + + spans = asyncio.run( + self.client.get_finished_spans()) + + pass if __name__ == '__main__': From c6e3abb4fe0090c6f4dc078f85df87df1f40fb58 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 3 Apr 2024 05:32:00 -0400 Subject: [PATCH 30/31] github actions: disable python38.yml --- .github/workflows/python38.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python38.yml b/.github/workflows/python38.yml index 3d75a378..f841b4ec 100644 --- a/.github/workflows/python38.yml +++ b/.github/workflows/python38.yml @@ -2,9 +2,9 @@ name: 3.8 on: push: - branches: [ '*' ] + branches: [ 'none' ] pull_request: - branches: [ '*' ] + branches: [ 'none' ] env: # Use docker.io for Docker Hub if empty From 60ab679a672cbebb3593acae47a838ac342835d1 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 3 Apr 2024 05:32:14 -0400 Subject: [PATCH 31/31] github actions: disable python39.yml --- .github/workflows/python39.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python39.yml b/.github/workflows/python39.yml index 6c525385..3b65deb7 100644 --- a/.github/workflows/python39.yml +++ b/.github/workflows/python39.yml @@ -2,9 +2,9 @@ name: 3.9 on: push: - branches: [ '*' ] + branches: [ 'none' ] pull_request: - branches: [ '*' ] + branches: [ 'none' ] env: # Use docker.io for Docker Hub if empty