Add swimlane

.github/workflows/python36.yml

@@ -2,9 +2,9 @@ name: 3.6 EOL
 
 on:
   push:
-    branches: [ '*' ]
+    branches: [ 'none' ]
   pull_request:
-    branches: [ '*' ]
+    branches: [ 'none' ]
 
 env:
   # Use docker.io for Docker Hub if empty

.github/workflows/python37.yml

@@ -2,9 +2,9 @@ name: 3.7 EOL
 
 on:
   push:
-    branches: [ '*' ]
+    branches: [ 'none' ]
   pull_request:
-    branches: [ '*' ]
+    branches: [ 'none' ]
 
 env:
   # Use docker.io for Docker Hub if empty

.github/workflows/python38.yml

@@ -2,9 +2,9 @@ name: 3.8
 
 on:
   push:
-    branches: [ '*' ]
+    branches: [ 'none' ]
   pull_request:
-    branches: [ '*' ]
+    branches: [ 'none' ]
 
 env:
   # Use docker.io for Docker Hub if empty

.github/workflows/python39.yml

@@ -2,9 +2,9 @@ name: 3.9
 
 on:
   push:
-    branches: [ '*' ]
+    branches: [ 'none' ]
   pull_request:
-    branches: [ '*' ]
+    branches: [ 'none' ]
 
 env:
   # Use docker.io for Docker Hub if empty

README.md

@@ -45,30 +45,17 @@ Github issues and feature requests welcomed.
 | Data Store      | minio<br/>swift                                             |
 | Devices         | snmp                                                        |
 | Google Cloud    | google auth api<br/>google people api<br/>google sheets api |
+| Helpers         | os<br/>subprocess<br/>threading<br/>socket<br/>datetime     |
 | Logging         | sentryio                                                    |
 | MacOS           | airport<br/>macchanger                                      |
 | Python          | logging<br/>requests                                        |
 | SOAR            | swimlane<br/>splunk soar                                    |
 | Recon           | nmap                                                        |
 | Test Automation | selenium                                                    |
-| Category          | Library                                                     |
-|-------------------|-------------------------------------------------------------|
-| API               | flask                                                       |
-| Chat              | slack                                                       |
-| Data Scraping     | beautifulsoup<br/>facebook groups<br/>instagram<br/>scrapy  |
-| Databases         | elasticsearch<br/>neo4j<br/>splunk                          |
-| Data Store        | minio<br/>swift                                             |
-| Devices           | snmp                                                        |
-| Google Cloud      | google auth api<br/>google people api<br/>google sheets api |
-| Tracing / Logging | openTelemetry<br/>sentryio                                  |
-| macOS             | airport<br/>macchanger                                      |
-| Python            | logging<br/>requests                                        |
-| Recon             | nmap                                                        |
-| Test Automation   | selenium                                                    |
 
 #### Requires
 
-- python >= 3.8
+- python >= 3.10
 
 _Note: install requirements.txt to use all integrations_
 

automon/helpers/__init__.py

@@ -1,5 +1,6 @@
 from .dates import Dates
 from .markdown import Chat, Format
+from .networking import Networking
 from .osWrapper import environ
 from .sleeper import Sleeper
 from .subprocessWrapper import Run

automon/integrations/datadogWrapper/__init__.py

@@ -0,0 +1,2 @@
+from .client import DatadogClientRest
+from .config import DatadogConfigRest

automon/integrations/datadogWrapper/api/__init__.py

@@ -0,0 +1,2 @@
+from .v1 import V1
+from .v2 import V2

automon/integrations/datadogWrapper/api/v1.py

@@ -0,0 +1,14 @@
+class V1(object):
+
+    def __init__(self, host):
+        self.endpoint = host
+
+    @property
+    def api(self):
+        self.endpoint += '/api/v1'
+        return self
+
+    @property
+    def validate(self):
+        self.endpoint += '/validate'
+        return self

automon/integrations/datadogWrapper/api/v2.py

@@ -0,0 +1,17 @@
+from automon import Networking
+
+
+class V2(object):
+
+    def __init__(self, host):
+        self.endpoint = host
+
+    @property
+    def api(self):
+        self.endpoint += '/api/v2'
+        return self
+
+    @property
+    def logs(self):
+        self.endpoint += '/logs'
+        return self

automon/integrations/datadogWrapper/client.py

@@ -0,0 +1,48 @@
+from automon.integrations.requestsWrapper import RequestsClient
+from automon.log import logging
+
+from .config import DatadogConfigRest
+from .api import V1, V2
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class DatadogClientRest(object):
+
+    def __init__(self, host: str = None, api_key: str = None):
+        self.config = DatadogConfigRest(host=host, api_key=api_key)
+        self.requests = RequestsClient()
+
+    async def is_ready(self):
+        if await self.config.is_ready():
+            if await self.validate():
+                return True
+        logger.error(f'client not ready')
+
+    async def log(self, ddsource: str, hostname: str, service: str, message: str, ddtags: str = 'env:test,version:0.1'):
+        url = V2(self.config.host_log).api.logs.endpoint
+
+        log = {
+            "ddsource": ddsource,
+            "ddtags": ddtags,
+            "hostname": hostname,
+            "service": service,
+            'message': message
+        }
+
+        logger.debug(log)
+
+        response = await self.requests.post(url=url, json=log)
+        response_log = await self.requests.to_dict()
+
+        return response_log
+
+    async def validate(self):
+        url = V1(self.config.host).api.validate.endpoint
+
+        self.requests.session.headers.update(await self.config.headers())
+        response = await self.requests.get(url=url)
+        response_validate = await self.requests.to_dict()
+
+        return response_validate

automon/integrations/datadogWrapper/config.py

@@ -0,0 +1,29 @@
+from automon import environ
+from automon.log import logging
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class DatadogConfigRest(object):
+    api_key: str
+
+    def __init__(self, host: str = 'https://api.datadoghq.com', api_key: str = None, app_key: str = None):
+        self.host = host or environ('DD_SITE')
+        self.host_log = 'https://http-intake.logs.us5.datadoghq.com'
+        self.api_key = api_key or environ('DD_API_KEY')
+        self.app_key = app_key or environ('DD_APP_KEY')
+
+    async def is_ready(self):
+        if self.host and self.api_key and self.app_key:
+            return True
+        logger.error(f'missing DD_SITE DD_API_KEY DD_APP_KEY')
+
+    async def headers(self):
+        if await self.is_ready():
+            return {
+                'DD-API-KEY': f'{self.api_key}',
+                'DD-APPLICATION-KEY': f'{self.app_key}',
+                'Accept': 'application/json',
+                'Content-Type': 'application/json',
+            }

automon/integrations/datadogWrapper/tests/test_auth.py

@@ -0,0 +1,20 @@
+import asyncio
+import unittest
+
+from automon.integrations.datadogWrapper import DatadogClientRest
+
+
+class MyTestCase(unittest.TestCase):
+    client = DatadogClientRest()
+
+    if asyncio.run(client.is_ready()):
+        def test_auth(self):
+            self.assertTrue(asyncio.run(
+                self.client.validate()
+            ))
+
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

automon/integrations/datadogWrapper/tests/test_log.py

@@ -0,0 +1,31 @@
+import asyncio
+import unittest
+
+from automon.integrations.datadogWrapper import DatadogClientRest
+from automon.integrations.openTelemetryWrapper import OpenTelemetryClient
+
+
+class MyTestCase(unittest.TestCase):
+    client = DatadogClientRest()
+    client_tracer = OpenTelemetryClient()
+
+    def test_log(self):
+        if asyncio.run(self.client.is_ready()):
+
+            asyncio.run(self.client_tracer.config.test())
+
+            spans = asyncio.run(self.client_tracer.to_datadog())
+            for span in spans:
+                asyncio.run(
+                    self.client.log(
+                        ddsource=span['ddsource'],
+                        ddtags=span['ddtags'],
+                        hostname=span['hostname'],
+                        service=span['service'],
+                        message=span['message'],
+                    ))
+
+            pass
+
+    if __name__ == '__main__':
+        unittest.main()

automon/helpers/tests/test_grok.py → automon/integrations/grok/tests/test_grok.py

@@ -1,6 +1,6 @@
 import unittest
 
-from automon.helpers.grok import Grok
+from automon.integrations.grok import Grok
 
 
 class GrokTest(unittest.TestCase):

automon/integrations/openTelemetryWrapper/__init__.py

@@ -1 +1,2 @@
-from .config import *
+from .client import OpenTelemetryClient
+from .config import OpenTelemetryConfig

automon/integrations/openTelemetryWrapper/client.py

@@ -0,0 +1,107 @@
+import json
+
+from opentelemetry.util import types
+from opentelemetry.trace import Status, StatusCode
+
+from automon.log import logging
+
+from .config import OpenTelemetryConfig
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class OpenTelemetryClient(object):
+
+    def __init__(self):
+        self.config = OpenTelemetryConfig()
+
+    async def add_event(self, name: str, attributes: types.Attributes = None, **kwargs):
+        logger.debug(dict(name=name, attributes=attributes, kwargs=kwargs))
+        span = await self.current_span()
+        return span.add_event(name=name, attributes=attributes, **kwargs)
+
+    async def clear(self):
+        logger.debug('clear')
+        return await self.config.clear()
+
+    async def current_span(self):
+        logger.debug('current_span')
+        return await self.config.current_span()
+
+    async def is_ready(self):
+        if await self.config.is_ready():
+            return True
+
+    async def get_finished_spans(self):
+        logger.debug('get_finished_spans')
+        return await self.config.get_finished_spans()
+
+    async def pop_finished_spans(self):
+        logger.debug('pop_finished_spans')
+        return await self.config.pop_finished_spans()
+
+    async def record_exception(self, exception: Exception):
+        logger.error(f'{exception}')
+        span = await self.current_span()
+        span.set_status(Status(StatusCode.ERROR))
+        return span.record_exception(exception=exception)
+
+    async def start_as_current_span(
+            self, name: str,
+            attributes: types.Attributes = None,
+            **kwargs
+    ):
+        logger.debug(dict(name=name, attributes=attributes, kwargs=kwargs))
+        return self.config.tracer.start_as_current_span(
+            name=name,
+            attributes=attributes,
+            **kwargs)
+
+    async def start_consumer(self):
+        """adds spans from memory to queue"""
+        while True:
+            pass
+        return
+
+    async def start_producer(self):
+        """"""
+        while True:
+            pass
+        return
+
+    async def test(self):
+        with await self.start_as_current_span(name='rootSpan') as trace_root:
+            await self.add_event('AAAAAAAA')
+
+            with await self.start_as_current_span(name='childSpan') as trace_child:
+                await self.add_event('AAAAAAAA')
+                await self.add_event('BBBBBBBB')
+
+            await self.add_event('BBBBBBBB')
+
+        return True
+
+    async def to_dict(self):
+        return [
+            json.loads(span.to_json())
+            for span in await self.get_finished_spans()
+        ]
+
+    async def to_datadog(self):
+        log = []
+        for span in await self.to_dict():
+            message = dict(span).copy()
+            ddsource = None
+            ddtags = ','.join([f"{x[0]}:{x[1]}" for x in span.get("resource").get("attributes").items()])
+            hostname = span['context']['trace_id']
+            service = span['context']['span_id']
+
+            log.append(dict(
+                ddsource=ddsource,
+                ddtags=ddtags,
+                hostname=hostname,
+                service=service,
+                message=message,
+            ))
+        return log