Merge branch 'master' into async-requests

TheShellLand · Mar 19, 2024 · c8ac005 · c8ac005
2 parents 2b3cd64 + 8708a23
commit c8ac005
Show file tree

Hide file tree

Showing 14 changed files with 115 additions and 30 deletions.
diff --git a/README.md b/README.md
@@ -36,20 +36,20 @@ Github issues and feature requests welcomed.
 
 ### Integrations
 
-| Category        | Library                                                     |
-|-----------------|-------------------------------------------------------------|
-| API             | flask                                                       |
-| Chat            | slack                                                       |
-| Data Scraping   | beautifulsoup<br/>facebook groups<br/>instagram<br/>scrapy  |
-| Databases       | elasticsearch<br/>neo4j<br/>splunk                          |
-| Data Store      | minio<br/>swift                                             |
-| Devices         | snmp                                                        |
-| Google Cloud    | google auth api<br/>google people api<br/>google sheets api |
-| Logging         | sentryio                                                    |
-| macOS           | airport<br/>macchanger                                      |
-| Python          | logging<br/>requests                                        |
-| Recon           | nmap                                                        |
-| Test Automation | selenium                                                    |
+| Category          | Library                                                     |
+|-------------------|-------------------------------------------------------------|
+| API               | flask                                                       |
+| Chat              | slack                                                       |
+| Data Scraping     | beautifulsoup<br/>facebook groups<br/>instagram<br/>scrapy  |
+| Databases         | elasticsearch<br/>neo4j<br/>splunk                          |
+| Data Store        | minio<br/>swift                                             |
+| Devices           | snmp                                                        |
+| Google Cloud      | google auth api<br/>google people api<br/>google sheets api |
+| Tracing / Logging | openTelemetry<br/>sentryio                                  |
+| macOS             | airport<br/>macchanger                                      |
+| Python            | logging<br/>requests                                        |
+| Recon             | nmap                                                        |
+| Test Automation   | selenium                                                    |
 
 #### Requires
 

diff --git a/automon/integrations/openTelemetryWrapper/README.md b/automon/integrations/openTelemetryWrapper/README.md
@@ -0,0 +1,51 @@
+# OpenTelemetry
+
+## Install Automatic Instrumentation
+
+* https://opentelemetry.io/docs/languages/python/automatic/
+
+```shell
+python3 -m pip install -U opentelemetry-distro opentelemetry-exporter-otlp
+python3 -m pip install git+https://github.com/TheShellLand/[email protected]#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server
+opentelemetry-bootstrap --action install
+
+# python logging
+python3 -m pip install -U opentelemetry-instrumentation-logging
+
+# django
+python3 -m pip install -U opentelemetry-instrumentation-django
+```
+
+## Known issues:
+
+### Problem 1 (fixed):
+
+fixed: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/2053#issuecomment-1962248084
+
+```
+ERROR: No matching distribution found for opentelemetry-instrumentation-aiohttp-server==0.42b0 
+```
+
+reference: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/2053
+
+Solution:
+
+* install using commit flag
+
+> I have added a new tag: 0.43b0hotfix. This can be used to install
+> opentelemetry-instrumentation-aiohttp-server and
+> opentelemetry-resource-detector-container with the following pip commands:
+
+opentelemetry-instrumentation-aiohttp-server:
+
+```shell
+python3 -m pip install git+https://github.com/open-telemetry/[email protected]#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server
+```
+
+opentelemetry-resource-detector-container:
+
+```shell
+python3 -m pip install git+https://github.com/open-telemetry/[email protected]#subdirectory=resource/opentelemetry-resource-detector-container
+```
+
+solution: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/2053#issuecomment-1928485674
diff --git a/automon/integrations/openTelemetryWrapper/__init__.py b/automon/integrations/openTelemetryWrapper/__init__.py
@@ -0,0 +1 @@
+from .config import *
diff --git a/automon/integrations/openTelemetryWrapper/config.py b/automon/integrations/openTelemetryWrapper/config.py
diff --git a/automon/integrations/openTelemetryWrapper/docker/Dockerfile b/automon/integrations/openTelemetryWrapper/docker/Dockerfile
@@ -0,0 +1,6 @@
+FROM python:3
+
+RUN python3 -m pip install -U opentelemetry-distro opentelemetry-exporter-otlp
+RUN python3 -m pip install git+https://github.com/TheShellLand/[email protected]#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server
+RUN opentelemetry-bootstrap --action install
+RUN python3 -m pip install -U opentelemetry-instrumentation-logging
diff --git a/automon/integrations/openTelemetryWrapper/docker/build.sh b/automon/integrations/openTelemetryWrapper/docker/build.sh
@@ -0,0 +1,7 @@
+#!/bin/bash 
+
+# build image 
+
+cd "$(dirname $0)" && set -xe
+
+docker build $@ -f Dockerfile -t opentelemetry-python .
diff --git a/automon/integrations/openTelemetryWrapper/requirements.txt b/automon/integrations/openTelemetryWrapper/requirements.txt
@@ -0,0 +1,6 @@
+# opentelemetry
+opentelemetry-distro>=0.44b0
+opentelemetry-exporter-otlp>=1.22.0
+opentelemetry-instrumentation-logging>=0.44b0
+# workaround
+git+https://github.com/TheShellLand/[email protected]#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server
diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py
@@ -327,6 +327,10 @@ async def find_xpath(
 
     async def get(self, url: str, **kwargs) -> bool:
         """get url"""
+        if not self.webdriver:
+            logger.error(f'missing webdriver')
+            raise Exception(f'missing webdriver')
+
         try:
             if self.webdriver.get(url, **kwargs) is None:
                 logger.info(str(dict(

diff --git a/automon/integrations/seleniumWrapper/tests/test_browser_headless.py b/automon/integrations/seleniumWrapper/tests/test_browser_headless.py
@@ -1,4 +1,5 @@
 import unittest
+import asyncio
 
 from automon.integrations.seleniumWrapper import SeleniumBrowser, ChromeWrapper
 
@@ -10,22 +11,17 @@
 
 class SeleniumClientTest(unittest.TestCase):
 
-    async def test(self):
-        if browser.run():
-            await browser.set_window_size(device_type='web-large')
-            while True:
+    def test(self):
 
-                try:
-                    if await browser.get('http://bing.com'):
-                        self.assertTrue(await browser.save_screenshot())
-                        self.assertTrue(await browser.save_screenshot())
-                        self.assertTrue(await browser.save_screenshot(folder='./'))
+        if asyncio.run(browser.run()):
+            asyncio.run(browser.set_window_size(device_type='web-large'))
 
-                    await browser.quit()
-                    break
+            if asyncio.run(browser.get('http://bing.com')):
+                self.assertTrue(asyncio.run(browser.save_screenshot()))
+                self.assertTrue(asyncio.run(browser.save_screenshot()))
+                self.assertTrue(asyncio.run(browser.save_screenshot(folder='./')))
 
-                except:
-                    pass
+            asyncio.run(browser.quit())
 
 
 if __name__ == '__main__':

diff --git a/automon/integrations/seleniumWrapper/webdriver_chrome.py b/automon/integrations/seleniumWrapper/webdriver_chrome.py
@@ -397,7 +397,7 @@ def update_paths(self, path: str):
 
                 return True
 
-        logger.error(f'not found: {path}')
+        logger.error(f'chrome driver not found: {path}')
 
     async def quit(self):
         """quit

diff --git a/docker/install.sh b/docker/install.sh
@@ -10,7 +10,7 @@ google-chrome --version
 # install chromedriver
 cd /tmp/
 # https://googlechromelabs.github.io/chrome-for-testing/#stable
-wget -q https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/119.0.6045.105/linux64/chromedriver-linux64.zip
+wget -q https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.111/linux64/chromedriver-linux64.zip
 unzip chromedriver-linux64.zip
 sudo mv chromedriver-linux64/chromedriver /usr/bin/chromedriver
 chromedriver --version
diff --git a/env-example.sh b/env-example.sh
@@ -55,6 +55,13 @@ OPENSTACK_INTERFACE=public
 OPENSTACK_IDENTITY_API_VERSION=3
 SWIFTCLIENT_INSECURE=True
 
+# OpenTelemetry
+# https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/logging/logging.html
+OTEL_PYTHON_LOG_CORRELATION=true
+OTEL_PYTHON_LOG_FORMAT='%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [trace_id=%(otelTraceID)s span_id=%(otelSpanID)s resource.service.name=%(otelServiceName)s trace_sampled=%(otelTraceSampled)s] - %(message)s'
+OTEL_PYTHON_LOG_LEVEL=debug
+
+
 # Splunk SOAR
 SPLUNK_SOAR_HOST=
 SPLUNK_SOAR_AUTH_TOKEN=

diff --git a/requirements.txt b/requirements.txt
@@ -40,6 +40,13 @@ neo4j>=4.4.9
 python-keystoneclient>=4.2.0
 python-swiftclient>=3.12.0
 
+# opentelemetry
+opentelemetry-distro>=0.44b0
+opentelemetry-exporter-otlp>=1.22.0
+opentelemetry-instrumentation-logging>=0.44b0
+# workaround
+git+https://github.com/TheShellLand/[email protected]#subdirectory=instrumentation/opentelemetry-instrumentation-aiohttp-server
+
 # splunk soar
 pytz>=2021.1
 

diff --git a/upload.sh b/upload.sh
@@ -16,7 +16,7 @@ if [ "$@" == '--local' ]; then
   python3 setup.py sdist bdist_wheel
   twine check dist/*
   python3 -m twine upload --repository $PYPI --repository-url $TWINE_REPOSITORY \
-    -u $TWINE_USERNAME -p $TWINE_PASSWORD --non-interactive --skip-existing dist/*
+    -u "$TWINE_USERNAME" -p "$TWINE_PASSWORD" --non-interactive --skip-existing dist/*
   python3 setup.py clean --all
 elif [ "$@" == '--docker' ]; then
   set -x