Merge branch 'refs/heads/master' into fix-depreciated-methods

# Conflicts:
#	docker/install.sh

README.md

@@ -49,7 +49,7 @@ Github issues and feature requests welcomed.
 | Logging         | sentryio                                                    |
 | MacOS           | airport<br/>macchanger<br/>wdutil                           |
 | Python          | logging<br/>requests                                        |
-| SOAR            | swimlane<br/>splunk soar                                    |
+| SOAR            | swimlane<br/>splunk soar<br/>xsoar                          |
 | Recon           | nmap                                                        |
 | Test Automation | selenium                                                    |
 

automon/integrations/instagram/client_browser.py

@@ -27,8 +27,7 @@ def __init__(self,
                  headless: bool = True):
         """Instagram Browser Client"""
         self.config = config or InstagramConfig(login=login, password=password)
-        self.browser = SeleniumBrowser()
-        self.browser.config.webdriver_wrapper = ChromeWrapper()
+        self.browser = None
 
         self.authenticated_browser = None
         self.useragent = None
@@ -237,13 +236,19 @@ def login(self) -> str:
         return self.config.login
 
     async def start(self):
-        self.useragent = await self.browser.get_random_user_agent()
+        try:
+            self.browser = SeleniumBrowser()
+            self.browser.config.webdriver_wrapper = ChromeWrapper()
+
+            self.useragent = await self.browser.get_random_user_agent()
 
-        if self.headless:
-            self.browser.config.webdriver_wrapper.in_headless()
-            self.browser.config.webdriver_wrapper.set_user_agent(self.useragent)
-        else:
-            self.browser.config.webdriver_wrapper.set_user_agent(self.useragent)
+            if self.headless:
+                self.browser.config.webdriver_wrapper.in_headless()
+                self.browser.config.webdriver_wrapper.set_user_agent(self.useragent)
+            else:
+                self.browser.config.webdriver_wrapper.set_user_agent(self.useragent)
+        except Exception as error:
+            logger.error(error)
 
     @property
     def urls(self):

automon/integrations/requestsWrapper/client.py

@@ -15,10 +15,10 @@ def __init__(self, url: str = None, data: dict = None, headers: dict = None,
 
         self.config = config or RequestsConfig()
 
-        self.url = url
-        self.data = data
-        self.errors = None
-        self.headers = headers
+        self.url: str = url
+        self.data: dict = data
+        self.errors: bytes = b''
+        self.headers: dict = headers
         self.response = None
         self.requests = requests
         self.session = self.requests.Session()
@@ -122,6 +122,8 @@ async def get(
             if self.status_code == 200:
                 return True
 
+            self.errors = self.content
+
             return False
         except Exception as e:
             self.errors = e
@@ -151,6 +153,8 @@ async def patch(
             if self.status_code == 200:
                 return True
 
+            self.errors = self.content
+
             return False
         except Exception as e:
             self.errors = e
@@ -180,6 +184,8 @@ async def post(
             if self.status_code == 200:
                 return True
 
+            self.errors = self.content
+
             return False
         except Exception as e:
             self.errors = e
@@ -209,6 +215,8 @@ async def put(
             if self.status_code == 200:
                 return True
 
+            self.errors = self.content
+
             return False
         except Exception as e:
             self.errors = e

automon/integrations/requestsWrapper/rest.py

@@ -6,6 +6,7 @@
 logger = log.logging.getLogger(__name__)
 logger.setLevel(log.DEBUG)
 
+
 class BaseRestClient:
     requests: RequestsClient
     config: RequestsConfig

automon/integrations/requestsWrapper/tests/test_rest.py

@@ -9,7 +9,7 @@
 class Client(unittest.TestCase):
     def test_get(self):
         self.assertTrue(asyncio.run(r.get('https://1.1.1.1')))
-        self.assertTrue(r.requests.get('https://1.1.1.1'))
+        self.assertTrue(asyncio.run(r.requests.get('https://1.1.1.1')))
         self.assertFalse(asyncio.run(r.get('x://127.0.0.1')))
 
 

automon/integrations/requestsWrapper/tests/test_rest_inherit.py

@@ -4,16 +4,16 @@
 from automon.integrations.requestsWrapper.rest import BaseRestClient
 
 
-class Test(BaseRestClient):
+class Inherit(BaseRestClient):
 
     def __init__(self):
-        BaseRestClient.__init__(self)
+        super().__init__()
         pass
 
 
 class Client(unittest.TestCase):
     def test_get(self):
-        self.assertTrue(asyncio.run(Test().get(url='https://1.1.1.1')))
+        self.assertTrue(asyncio.run(Inherit().get(url='https://1.1.1.1')))
 
 
 if __name__ == '__main__':

automon/integrations/seleniumWrapper/browser.py

@@ -6,6 +6,7 @@
 import tempfile
 import selenium
 import selenium.webdriver
+import selenium.webdriver.remote.webelement
 
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
@@ -58,7 +59,7 @@ def by(self) -> By:
     def config(self):
         return self._config
 
-    async def cookie_file_to_dict(self, file: str = 'cookies.txt') -> list:
+    async def cookie_file_to_dict(self, file: str = 'cookies.json') -> list:
         logger.debug(dict(
             cookie_file_to_dict=file
         ))
@@ -180,31 +181,26 @@ def _screenshot_name(self, prefix=None):
 
     async def action_click(
             self,
-            xpath: str, **kwargs
-    ) -> selenium.webdriver.Chrome.find_element:
-        """perform mouse command"""
+            element: selenium.webdriver.remote.webelement.WebElement, **kwargs):
+        """perform mouse click"""
         try:
-            logger.debug(str(dict(
-                xpath=xpath,
-            )))
-            return await self.find_element(value=xpath, by=self.by.XPATH, **kwargs).click()
+            logger.debug(dict(element=element))
+            return element.click()
 
         except Exception as error:
             raise Exception(error)
 
     async def action_type(
             self,
             key: str or Keys,
-            secret: bool = True,
+            secret: bool = False,
     ) -> selenium.webdriver.common.action_chains.ActionChains:
         """perform keyboard command"""
 
         if secret:
-            key = f'*' * len(key)
-
-        logger.debug(str(dict(
-            send_keys=key,
-        )))
+            logger.debug(dict(send_keys=f'*' * len(f'{key}')))
+        else:
+            logger.debug(dict(send_keys=key))
 
         try:
             return selenium.webdriver.common.action_chains.ActionChains(
@@ -213,6 +209,44 @@ async def action_type(
         except Exception as error:
             raise Exception(error)
 
+    async def action_type_up(
+            self,
+            key: str or Keys,
+            secret: bool = False,
+    ) -> selenium.webdriver.common.action_chains.ActionChains:
+        """release key"""
+
+        if secret:
+            logger.debug(dict(send_keys=f'*' * len(f'{key}')))
+        else:
+            logger.debug(dict(send_keys=key))
+
+        try:
+            return selenium.webdriver.common.action_chains.ActionChains(
+                self.webdriver).key_up(key).perform()
+
+        except Exception as error:
+            raise Exception(error)
+
+    async def action_type_down(
+            self,
+            key: str or Keys,
+            secret: bool = False,
+    ) -> selenium.webdriver.common.action_chains.ActionChains:
+        """hold key down"""
+
+        if secret:
+            logger.debug(dict(send_keys=f'*' * len(f'{key}')))
+        else:
+            logger.debug(dict(send_keys=key))
+
+        try:
+            return selenium.webdriver.common.action_chains.ActionChains(
+                self.webdriver).key_down(key).perform()
+
+        except Exception as error:
+            raise Exception(error)
+
     async def add_cookie(self, cookie_dict: dict) -> bool:
         result = self.webdriver.add_cookie(cookie_dict=cookie_dict)
 
@@ -289,7 +323,7 @@ async def delete_all_cookies(self) -> None:
     async def _url_filename(self, url: str):
         parsed = await self.urlparse(url)
         hostname = parsed.hostname
-        cookie_file = f'cookies-{hostname}.txt'
+        cookie_file = f'cookies-{hostname}.json'
         logger.debug(dict(
             _url_filename=cookie_file
         ))
@@ -364,6 +398,75 @@ async def error_parsing(error) -> tuple:
 
         return error, None, None
 
+    async def find_anything(
+            self,
+            value: str,
+            case_insensitivity: bool = True,
+            contains: bool = True,
+            **kwargs
+    ) -> selenium.webdriver.Chrome.find_element:
+        """fuzzy search through everything
+
+        find all tags
+        find all matches within meta data
+        """
+        logger.info(dict(
+            current_url=self.current_url,
+            value=value,
+            case_insensitivity=case_insensitivity,
+            contains=contains,
+            kwargs=kwargs,
+        ))
+
+        by_types = [
+            self.by.TAG_NAME,
+        ]
+
+        MATCHED = []
+
+        for by in by_types:
+            elements = self.webdriver.find_elements(value='*', by=by)
+            for element in elements:
+                dirs = dir(element)
+                dir_meta = []
+                for dir_ in dirs:
+                    try:
+                        dir_meta.append(
+                            getattr(element, f'{dir_}')
+                        )
+
+                        MATCH = f'{value}'
+                        AGAINST = f'''{getattr(element, f'{dir_}')}'''
+
+                        if case_insensitivity:
+                            MATCH = f'{value}'.lower()
+                            AGAINST = f'''{getattr(element, f'{dir_}')}'''.lower()
+
+                    except:
+                        pass
+
+                    FOUND = None
+
+                    if MATCH == AGAINST and not contains:
+                        FOUND = element
+
+                    if MATCH in AGAINST and contains:
+                        FOUND = element
+
+                    if FOUND and FOUND not in MATCHED:
+                        logger.info(dict(
+                            MATCH=MATCH,
+                            AGAINST=AGAINST,
+                            attribute=dir_,
+                            element=element,
+                        ))
+                        MATCHED.append(FOUND)
+
+                pass
+            pass
+
+        return MATCHED
+
     async def find_element(
             self,
             value: str,
@@ -377,6 +480,19 @@ async def find_element(
         )))
         return self.webdriver.find_element(value=value, by=by, **kwargs)
 
+    async def find_elements(
+            self,
+            value: str,
+            by: By.ID = By.ID,
+            **kwargs
+    ) -> selenium.webdriver.Chrome.find_elements:
+        """find elements"""
+        logger.info(str(dict(
+            current_url=self.current_url,
+            value=value,
+        )))
+        return self.webdriver.find_elements(value=value, by=by, **kwargs)
+
     async def find_xpath(
             self,
             value: str,
@@ -388,7 +504,7 @@ async def find_xpath(
             current_url=self.current_url,
             value=value,
         )))
-        return self.find_element(value=value, by=by, **kwargs)
+        return await self.find_element(value=value, by=by, **kwargs)
 
     async def get(self, url: str, **kwargs) -> bool:
         """get url"""
@@ -600,7 +716,7 @@ async def set_window_position(self, x: int = 0, y: int = 0):
 
     async def start(self):
         """alias to run"""
-        return self.run()
+        return await self.run()
 
     async def wait_for(
             self,

automon/integrations/seleniumWrapper/webdriver_chrome.py

@@ -22,8 +22,6 @@ def __init__(self):
         self._ChromeService = None
         self._window_size = set_window_size()
 
-        self.update_paths(self.chromedriver_path)
-
     def __repr__(self):
         if self._webdriver:
             return str(dict(
@@ -56,7 +54,7 @@ def chromedriver_path(self):
             if os.path.exists(path):
                 return path
 
-        logger.error('missing SELENIUM_CHROMEDRIVER_PATH')
+        raise Exception('missing SELENIUM_CHROMEDRIVER_PATH')
 
     @property
     def chromedriverVersion(self):
@@ -313,6 +311,8 @@ def in_sandbox_disabled(self):
 
     async def run(self) -> True or Exception:
         try:
+            self.update_paths(self.chromedriver_path)
+
             if self.chromedriver_path:
                 self._ChromeService = selenium.webdriver.ChromeService(
                     executable_path=self.chromedriver_path
@@ -372,6 +372,7 @@ def set_user_agent(self, user_agent: str):
         return self
 
     def set_window_size(self, *args, **kwargs):
+        """has to be set after setting webdriver"""
         self._window_size = set_window_size(*args, **kwargs)
         width, height = self.window_size
         self.webdriver.set_window_size(width=width, height=height)

automon/integrations/xsoar/__init__.py

@@ -0,0 +1,2 @@
+from .client import XSOARClient
+from .config import XSOARConfig