selenium: add option to autosave cookies. update logging. update type…

… hinting. fix tests.

automon/integrations/seleniumWrapper/browser.py

@@ -55,8 +55,10 @@ def by(self) -> By:
     def config(self):
         return self._config
 
-    async def cookie_file_to_dict(self, file: str = 'cookies.txt'):
-        logger.debug(f'{file}')
+    async def cookie_file_to_dict(self, file: str = 'cookies.txt') -> list:
+        logger.debug(dict(
+            cookie_file_to_dict=file
+        ))
         with open(file, 'r') as file:
             return json.loads(file.read())
 
@@ -108,7 +110,9 @@ def user_agent(self):
     @property
     def current_url(self):
         if self.webdriver:
-            logger.debug(self._current_url)
+            logger.debug(dict(
+                current_url=self._current_url
+            ))
             if self._current_url == 'data:,':
                 return ''
             return self._current_url
@@ -124,7 +128,7 @@ def _screenshot_name(self, prefix=None):
         """Generate a unique filename"""
 
         title = self.webdriver.title
-        url = self._current_url
+        url = self.current_url
         hostname = urlparse(url).hostname
 
         hostname_ = Sanitation.ascii_numeric_only(hostname)
@@ -176,10 +180,18 @@ async def add_cookie(self, cookie_dict: dict) -> bool:
         result = self.webdriver.add_cookie(cookie_dict=cookie_dict)
 
         if result is None:
-            logger.debug(f'{cookie_dict}')
+            logger.debug(dict(
+                domain=cookie_dict.get('domain'),
+                path=cookie_dict.get('path'),
+                secure=cookie_dict.get('secure'),
+                expiry=cookie_dict.get('expiry'),
+                name=cookie_dict.get('name'),
+            ))
             return True
 
-        logger.error(f'{cookie_dict}')
+        logger.error(dict(
+            add_cookie=cookie_dict
+        ))
         return False
 
     async def add_cookie_from_file(self, file: str) -> bool:
@@ -197,7 +209,9 @@ async def add_cookies_from_list(self, cookies_list: list) -> bool:
         for cookie in cookies_list:
             await self.add_cookie(cookie_dict=cookie)
 
-        logger.debug(f'{True}')
+        logger.debug(dict(
+            add_cookies_from_list=len(cookies_list)
+        ))
         return True
 
     async def add_cookie_from_current_url(self):
@@ -210,7 +224,7 @@ async def add_cookie_from_url(self, url: str) -> bool:
 
         if os.path.exists(cookie_file):
             logger.info(f'{cookie_file}')
-            return self.add_cookie_from_file(file=cookie_file)
+            return await self.add_cookie_from_file(file=cookie_file)
 
         logger.error(f'{cookie_file}')
 
@@ -225,6 +239,11 @@ async def add_cookie_from_base64(self, base64_str: str) -> bool:
         logger.error(f'{base64_str}')
         return False
 
+    async def autosave_cookies(self) -> bool:
+        if self.current_url:
+            await self.save_cookies_for_current_url()
+            return await self.load_cookies_for_current_url()
+
     async def delete_all_cookies(self) -> None:
         result = self.webdriver.delete_all_cookies()
         logger.info(f'{True}')
@@ -234,7 +253,9 @@ async def _url_filename(self, url: str):
         parsed = await self.urlparse(url)
         hostname = parsed.hostname
         cookie_file = f'cookies-{hostname}.txt'
-        logger.info(f'{cookie_file}')
+        logger.info(dict(
+            _url_filename=cookie_file
+        ))
         return cookie_file
 
     async def get_cookie(self, name: str) -> dict:
@@ -243,24 +264,28 @@ async def get_cookie(self, name: str) -> dict:
         return result
 
     async def get_cookies(self) -> [dict]:
-        result = self.webdriver.get_cookies()
-        logger.debug(f'{True}')
-        return result
+        cookies = self.webdriver.get_cookies()
+        logger.debug(dict(
+            get_cookies=len(cookies)
+        ))
+        return cookies
 
     async def get_cookies_base64(self) -> base64:
-        result = self.get_cookies()
+        cookies = await self.get_cookies()
         logger.debug(f'{True}')
         return base64.b64encode(
-            json.dumps(result).encode()
+            json.dumps(cookies).encode()
         ).decode()
 
     async def get_cookies_json(self) -> json.dumps:
-        cookies = self.get_cookies()
-        logger.debug(f'{True}')
+        cookies = await self.get_cookies()
+        logger.debug(dict(
+            get_cookies_json=len(cookies)
+        ))
         return json.dumps(cookies)
 
     async def get_cookies_summary(self):
-        result = self.get_cookies()
+        result = await self.get_cookies()
         summary = {}
         if result:
             for cookie in result:
@@ -338,11 +363,15 @@ async def get(self, url: str, **kwargs) -> bool:
                     current_url=self.current_url,
                     kwargs=kwargs
                 )))
+
+            if self.config.cookies_autosave:
+                await self.autosave_cookies()
+
             return True
         except Exception as error:
-            logger.error(str(dict(
+            logger.error(dict(
                 error=error,
-            )))
+            ))
 
         return False
 
@@ -360,7 +389,7 @@ async def get_page_source_beautifulsoup(
             features: str = 'lxml') -> BeautifulSoup:
         """read page source with beautifulsoup"""
         if not markdup:
-            markdup = self.get_page_source()
+            markdup = await self.get_page_source()
         return BeautifulSoup(
             markup=markdup,
             features=features)
@@ -408,9 +437,19 @@ async def is_running(self) -> bool:
         logger.error(f'{False}')
         return False
 
+    async def load_cookies_for_current_url(self) -> bool:
+        filename = await self._url_filename(url=self.url)
+        logger.info(dict(
+            load_cookies_for_current_url=filename,
+            url=self.url,
+        ))
+        return await self.add_cookie_from_file(file=filename)
+
     async def urlparse(self, url: str):
         parsed = urlparse(url=url)
-        logger.debug(f'{parsed}')
+        logger.debug(dict(
+            urlparse=parsed
+        ))
         return parsed
 
     async def quit(self) -> bool:
@@ -432,26 +471,37 @@ async def quit(self) -> bool:
     async def run(self):
         """run browser"""
         try:
-            await self.config.run()
-        except:
+            return await self.config.run()
+        except Exception as error:
+            logger.error(dict(
+                error=error
+            ))
             return False
 
-    async def save_cookies_for_current_url(self):
-        filename = self._url_filename(url=self.url)
-        logger.info(f'{filename}')
+    async def save_cookies_for_current_url(self) -> bool:
+        filename = await self._url_filename(url=self.url)
+        logger.info(dict(
+            save_cookies_for_current_url=filename,
+            url=self.url,
+        ))
         return await self.save_cookies_to_file(file=filename)
 
-    async def save_cookies_to_file(self, file: str):
+    async def save_cookies_to_file(self, file: str) -> bool:
         with open(file, 'w') as cookies:
             cookies.write(
                 await self.get_cookies_json()
             )
 
         if os.path.exists(file):
-            logger.info(f'{os.path.abspath(file)} ({os.stat(file).st_size} B)')
+            logger.info(dict(
+                save_cookies_to_file=os.path.abspath(file),
+                bytes=os.stat(file).st_size
+            ))
             return True
 
-        logger.error(f'{file}')
+        logger.error(dict(
+            file=file
+        ))
         return False
 
     async def save_screenshot(

automon/integrations/seleniumWrapper/config.py

@@ -12,6 +12,7 @@ def __init__(self):
         self._webdriver = None
         self.webdriver_wrapper = None
 
+        self.cookies_autosave: bool = environ('SELENIUM_COOKIES_AUTOSAVE', False)
         self._cookies_base64 = environ('SELENIUM_COOKIES_BASE64')
         self._cookies_file = environ('SELENIUM_COOKIES_FILE')
 

automon/integrations/seleniumWrapper/tests/test_browser.py

@@ -12,7 +12,7 @@ class SeleniumClientTest(unittest.TestCase):
     if asyncio.run(browser.run()):
 
         def test_fake_page(self):
-            self.assertFalse(browser.get('http://555.555.555.555'))
+            self.assertFalse(asyncio.run(browser.get('http://555.555.555.555')))
 
         def test_real_page(self):
             if asyncio.run(browser.get('http://1.1.1.1')):
@@ -31,6 +31,8 @@ def test_screenshot_file(self):
                 self.assertTrue(asyncio.run(browser.save_screenshot()))
                 self.assertTrue(asyncio.run(browser.save_screenshot(folder='./')))
 
+    asyncio.run(browser.quit())
+
 
 if __name__ == '__main__':
     unittest.main()

automon/integrations/seleniumWrapper/tests/test_browser_cookies_autosave.py

@@ -0,0 +1,27 @@
+import unittest
+import asyncio
+
+from automon.integrations.seleniumWrapper import SeleniumBrowser, ChromeWrapper
+
+
+class Test(unittest.TestCase):
+    browser = SeleniumBrowser()
+    browser.config.webdriver_wrapper = ChromeWrapper()
+    browser.config.webdriver_wrapper.enable_defaults().enable_headless()
+
+    # if asyncio.run(browser.run()):
+    asyncio.run(browser.run())
+
+    def test_autosave(self):
+        if asyncio.run(self.browser.run()):
+
+            asyncio.run(self.browser.set_window_size(device_type='web-large'))
+
+            if asyncio.run(self.browser.get('http://bing.com')):
+                self.assertTrue(asyncio.run(self.browser.autosave_cookies()))
+
+            asyncio.run(self.browser.quit())
+
+
+if __name__ == '__main__':
+    unittest.main()

automon/integrations/seleniumWrapper/tests/test_user_agent.py

@@ -12,7 +12,6 @@ def test_filter(self):
         self.assertFalse(test.filter_agent('xxxxx'))
         self.assertFalse(test.filter_agent('xxxxx', case_sensitive=True))
 
-
     def test_random(self):
         test = SeleniumUserAgentBuilder()
         self.assertTrue(test.get_random_agent('applewebkit'))

automon/integrations/seleniumWrapper/webdriver_chrome.py

@@ -24,9 +24,6 @@ def __init__(self):
 
         self.update_paths(self.chromedriver_path)
 
-        if not self.chromedriver_path:
-            logger.error('missing SELENIUM_CHROMEDRIVER_PATH')
-
     def __repr__(self):
         if self._webdriver:
             return str(dict(
@@ -59,6 +56,8 @@ def chromedriver_path(self):
             if os.path.exists(path):
                 return path
 
+        logger.error('missing SELENIUM_CHROMEDRIVER_PATH')
+
     @property
     def chromedriverVersion(self):
         if self.webdriver:
@@ -304,7 +303,7 @@ def in_sandbox_disabled(self):
         self.disable_sandbox()
         return self
 
-    async def run(self) -> selenium.webdriver.Chrome:
+    async def run(self) -> bool:
         try:
             if self.chromedriver_path:
                 self._ChromeService = selenium.webdriver.ChromeService(
@@ -320,12 +319,12 @@ async def run(self) -> selenium.webdriver.Chrome:
                 )
                 logger.info(f'{self}')
 
-                return self.webdriver
+                return True
 
             self._webdriver = selenium.webdriver.Chrome(options=self.chrome_options)
             logger.info(f'{self}')
 
-            return self.webdriver
+            return True
         except Exception as error:
             logger.error(f'{error}')
             raise Exception(error)
@@ -400,9 +399,7 @@ def update_paths(self, path: str):
 
                 return True
 
-        logger.error(dict(
-            chromedriver_path=path
-        ))
+        return False
 
     async def quit(self):
         """quit

env-example.sh

@@ -101,6 +101,7 @@ TWINE_PASSWORD=
 # Selenium
 SELENIUM_CHROMEDRIVER_PATH=
 SELENIUM_OPT=
+SELENIUM_COOKIES_AUTOSAVE=False
 SELENIUM_COOKIES_BASE64=
 SELENIUM_COOKIES_FILE=