From 62ca361a46bf7bf9554f241a1647678ad615bb5d Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 17 Apr 2023 18:50:18 +0800 Subject: [PATCH 01/62] instagram: update config --- automon/integrations/instagram/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/automon/integrations/instagram/config.py b/automon/integrations/instagram/config.py index 53e154c5..7055bb64 100644 --- a/automon/integrations/instagram/config.py +++ b/automon/integrations/instagram/config.py @@ -20,6 +20,4 @@ def is_configured(self): return False def __repr__(self): - if self.is_configured: - return f'ready' - return f'not ready' + return f'{self.login}' From 38dc5e354b29daed3ca5a159c5817342258aa902 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 17 Apr 2023 18:51:09 +0800 Subject: [PATCH 02/62] instagram: support browser_options --- automon/integrations/instagram/client_browser.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/automon/integrations/instagram/client_browser.py b/automon/integrations/instagram/client_browser.py index 04349f6f..07e03ef3 100644 --- a/automon/integrations/instagram/client_browser.py +++ b/automon/integrations/instagram/client_browser.py @@ -23,14 +23,16 @@ def __init__(self, login: str = None, password: str = None, config: InstagramConfig = None, - headless: bool = True): + headless: bool = True, + browser_options: list = None): """Instagram Browser Client""" self.config = config or InstagramConfig(login=login, password=password) self.browser = SeleniumBrowser() - self.browser.set_browser(self.browser.type.chrome()) if headless: - self.browser.set_browser(self.browser.type.chrome_headless()) + self.browser.set_browser(self.browser.type.chrome_headless(options=browser_options)) + else: + self.browser.set_browser(self.browser.type.chrome(options=browser_options)) def __repr__(self): return f'{self.__dict__}' @@ -42,6 +44,7 @@ def wrapped(self, *args, **kwargs): if self.browser.is_running(): return func(self, *args, **kwargs) return False + return wrapped def _is_authenticated(func): From 36f2a1b0e0fdc60cb767fa41e4b1a70a5ff5dbbf Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 17 Apr 2023 19:16:47 +0800 Subject: [PATCH 03/62] instagram: use a random user agent --- automon/integrations/instagram/client_browser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/automon/integrations/instagram/client_browser.py b/automon/integrations/instagram/client_browser.py index 07e03ef3..7d32c061 100644 --- a/automon/integrations/instagram/client_browser.py +++ b/automon/integrations/instagram/client_browser.py @@ -29,6 +29,12 @@ def __init__(self, self.config = config or InstagramConfig(login=login, password=password) self.browser = SeleniumBrowser() + useragent = self.browser.get_random_user_agent() + if browser_options: + browser_options.extend([f"user-agent={useragent}"]) + else: + browser_options = [f"user-agent={useragent}"] + if headless: self.browser.set_browser(self.browser.type.chrome_headless(options=browser_options)) else: From 03562f458bf1f76de49cae16dc4b81de7383a003 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 17 Apr 2023 19:17:07 +0800 Subject: [PATCH 04/62] instagram: update authenticate --- automon/integrations/instagram/client_browser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automon/integrations/instagram/client_browser.py b/automon/integrations/instagram/client_browser.py index 7d32c061..8ad5e478 100644 --- a/automon/integrations/instagram/client_browser.py +++ b/automon/integrations/instagram/client_browser.py @@ -173,8 +173,8 @@ def authenticate(self): self.browser.get(self.urls.login_page) # user - self.browser.wait_for_xpath(self.xpaths.login_user) - self.browser.action_click(self.xpaths.login_user, 'user') + login_user = self.browser.wait_for_xpath(self.xpaths.login_user) + self.browser.action_click(login_user, 'user') self.browser.action_type(self.login) # password From 147278eca3d57d75e0754ee0ad082b774c2b8685 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 17 Apr 2023 19:17:14 +0800 Subject: [PATCH 05/62] instagram: update xpaths --- automon/integrations/instagram/xpaths.py | 33 +++++++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/automon/integrations/instagram/xpaths.py b/automon/integrations/instagram/xpaths.py index aee13f0a..2d5e20db 100644 --- a/automon/integrations/instagram/xpaths.py +++ b/automon/integrations/instagram/xpaths.py @@ -5,19 +5,31 @@ def __repr__(self): @property def login_user(self): - return '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[1]/div/label/input' + return [ + '/html/body/div[2]/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[1]/div/label/input', + '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[1]/div/label/input', + ] @property def login_pass(self): - return '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[2]/div/label/input' + return [ + '/html/body/div[2]/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[2]/div/label/input', + '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[2]/div/label/input' + ] @property def login_btn(self): - return '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[3]/button' + return [ + '/html/body/div[2]/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[3]/button', + '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[3]/button' + ] @property def profile_picture(self): - return '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/nav/div[2]/div/div/div[3]/div/div[6]' + return [ + '/html/body/div[2]/div/div/div[2]/div/div/div/div[1]/div[1]/div[2]/section/main/div[1]/section/div[3]/div[1]/div/div/div/div/div/div[1]/div/div/span/img', + '/html/body/div[2]/div/div/div[2]/div/div/div/div[1]/div[1]/div[2]/section/main/div[1]/section/div/div[2]/div/div/div/div/ul/li[3]/div/button/div[1]/span/img', + ] @property def save_info(self): @@ -25,12 +37,19 @@ def save_info(self): @property def save_info_not_now(self): - return '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div/div/button' + return ['/html/body/div[2]/div/div/div[2]/div/div/div/div[1]/div[1]/div[2]/section/main/div/div/div/div', + '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div/div/button'] @property def turn_on_notifications(self): - return '/html/body/div[2]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[1]' + return [ + '/html/body/div[2]/div/div/div[3]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[1]', + '/html/body/div[2]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[1]' + ] @property def turn_on_notifications_not_now(self): - return '/html/body/div[2]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[2]' \ No newline at end of file + return [ + '/html/body/div[2]/div/div/div[3]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[2]', + '/html/body/div[2]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[2]' + ] From 172df09b184f551a836814d55cd641cfb52fadcb Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 17 Apr 2023 19:17:26 +0800 Subject: [PATCH 06/62] instagram: tests --- .../tests/test_instagram_browser_auth.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 automon/integrations/instagram/tests/test_instagram_browser_auth.py diff --git a/automon/integrations/instagram/tests/test_instagram_browser_auth.py b/automon/integrations/instagram/tests/test_instagram_browser_auth.py new file mode 100644 index 00000000..5b8d8ce7 --- /dev/null +++ b/automon/integrations/instagram/tests/test_instagram_browser_auth.py @@ -0,0 +1,21 @@ +import unittest + +from automon.integrations.instagram.client_browser import InstagramBrowserClient + +c = InstagramBrowserClient(headless=False) + + +class InstagramClientTest(unittest.TestCase): + if c.is_running(): + c.browser.get(c.urls.login_page) + + # user + login_user = c.browser.wait_for_xpath(c.xpaths.login_user) + c.browser.action_click(login_user, 'user') + c.browser.action_type(c.login) + + c.browser.quit() + + +if __name__ == '__main__': + unittest.main() From 11412c61a2742a71cd146eff377d416aaab36a41 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 19 Apr 2023 04:49:50 +0800 Subject: [PATCH 07/62] instagram: urls for followers and following --- automon/integrations/instagram/urls.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/automon/integrations/instagram/urls.py b/automon/integrations/instagram/urls.py index 0e8786d0..94081889 100644 --- a/automon/integrations/instagram/urls.py +++ b/automon/integrations/instagram/urls.py @@ -6,3 +6,11 @@ def __repr__(self): @property def login_page(self): return 'https://www.instagram.com/accounts/login/?source=auth_switcher' + + @staticmethod + def followers(account: str): + return f'https://www.instagram.com/{account}/followers/' + + @staticmethod + def following(account: str): + return f'https://www.instagram.com/{account}/following/' From 598d2c90fd3df34497e5eb3fcbbaa34f82687acd Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 19 Apr 2023 04:49:57 +0800 Subject: [PATCH 08/62] instagram: update xpaths --- automon/integrations/instagram/xpaths.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/automon/integrations/instagram/xpaths.py b/automon/integrations/instagram/xpaths.py index 2d5e20db..5b2c64ac 100644 --- a/automon/integrations/instagram/xpaths.py +++ b/automon/integrations/instagram/xpaths.py @@ -21,6 +21,7 @@ def login_pass(self): def login_btn(self): return [ '/html/body/div[2]/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[3]/button', + '/html/body/div[2]/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[4]/button', '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[3]/button' ] @@ -35,10 +36,19 @@ def profile_picture(self): def save_info(self): return '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div/section/div/button' + @property + def save_info_not_now_div(self): + return [ + '/html/body/div[2]/div/div/div[3]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div', + ] + @property def save_info_not_now(self): - return ['/html/body/div[2]/div/div/div[2]/div/div/div/div[1]/div[1]/div[2]/section/main/div/div/div/div', - '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div/div/button'] + return [ + '/html/body/div[2]/div/div/div[3]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[2]', + '/html/body/div[2]/div/div/div[2]/div/div/div/div[1]/div[1]/div[2]/section/main/div/div/div/div', + '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div/div/button' + ] @property def turn_on_notifications(self): From 1230bc9477d76947422050fed668d805dfc2e5de Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 19 Apr 2023 04:50:01 +0800 Subject: [PATCH 09/62] instagram: update xpaths --- .../integrations/instagram/client_browser.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/automon/integrations/instagram/client_browser.py b/automon/integrations/instagram/client_browser.py index 8ad5e478..12d207b3 100644 --- a/automon/integrations/instagram/client_browser.py +++ b/automon/integrations/instagram/client_browser.py @@ -62,22 +62,22 @@ def wrapped(self, *args, **kwargs): return wrapped - def _get_page(self, account): + def get_page(self, account: str): """ Get page """ - log.debug('[_get_page] getting {}'.format(account)) + log.debug(f'[get_page] getting {account}') - page = 'https://instagram.com/{}'.format(account) + page = f'https://instagram.com/{account}' browser = self.authenticated_browser return browser.get(page) - def _get_stories(self, account): + def get_stories(self, account: str): """ Retrieve story """ - story = 'https://www.instagram.com/stories/{}/'.format(account) + story = f'https://www.instagram.com/stories/{account}/' num_of_stories = 0 - log.debug('[get_stories] {}'.format(story)) + log.debug(f'[get_stories] {story}') browser = self.authenticated_browser browser.get(story) @@ -85,7 +85,7 @@ def _get_stories(self, account): prefix='instagram/' + account) if 'Page Not Found' in browser.browser.title: - log.debug('[get_stories] no stories for {}'.format(account)) + log.debug(f'[get_stories] no stories for {account}') return num_of_stories Sleeper.seconds('instagram', 2) @@ -181,14 +181,18 @@ def authenticate(self): login_pass = self.browser.wait_for_xpath(self.xpaths.login_pass) self.browser.action_click(login_pass, 'login') self.browser.action_type(self.config.password, secret=True) + self.browser.action_type(self.browser.keys.ENTER) # login - login_btn = self.browser.wait_for_xpath(self.xpaths.login_btn) - self.browser.action_click(login_btn, 'login button') + # login_btn = self.browser.wait_for_xpath(self.xpaths.login_btn) + # self.browser.action_click(login_btn, 'login button') # check for "save your login info" dialogue - not_now = self.browser.wait_for_xpath(self.xpaths.save_info_not_now) - self.browser.action_click(not_now, 'dont save login info') + not_now = self.browser.wait_for_xpath(self.xpaths.save_info_not_now_div) + self.browser.action_type(self.browser.keys.TAB) + self.browser.action_type(self.browser.keys.TAB) + self.browser.action_type(self.browser.keys.ENTER) + # self.browser.action_click(not_now, 'dont save login info') # check for "notifications" dialogue notifications_not_now = self.browser.wait_for_xpath(self.xpaths.turn_on_notifications_not_now) @@ -204,6 +208,12 @@ def authenticate(self): return False + @_is_running + @_is_authenticated + def get_followers(self, account: str): + url = self.urls.followers(account) + self.browser.get(url) + @_is_running @_is_authenticated def is_authenticated(self): @@ -224,13 +234,3 @@ def urls(self): @property def xpaths(self): return XPaths() - - -def get_page(authenticated_browser, account): - """ Get page - """ - # TODO: need to download page - log.debug('[get_page] getting {}'.format(account)) - page = 'https://instagram.com/{}'.format(account) - browser = authenticated_browser - return browser.get(page) From e7c4fa31017cfcc6c852613ce1e421e2df52ddd0 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Sat, 22 Apr 2023 02:43:00 +0800 Subject: [PATCH 10/62] selenium: get current page with beautifulsoup --- automon/integrations/seleniumWrapper/browser.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 6e47a907..f57f2d06 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -6,6 +6,7 @@ from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from urllib.parse import urlparse +from bs4 import BeautifulSoup from automon.log import Logging from automon.helpers.dates import Dates @@ -109,7 +110,7 @@ def action_click(self, xpath: str, note: str = None) -> str or False: return False @_is_running - def action_type(self, key: str or Keys, secret: bool = False): + def action_type(self, key: str or Keys, secret: bool = False, ): """perform keyboard command""" try: actions = selenium.webdriver.common.action_chains.ActionChains( @@ -171,10 +172,17 @@ def get_page(self, *args, **kwargs): return self.get(*args, **kwargs) @_is_running - def get_page_source(self): + def get_page_source(self) -> str: """get page source""" return self.driver.page_source + @_is_running + def get_page_source_beautifulsoup(self, markdup: str = None, features: str = 'lxml') -> BeautifulSoup: + """read page source with beautifulsoup""" + if not markdup: + markdup = self.get_page_source() + return BeautifulSoup(markup=markdup, features=features) + def get_random_user_agent(self, filter: list or str = None, case_sensitive: bool = False) -> list: return SeleniumUserAgentBuilder().get_random(filter=filter, case_sensitive=case_sensitive) From 6adea05f44bab58dc81bbfeff987535c35f6bd25 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Sat, 22 Apr 2023 02:44:41 +0800 Subject: [PATCH 11/62] selenium: add authenticated xpaths --- automon/integrations/instagram/xpaths.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/automon/integrations/instagram/xpaths.py b/automon/integrations/instagram/xpaths.py index 5b2c64ac..fded5aa9 100644 --- a/automon/integrations/instagram/xpaths.py +++ b/automon/integrations/instagram/xpaths.py @@ -25,6 +25,19 @@ def login_btn(self): '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/div[1]/div[2]/form/div/div[3]/button' ] + @property + def authenticated_paths(self): + authenticated = [] + authenticated.extend(self.profile_picture) + authenticated.extend(self.home) + return authenticated + + @property + def home(self): + return [ + '/html/body/div[2]/div/div/div[2]/div/div/div/div[1]/div[1]/div[1]/div/div/div/div/div[2]/div[1]/div/div/a/div', + ] + @property def profile_picture(self): return [ From 1c0862a8ee4e43ca547fbd4f2bb3417ee469bd3f Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Fri, 28 Apr 2023 04:46:38 +0900 Subject: [PATCH 12/62] selenium: small typo --- automon/integrations/seleniumWrapper/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index f57f2d06..6c75d4bc 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -110,7 +110,7 @@ def action_click(self, xpath: str, note: str = None) -> str or False: return False @_is_running - def action_type(self, key: str or Keys, secret: bool = False, ): + def action_type(self, key: str or Keys, secret: bool = False): """perform keyboard command""" try: actions = selenium.webdriver.common.action_chains.ActionChains( From 31db9286d74025f6b167ff20e1099ac70b75c8df Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Fri, 28 Apr 2023 07:38:19 +0900 Subject: [PATCH 13/62] selenium: move set_window_size in config --- .../integrations/seleniumWrapper/browser.py | 59 ++++--------------- .../integrations/seleniumWrapper/config.py | 53 +++++++++++++++++ .../tests/test_browser_headless.py | 2 +- 3 files changed, 65 insertions(+), 49 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 6c75d4bc..f2aa9fc8 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -29,7 +29,6 @@ def __init__(self, config: SeleniumConfig = None): self.config = config or SeleniumConfig() self.driver = 'not set' or self.type.chrome_headless() - self.window_size = '' self.status = '' def __repr__(self): @@ -37,6 +36,10 @@ def __repr__(self): return f'{self.browser.name} {self.status} {self.browser.current_url} {self.window_size}' return f'{self.browser.name} {self.window_size}' + @property + def config_browser(self): + return self.config + @property def browser(self): """alias to selenium driver""" @@ -67,6 +70,10 @@ def url(self): return '' return self.browser.current_url + @property + def window_size(self): + return self.config.window_size + def _is_running(func) -> functools.wraps: @functools.wraps(func) def wrapped(self, *args, **kwargs): @@ -258,57 +265,13 @@ def set_driver(self, driver: SeleniumBrowserType) -> True: return True @_is_running - def set_resolution(self, width=1920, height=1080, device_type=None) -> bool: + def set_window_size(self, width=1920, height=1080, device_type=None) -> bool: """set browser resolution""" - if device_type == 'pixel3': - width = 1080 - height = 2160 - - if device_type == 'web-small' or device_type == '800x600': - width = 800 - height = 600 - - if device_type == 'web-small-2' or device_type == '1024x768': - width = 1024 - height = 768 - - if device_type == 'web-small-3' or device_type == '1280x960': - width = 1280 - height = 960 - - if device_type == 'web-small-4' or device_type == '1280x1024': - width = 1280 - height = 1024 - - if device_type == 'web' or device_type == '1920x1080': - width = 1920 - height = 1080 - - if device_type == 'web-2' or device_type == '1600x1200': - width = 1600 - height = 1200 - - if device_type == 'web-3' or device_type == '1920x1200': - width = 1920 - height = 1200 - - if device_type == 'web-large' or device_type == '2560x1400': - width = 2560 - height = 1400 - - if device_type == 'web-long' or device_type == '1920x3080': - width = 1920 - height = 3080 - - if not width and not height: - width = 1920 - height = 1080 - - self.window_size = width, height + self.config_browser.set_window_size(width=width, height=height, device_type=device_type) try: - self.browser.set_window_size(width, height) + self.browser.set_window_size(self.config.window_size) except Exception as error: log.error(f'failed to set resolution. {error}') return False diff --git a/automon/integrations/seleniumWrapper/config.py b/automon/integrations/seleniumWrapper/config.py index c2fb4ef7..3aaceedc 100644 --- a/automon/integrations/seleniumWrapper/config.py +++ b/automon/integrations/seleniumWrapper/config.py @@ -20,9 +20,62 @@ def __init__(self, webdriver=None, chromedriver: str = None): if self.selenium_chromedriver_path: os.environ['PATH'] = f"{os.getenv('PATH')}:{self.selenium_chromedriver_path}" + self.window_size = '' + def __repr__(self): return f'{self.__dict__}' + def set_window_size(self, width=1920, height=1080, device_type=None): + """set browser resolution""" + + if device_type == 'pixel3': + width = 1080 + height = 2160 + + if device_type == 'web-small' or device_type == '800x600': + width = 800 + height = 600 + + if device_type == 'web-small-2' or device_type == '1024x768': + width = 1024 + height = 768 + + if device_type == 'web-small-3' or device_type == '1280x960': + width = 1280 + height = 960 + + if device_type == 'web-small-4' or device_type == '1280x1024': + width = 1280 + height = 1024 + + if device_type == 'web' or device_type == '1920x1080': + width = 1920 + height = 1080 + + if device_type == 'web-2' or device_type == '1600x1200': + width = 1600 + height = 1200 + + if device_type == 'web-3' or device_type == '1920x1200': + width = 1920 + height = 1200 + + if device_type == 'web-large' or device_type == '2560x1400': + width = 2560 + height = 1400 + + if device_type == 'web-long' or device_type == '1920x3080': + width = 1920 + height = 3080 + + if not width and not height: + width = 1920 + height = 1080 + + self.window_size = width, height + + return self + def chrome(self): """Chrome with no options diff --git a/automon/integrations/seleniumWrapper/tests/test_browser_headless.py b/automon/integrations/seleniumWrapper/tests/test_browser_headless.py index 217bbc79..3acea008 100644 --- a/automon/integrations/seleniumWrapper/tests/test_browser_headless.py +++ b/automon/integrations/seleniumWrapper/tests/test_browser_headless.py @@ -4,7 +4,7 @@ browser = SeleniumBrowser() browser.set_driver(browser.type.chrome_headless()) -browser.set_resolution(device_type='web-large') +browser.set_window_size(device_type='web-large') class SeleniumClientTest(unittest.TestCase): From e0d7c115d05b9af3278cf4d63428185629578ca0 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 26 Jun 2023 07:20:56 -0400 Subject: [PATCH 14/62] selenium: refactor browser and config --- .../integrations/seleniumWrapper/browser.py | 98 +++--- .../seleniumWrapper/browser_capabilities.py | 20 ++ .../integrations/seleniumWrapper/config.py | 255 ++-------------- .../seleniumWrapper/config_webdriver.py | 88 ++++++ .../config_webdriver_chrome.py | 284 ++++++++++++++++++ .../seleniumWrapper/config_window_size.py | 48 +++ .../seleniumWrapper/tests/test_browser.py | 4 +- .../tests/test_browser_headless.py | 7 +- .../tests/test_browser_useragent.py | 6 +- .../seleniumWrapper/tests/test_new_browser.py | 16 + 10 files changed, 534 insertions(+), 292 deletions(-) create mode 100644 automon/integrations/seleniumWrapper/browser_capabilities.py create mode 100644 automon/integrations/seleniumWrapper/config_webdriver.py create mode 100644 automon/integrations/seleniumWrapper/config_webdriver_chrome.py create mode 100644 automon/integrations/seleniumWrapper/config_window_size.py create mode 100644 automon/integrations/seleniumWrapper/tests/test_new_browser.py diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index f2aa9fc8..d8059e59 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -2,6 +2,7 @@ import tempfile import functools import selenium +import selenium.webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys @@ -22,38 +23,42 @@ class SeleniumBrowser(object): config: SeleniumConfig - type: SeleniumBrowserType + webdriver: selenium.webdriver + status: str def __init__(self, config: SeleniumConfig = None): """A selenium wrapper""" - self.config = config or SeleniumConfig() - self.driver = 'not set' or self.type.chrome_headless() + self._config = config or SeleniumConfig() self.status = '' def __repr__(self): if self.url: - return f'{self.browser.name} {self.status} {self.browser.current_url} {self.window_size}' - return f'{self.browser.name} {self.window_size}' - - @property - def config_browser(self): - return self.config + return f'{self.webdriver.name} {self.status} {self.webdriver.current_url} {self.window_size}' + return f'{self.webdriver}' @property def browser(self): - """alias to selenium driver""" - return self.driver + """alias to webdriver""" + return self.webdriver @property def by(self) -> By: """Set of supported locator strategies""" return selenium.webdriver.common.by.By() + @property + def config(self): + return self._config + + @property + def webdriver(self): + return self.config.webdriver + @property def get_log(self, log_type: str = 'browser') -> list: """Gets the log for a given log type""" - return self.browser.get_log(log_type) + return self.webdriver.get_log(log_type) @property def keys(self): @@ -61,23 +66,23 @@ def keys(self): return selenium.webdriver.common.keys.Keys @property - def type(self): + def type(self) -> SeleniumBrowserType: return SeleniumBrowserType(self.config) @property def url(self): - if self.browser.current_url == 'data:,': + if self.webdriver.current_url == 'data:,': return '' - return self.browser.current_url + return self.webdriver.current_url @property def window_size(self): - return self.config.window_size + return self.config.set_webdriver.window_size def _is_running(func) -> functools.wraps: @functools.wraps(func) def wrapped(self, *args, **kwargs): - if self.browser != 'not set': + if self.webdriver is not None: return func(self, *args, **kwargs) log.error(f'Browser is not set!', enable_traceback=False) return False @@ -87,8 +92,8 @@ def wrapped(self, *args, **kwargs): def _screenshot_name(self, prefix=None): """Generate a unique filename""" - title = self.browser.title - url = self.browser.current_url + title = self.webdriver.title + url = self.webdriver.current_url hostname = urlparse(url).hostname hostname_ = Sanitation.ascii_numeric_only(hostname) @@ -121,7 +126,7 @@ def action_type(self, key: str or Keys, secret: bool = False): """perform keyboard command""" try: actions = selenium.webdriver.common.action_chains.ActionChains( - self.browser) + self.webdriver) actions.send_keys(key) actions.perform() @@ -138,16 +143,16 @@ def action_type(self, key: str or Keys, secret: bool = False): def close(self): """close browser""" log.info(f'Browser closed') - self.browser.close() + self.webdriver.close() @_is_running def find_element( self, value: str, - by: By = By.ID, + by: By.ID = By.ID, **kwargs): """find element""" - return self.browser.find_element(value=value, by=by, **kwargs) + return self.webdriver.find_element(value=value, by=by, **kwargs) @_is_running def find_xpath(self, value: str, by: By = By.XPATH, **kwargs): @@ -158,10 +163,10 @@ def find_xpath(self, value: str, by: By = By.XPATH, **kwargs): def get(self, url: str, **kwargs) -> bool: """get url""" try: - self.browser.get(url, **kwargs) + self.webdriver.get(url, **kwargs) self.status = 'OK' - msg = f'GET {self.status} {self.browser.current_url}' + msg = f'GET {self.status} {self.webdriver.current_url}' if kwargs: msg += f', {kwargs}' log.debug(msg) @@ -181,7 +186,7 @@ def get_page(self, *args, **kwargs): @_is_running def get_page_source(self) -> str: """get page source""" - return self.driver.page_source + return self.webdriver.page_source @_is_running def get_page_source_beautifulsoup(self, markdup: str = None, features: str = 'lxml') -> BeautifulSoup: @@ -196,16 +201,16 @@ def get_random_user_agent(self, filter: list or str = None, case_sensitive: bool @_is_running def get_screenshot_as_png(self, **kwargs): """screenshot as png""" - return self.browser.get_screenshot_as_png(**kwargs) + return self.webdriver.get_screenshot_as_png(**kwargs) @_is_running def get_screenshot_as_base64(self, **kwargs): """screenshot as base64""" - return self.browser.get_screenshot_as_base64(**kwargs) + return self.webdriver.get_screenshot_as_base64(**kwargs) @_is_running def get_user_agent(self): - return self.browser.execute_script("return navigator.userAgent") + return self.webdriver.execute_script("return navigator.userAgent") @_is_running def is_running(self) -> bool: @@ -216,9 +221,9 @@ def is_running(self) -> bool: def quit(self) -> bool: """gracefully quit browser""" try: - self.browser.close() - self.browser.quit() - self.browser.stop_client() + self.webdriver.close() + self.webdriver.quit() + self.webdriver.stop_client() except Exception as error: log.error(f'failed to quit browser. {error}') return False @@ -248,35 +253,28 @@ def save_screenshot( log.info(f'Saving screenshot to: {save}') - return self.browser.save_screenshot(save, **kwargs) - - def set_browser(self, browser: SeleniumBrowserType) -> True: - """set browser driver""" - return self.set_driver(driver=browser) - - def set_driver(self, driver: SeleniumBrowserType) -> True: - """set driver - - setting driver will launch browser - """ - if driver: - self.driver = driver - log.info(f'Launching {self.browser.name}') - return True + return self.webdriver.save_screenshot(save, **kwargs) @_is_running def set_window_size(self, width=1920, height=1080, device_type=None) -> bool: """set browser resolution""" - self.config_browser.set_window_size(width=width, height=height, device_type=device_type) - try: - self.browser.set_window_size(self.config.window_size) + self.config.set_webdriver.webdriver_wrapper.set_window_size(width=width, height=height, + device_type=device_type) except Exception as error: log.error(f'failed to set resolution. {error}') return False return True + def run(self): + """run browser""" + return self.config.set_webdriver.run() + + def start(self): + """alias to run""" + return self.run() + def wait_for( self, value: str or list, diff --git a/automon/integrations/seleniumWrapper/browser_capabilities.py b/automon/integrations/seleniumWrapper/browser_capabilities.py new file mode 100644 index 00000000..55fea37a --- /dev/null +++ b/automon/integrations/seleniumWrapper/browser_capabilities.py @@ -0,0 +1,20 @@ +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities + +caps = DesiredCapabilities.CHROME + #as per latest docs +caps['goog:loggingPrefs'] = {'performance': 'ALL'} +driver = webdriver.Chrome(desired_capabilities=caps) + + +class SeleniumDesiredCapabilities(DesiredCapabilities): + + def __init__(self): + pass + + @property + def DesiredCapabilities(self): + return DesiredCapabilities + + @property + def CHROME(self): + return self.DesiredCapabilities.CHROME \ No newline at end of file diff --git a/automon/integrations/seleniumWrapper/config.py b/automon/integrations/seleniumWrapper/config.py index 3aaceedc..0f9da585 100644 --- a/automon/integrations/seleniumWrapper/config.py +++ b/automon/integrations/seleniumWrapper/config.py @@ -1,245 +1,32 @@ -import os -import warnings -import selenium import selenium.webdriver from automon.log import Logging -from automon.helpers.osWrapper.environ import environ + +from .config_webdriver import ConfigWebdriver log = Logging(name='SeleniumConfig', level=Logging.INFO) class SeleniumConfig(object): - webdriver: selenium.webdriver - selenium_chromedriver_path: str - - def __init__(self, webdriver=None, chromedriver: str = None): - self.webdriver = webdriver or selenium.webdriver - self.selenium_chromedriver_path = chromedriver or environ('SELENIUM_CHROMEDRIVER_PATH', '') + webdriver_wrapper: ConfigWebdriver - if self.selenium_chromedriver_path: - os.environ['PATH'] = f"{os.getenv('PATH')}:{self.selenium_chromedriver_path}" - - self.window_size = '' + def __init__(self): + self._webdriver_wrapper = ConfigWebdriver() def __repr__(self): - return f'{self.__dict__}' - - def set_window_size(self, width=1920, height=1080, device_type=None): - """set browser resolution""" - - if device_type == 'pixel3': - width = 1080 - height = 2160 - - if device_type == 'web-small' or device_type == '800x600': - width = 800 - height = 600 - - if device_type == 'web-small-2' or device_type == '1024x768': - width = 1024 - height = 768 - - if device_type == 'web-small-3' or device_type == '1280x960': - width = 1280 - height = 960 - - if device_type == 'web-small-4' or device_type == '1280x1024': - width = 1280 - height = 1024 - - if device_type == 'web' or device_type == '1920x1080': - width = 1920 - height = 1080 - - if device_type == 'web-2' or device_type == '1600x1200': - width = 1600 - height = 1200 - - if device_type == 'web-3' or device_type == '1920x1200': - width = 1920 - height = 1200 - - if device_type == 'web-large' or device_type == '2560x1400': - width = 2560 - height = 1400 - - if device_type == 'web-long' or device_type == '1920x3080': - width = 1920 - height = 3080 - - if not width and not height: - width = 1920 - height = 1080 - - self.window_size = width, height - - return self - - def chrome(self): - """Chrome with no options - - """ - opt = SeleniumOptions(self.webdriver) - opt.default() - - def chrome_maximized(self): - """Chrome with no options - - """ - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.maximized() - - def chrome_fullscreen(self): - """Chrome with no options - - """ - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.fullscreen() - - def chrome_for_docker(self): - """Chrome best used with docker - - """ - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.nosandbox() - opt.headless() - opt.noinfobars() - opt.noextensions() - opt.nonotifications() - - def chrome_sandboxed(self): - """Chrome with sandbox enabled - - """ - warnings.warn('Docker does not support sandbox option') - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) - - opt = SeleniumOptions(self.webdriver) - opt.default() - - def chrome_nosandbox(self): - """Chrome with sandbox disabled - - """ - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) - - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.nosandbox() - - def chrome_headless_sandboxed(self): - """Headless Chrome with sandbox enabled - - """ - warnings.warn('Docker does not support sandbox option') - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) - - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.headless() - - def chrome_headless_nosandbox(self): - """Headless Chrome with sandbox disabled - - """ - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) - - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.headless() - opt.nosandbox() - - def chrome_headless_nosandbox_unsafe(self): - """Headless Chrome with sandbox disabled with not certificate verification - - """ - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) - - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.headless() - opt.nosandbox() - opt.unsafe() - - def chrome_headless_nosandbox_noshm(self): - """Headless Chrome with sandbox disabled - - """ - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.headless() - opt.nosandbox() - opt.noshm() - - def chrome_headless_nosandbox_bigshm(self): - """Headless Chrome with sandbox disabled - - """ - warnings.warn('Larger shm option is not implemented', Warning) - - opt = SeleniumOptions(self.webdriver) - opt.default() - opt.headless() - opt.nosandbox() - opt.bigshm() - - def chrome_remote(self, host: str = '127.0.0.1', port: str = '4444', executor_path: str = '/wd/hub'): - """Remote Selenium - - """ - log.info( - f'Remote WebDriver Hub URL: http://{host}:{port}{executor_path}/static/resource/hub.html') - - self.webdriver.Remote( - command_executor=f'http://{host}:{port}{executor_path}', - desired_capabilities=selenium.webdriver.common.desired_capabilities.DesiredCapabilities.CHROME - ) - - -class SeleniumOptions: - - def __init__(self, webdriver): - self.webdriver = webdriver or selenium.webdriver - self.options = self.webdriver.ChromeOptions() - - def default(self): - self.options.add_argument('start-maximized') - - def unsafe(self): - warnings.warn('Certificates are not verified', Warning) - self.options.add_argument('--ignore-certificate-errors') - - def nosandbox(self): - self.options.add_argument('--no-sandbox') - - def headless(self): - self.options.add_argument('headless') - - def noshm(self): - warnings.warn('Disabled shm will use disk I/O, and will be slow', Warning) - self.options.add_argument('--disable-dev-shm-usage') - - def bigshm(self): - warnings.warn('Big shm not yet implemented', Warning) - - def noinfobars(self): - self.options.add_argument("--disable-infobars") - - def noextensions(self): - self.options.add_argument("--disable-extensions") - - def maximized(self): - self.options.add_argument("--start-maximized") - - def fullscreen(self): - self.options.add_argument("--start-fullscreen") - - def nonotifications(self): - # Pass the argument 1 to allow and 2 to block - self.options.add_experimental_option( - "prefs", {"profile.default_content_setting_values.notifications": 1} - ) + return f'{self.driver}' + + @property + def set_webdriver(self): + return self._webdriver_wrapper + + @property + def driver(self): + return self.webdriver + + @property + def webdriver(self): + if self.set_webdriver.webdriver: + return self.set_webdriver.webdriver + else: + log.error('driver not set. configure a driver first') diff --git a/automon/integrations/seleniumWrapper/config_webdriver.py b/automon/integrations/seleniumWrapper/config_webdriver.py new file mode 100644 index 00000000..8507959f --- /dev/null +++ b/automon/integrations/seleniumWrapper/config_webdriver.py @@ -0,0 +1,88 @@ +import selenium.webdriver + +from automon.log import Logging + +from .config_webdriver_chrome import ConfigChrome + +log = Logging(name='ConfigWebdriver', level=Logging.INFO) + + +class ConfigWebdriver(object): + webdriver: selenium.webdriver + webdriver_wrapper: any + + Chrome: ConfigChrome + Edge: NotImplemented + Firefox: NotImplemented + + def __init__(self): + self._webdriver_wrapper = None + + self._chrome = ConfigChrome() + self._edge = NotImplemented + self._firefox = NotImplemented + + def __repr__(self): + if self._webdriver_wrapper: + return f'{self._webdriver_wrapper}' + return f'webdriver not configured. try selecting a webdriver' + + @property + def driver(self): + """alias to webdriver + + """ + return self.webdriver + + @property + def webdriver(self): + """selenium webdriver + + """ + return self.webdriver_wrapper.webdriver + + @property + def webdriver_wrapper(self) -> any or ConfigChrome: + """webdriver wrapper + + """ + return self._webdriver_wrapper + + @property + def window_size(self): + """get window size + + """ + if self.webdriver_wrapper: + return self.webdriver_wrapper.window_size + + def Chrome(self): + """selenium Chrome webdriver + + """ + self._webdriver_wrapper = self._chrome + return self._webdriver_wrapper + + def Edge(self): + """selenium Edge webdriver + + """ + return self._edge + + def Firefox(self): + """selenium Firefox webdriver + + """ + return self._firefox + + def run(self): + """run webdriver""" + return self.webdriver_wrapper.run() + + def start(self): + """alias to run""" + return self.run() + + def quit(self): + """quit webdriver""" + return diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py new file mode 100644 index 00000000..ef7948ca --- /dev/null +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -0,0 +1,284 @@ +import os +import warnings +import selenium +import selenium.webdriver + +from automon.log import Logging +from automon.helpers.osWrapper.environ import environ + +from .config_window_size import set_window_size + +log = Logging(name='ConfigChrome', level=Logging.INFO) + + +class ConfigChrome(object): + + def __init__(self): + self._webdriver = None + self._chrome_options = selenium.webdriver.ChromeOptions() + self._chromedriver = environ('SELENIUM_CHROMEDRIVER_PATH') + + self._path_updated = None + self.update_paths() + + self._window_size = set_window_size() + + def __repr__(self): + if self.chromedriver: + return f'Chrome {self.chromedriver}' + return f'Chrome' + + @property + def chrome_options(self): + return self._chrome_options + + @property + def chrome_options_arg(self): + return self.chrome_options.arguments + + @property + def chromedriver(self): + return self._chromedriver + + @property + def webdriver(self) -> selenium.webdriver.Chrome: + return self._webdriver + + @property + def window_size(self): + return self._window_size + + def disable_certificate_verification(self): + warnings.warn('Certificates are not verified', Warning) + self.chrome_options.add_argument('--ignore-certificate-errors') + return self + + def disable_extensions(self): + self.chrome_options.add_argument("--disable-extensions") + + def disable_infobars(self): + self.chrome_options.add_argument("--disable-infobars") + return self + + def disable_notifications(self): + """Pass the argument 1 to allow and 2 to block + + """ + self.chrome_options.add_experimental_option( + "prefs", {"profile.default_content_setting_values.notifications": 2} + ) + return self + + def disable_sandbox(self): + self.chrome_options.add_argument('--no-sandbox') + return self + + def disable_shm(self): + warnings.warn('Disabled shm will use disk I/O, and will be slow', Warning) + self.chrome_options.add_argument('--disable-dev-shm-usage') + return self + + def enable_bigshm(self): + warnings.warn('Big shm not yet implemented', Warning) + return self + + def enable_defaults(self): + self.enable_maximized() + return self + + def enable_fullscreen(self): + self.chrome_options.add_argument("--start-fullscreen") + return self + + def enable_headless(self): + self.chrome_options.add_argument('headless') + return self + + def enable_notifications(self): + """Pass the argument 1 to allow and 2 to block + + """ + self.chrome_options.add_experimental_option( + "prefs", {"profile.default_content_setting_values.notifications": 1} + ) + return self + + def enable_maximized(self): + self.chrome_options.add_argument('--start-maximized') + return self + + def close(self): + """close + + """ + return self.webdriver.close() + + def in_docker(self): + """Chrome best used with docker + + """ + self.enable_defaults() + self.enable_headless() + self.disable_sandbox() + self.disable_infobars() + self.disable_extensions() + self.disable_notifications() + return self + + def in_headless(self): + """alias to headless sandboxed + + """ + return self.in_headless_sandboxed() + + def in_headless_sandboxed(self): + """Headless Chrome with sandbox enabled + + """ + warnings.warn('Docker does not support sandbox option') + warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + + self.enable_defaults() + self.enable_headless() + return self + + def in_headless_sandbox_disabled(self): + """Headless Chrome with sandbox disabled + + """ + warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + + self.enable_defaults() + self.enable_headless() + self.disable_sandbox() + return self + + def in_headless_sandbox_disabled_certificate_unverified(self): + """Headless Chrome with sandbox disabled with no certificate verification + + """ + warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + + self.enable_defaults() + self.enable_headless() + self.disable_sandbox() + self.disable_certificate_verification() + return self + + def in_headless_sandbox_disabled_shm_disabled(self): + """Headless Chrome with sandbox disabled + + """ + self.enable_defaults() + self.enable_headless() + self.disable_sandbox() + self.disable_shm() + return self + + def in_headless_sandbox_disabled_bigshm(self): + """Headless Chrome with sandbox disabled + + """ + warnings.warn('Larger shm option is not implemented', Warning) + + self.enable_defaults() + self.enable_headless() + self.enable_bigshm() + self.disable_sandbox() + return self + + def in_remote_driver(self, host: str = '127.0.0.1', port: str = '4444', executor_path: str = '/wd/hub'): + """Remote Selenium + + """ + log.info( + f'Remote WebDriver Hub URL: http://{host}:{port}{executor_path}/static/resource/hub.html') + + selenium.webdriver.Remote( + command_executor=f'http://{host}:{port}{executor_path}', + desired_capabilities=selenium.webdriver.common.desired_capabilities.DesiredCapabilities.CHROME + ) + + def in_sandbox(self): + """Chrome with sandbox enabled + + """ + warnings.warn('Docker does not support sandbox option') + warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + + self.enable_defaults() + return self + + def in_sandbox_disabled(self): + """Chrome with sandbox disabled + + """ + warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + + self.enable_defaults() + self.disable_sandbox() + return self + + def run(self): + log.info(f'starting {self}') + try: + if self.chromedriver: + self._webdriver = selenium.webdriver.Chrome(executable_path=self.chromedriver, + options=self.chrome_options) + return self.webdriver + + self._webdriver = selenium.webdriver.Chrome(options=self.chrome_options) + return self.webdriver + except Exception as e: + log.error(f'Browser not set. {e}', enable_traceback=False) + + def set_chromedriver(self, chromedriver: str): + self._chromedriver = chromedriver + self.update_paths() + return self + + def set_user_agent(self, user_agent: str): + self.chrome_options.add_argument(f"user-agent={user_agent}") + return self + + def set_window_size(self, *args, **kwargs): + self._window_size = set_window_size(*args, **kwargs) + width, height = self.window_size + self.webdriver.set_window_size(width=width, height=height) + return self + + def start(self): + """alias to run + + """ + return self.run() + + def stop_client(self): + """stop client + + """ + return self.webdriver.stop_client() + + def update_paths(self): + if self.chromedriver: + if not self._path_updated: + os.environ['PATH'] = f"{os.getenv('PATH')}:{self._chromedriver}" + + def quit(self): + """quit + + """ + return self.webdriver.quit() + + def quit_gracefully(self): + """gracefully quit webdriver + + """ + try: + self.close() + self.quit() + self.stop_client() + except Exception as error: + log.error(f'failed to gracefully quit. {error}') + return False + return True diff --git a/automon/integrations/seleniumWrapper/config_window_size.py b/automon/integrations/seleniumWrapper/config_window_size.py new file mode 100644 index 00000000..fb424e03 --- /dev/null +++ b/automon/integrations/seleniumWrapper/config_window_size.py @@ -0,0 +1,48 @@ +def set_window_size(width=1920, height=1080, device_type=None) -> (int, int): + """set browser resolution""" + + if device_type == 'pixel3': + width = 1080 + height = 2160 + + if device_type == 'web-small' or device_type == '800x600': + width = 800 + height = 600 + + if device_type == 'web-small-2' or device_type == '1024x768': + width = 1024 + height = 768 + + if device_type == 'web-small-3' or device_type == '1280x960': + width = 1280 + height = 960 + + if device_type == 'web-small-4' or device_type == '1280x1024': + width = 1280 + height = 1024 + + if device_type == 'web' or device_type == '1920x1080': + width = 1920 + height = 1080 + + if device_type == 'web-2' or device_type == '1600x1200': + width = 1600 + height = 1200 + + if device_type == 'web-3' or device_type == '1920x1200': + width = 1920 + height = 1200 + + if device_type == 'web-large' or device_type == '2560x1400': + width = 2560 + height = 1400 + + if device_type == 'web-long' or device_type == '1920x3080': + width = 1920 + height = 3080 + + if not width and not height: + width = 1920 + height = 1080 + + return width, height diff --git a/automon/integrations/seleniumWrapper/tests/test_browser.py b/automon/integrations/seleniumWrapper/tests/test_browser.py index edf7395a..a190044c 100644 --- a/automon/integrations/seleniumWrapper/tests/test_browser.py +++ b/automon/integrations/seleniumWrapper/tests/test_browser.py @@ -3,11 +3,11 @@ from automon.integrations.seleniumWrapper.browser import SeleniumBrowser browser = SeleniumBrowser() -browser.set_driver(browser.type.chrome()) +browser.config.set_webdriver.Chrome().enable_defaults() class SeleniumClientTest(unittest.TestCase): - if browser.is_running(): + if browser.run(): def test(self): self.assertFalse(browser.get('http://555.555.555.555')) if browser.get('http://1.1.1.1'): diff --git a/automon/integrations/seleniumWrapper/tests/test_browser_headless.py b/automon/integrations/seleniumWrapper/tests/test_browser_headless.py index 3acea008..ae5b4d0c 100644 --- a/automon/integrations/seleniumWrapper/tests/test_browser_headless.py +++ b/automon/integrations/seleniumWrapper/tests/test_browser_headless.py @@ -3,12 +3,13 @@ from automon.integrations.seleniumWrapper.browser import SeleniumBrowser browser = SeleniumBrowser() -browser.set_driver(browser.type.chrome_headless()) -browser.set_window_size(device_type='web-large') +browser.config.set_webdriver.Chrome().enable_defaults() class SeleniumClientTest(unittest.TestCase): - if browser.is_running(): + if browser.run(): + browser.set_window_size(device_type='web-large') + def test(self): while True: diff --git a/automon/integrations/seleniumWrapper/tests/test_browser_useragent.py b/automon/integrations/seleniumWrapper/tests/test_browser_useragent.py index b2e7e1eb..957df4cc 100644 --- a/automon/integrations/seleniumWrapper/tests/test_browser_useragent.py +++ b/automon/integrations/seleniumWrapper/tests/test_browser_useragent.py @@ -3,15 +3,15 @@ from automon.integrations.seleniumWrapper.browser import SeleniumBrowser browser = SeleniumBrowser() +browser.config.set_webdriver.Chrome().enable_defaults() agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:112.0) Gecko/20100101 Firefox/112.0' -opts = [f"user-agent={agent}"] -browser.set_driver(browser.type.chrome(options=opts)) +browser.config.set_webdriver.webdriver_wrapper.set_user_agent(agent) class SeleniumClientTest(unittest.TestCase): - if browser.is_running(): + if browser.run(): def test_user_agent(self): self.assertEqual(browser.get_user_agent(), agent) diff --git a/automon/integrations/seleniumWrapper/tests/test_new_browser.py b/automon/integrations/seleniumWrapper/tests/test_new_browser.py new file mode 100644 index 00000000..d5a30e90 --- /dev/null +++ b/automon/integrations/seleniumWrapper/tests/test_new_browser.py @@ -0,0 +1,16 @@ +import unittest + +from automon.integrations.seleniumWrapper.browser import SeleniumBrowser + +browser = SeleniumBrowser() +browser.config.set_webdriver.Chrome().enable_defaults() + + +class SeleniumClientTest(unittest.TestCase): + if browser.run(): + def test(self): + browser.quit() + + +if __name__ == '__main__': + unittest.main() From 9b9fd6a0346fa7b49782238508e6813ac5ac53a7 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 6 Jul 2023 19:10:45 +0800 Subject: [PATCH 15/62] instagram: update to new selenium --- automon/integrations/instagram/client_browser.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/automon/integrations/instagram/client_browser.py b/automon/integrations/instagram/client_browser.py index 12d207b3..8ce2b74f 100644 --- a/automon/integrations/instagram/client_browser.py +++ b/automon/integrations/instagram/client_browser.py @@ -23,22 +23,17 @@ def __init__(self, login: str = None, password: str = None, config: InstagramConfig = None, - headless: bool = True, - browser_options: list = None): + headless: bool = True): """Instagram Browser Client""" self.config = config or InstagramConfig(login=login, password=password) self.browser = SeleniumBrowser() useragent = self.browser.get_random_user_agent() - if browser_options: - browser_options.extend([f"user-agent={useragent}"]) - else: - browser_options = [f"user-agent={useragent}"] if headless: - self.browser.set_browser(self.browser.type.chrome_headless(options=browser_options)) + self.browser.config.set_webdriver.Chrome().in_headless().set_user_agent(useragent) else: - self.browser.set_browser(self.browser.type.chrome(options=browser_options)) + self.browser.config.set_webdriver.Chrome().in_headless() def __repr__(self): return f'{self.__dict__}' From 8e742e7c83dc067ad4e0df7688d130c832ca62d7 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 6 Jul 2023 19:16:04 +0800 Subject: [PATCH 16/62] 0.3.0 Change log: selenium: refactor browser and config selenium: move set_window_size in config selenium: small typo selenium: add authenticated xpaths selenium: get current page with beautifulsoup instagram: update to new selenium instagram: update xpaths instagram: urls for followers and following instagram: tests instagram: update authenticate instagram: use a random user agent instagram: support browser_options instagram: update config --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4dfd8122..bcc543bc 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.2.50", + version="0.3.0", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From 8d8bb7a77696348b5d39367569b1d239679b0161 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 27 Jul 2023 05:16:11 +0800 Subject: [PATCH 17/62] move people/tests --- .../google/people/tests/__init__.py | 0 .../people/tests/test_google_contacts.py | 15 +++++++++++++ .../tests/test_google_contacts_neo4j.py | 21 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 automon/integrations/google/people/tests/__init__.py create mode 100644 automon/integrations/google/people/tests/test_google_contacts.py create mode 100644 automon/integrations/google/people/tests/test_google_contacts_neo4j.py diff --git a/automon/integrations/google/people/tests/__init__.py b/automon/integrations/google/people/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/automon/integrations/google/people/tests/test_google_contacts.py b/automon/integrations/google/people/tests/test_google_contacts.py new file mode 100644 index 00000000..a5c18459 --- /dev/null +++ b/automon/integrations/google/people/tests/test_google_contacts.py @@ -0,0 +1,15 @@ +import unittest + +from automon.integrations.google import PeopleClient + +c = PeopleClient() + + +class TestClient(unittest.TestCase): + def test_list_connections(self): + if c.isConnected(): + self.assertTrue(list(c.list_connection_generator())) + + +if __name__ == '__main__': + unittest.main() diff --git a/automon/integrations/google/people/tests/test_google_contacts_neo4j.py b/automon/integrations/google/people/tests/test_google_contacts_neo4j.py new file mode 100644 index 00000000..15cf1c3d --- /dev/null +++ b/automon/integrations/google/people/tests/test_google_contacts_neo4j.py @@ -0,0 +1,21 @@ +import unittest + +from automon.integrations.google import PeopleClient +from automon.integrations.neo4jWrapper import Neo4jClient + +c = PeopleClient() +n = Neo4jClient() + + +class TestClient(unittest.TestCase): + + def test_create_nodes(self): + if c.isConnected(): + contacts = c.list_connections().contacts + for contact in contacts: + n.merge_dict(contact) + pass + + +if __name__ == '__main__': + unittest.main() From d6d20521099a4f1dc042af33c70237608ca57130 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 27 Jul 2023 05:16:49 +0800 Subject: [PATCH 18/62] add google/auth --- automon/integrations/google/__init__.py | 1 + automon/integrations/google/auth/__init__.py | 2 + automon/integrations/google/auth/client.py | 129 ++++++++++++++++++ automon/integrations/google/auth/config.py | 120 ++++++++++++++++ .../google/auth/tests/__init__.py | 0 .../auth/tests/test_config_Credentials.py | 14 ++ .../google/auth/tests/test_google_auth.py | 16 +++ 7 files changed, 282 insertions(+) create mode 100644 automon/integrations/google/auth/__init__.py create mode 100644 automon/integrations/google/auth/client.py create mode 100644 automon/integrations/google/auth/config.py create mode 100644 automon/integrations/google/auth/tests/__init__.py create mode 100644 automon/integrations/google/auth/tests/test_config_Credentials.py create mode 100644 automon/integrations/google/auth/tests/test_google_auth.py diff --git a/automon/integrations/google/__init__.py b/automon/integrations/google/__init__.py index d14fd0e2..dc84b86a 100644 --- a/automon/integrations/google/__init__.py +++ b/automon/integrations/google/__init__.py @@ -1,2 +1,3 @@ +from .auth import AuthClient from .gmail import GmailClientV1 from .people import PeopleClient diff --git a/automon/integrations/google/auth/__init__.py b/automon/integrations/google/auth/__init__.py new file mode 100644 index 00000000..293d82bd --- /dev/null +++ b/automon/integrations/google/auth/__init__.py @@ -0,0 +1,2 @@ +from .client import AuthClient +from .config import AuthConfig diff --git a/automon/integrations/google/auth/client.py b/automon/integrations/google/auth/client.py new file mode 100644 index 00000000..b8453e9a --- /dev/null +++ b/automon/integrations/google/auth/client.py @@ -0,0 +1,129 @@ +import functools +import googleapiclient.http +import googleapiclient.discovery +import google.auth.transport.requests + +from automon.log import Logging + +from .config import AuthConfig + +log = Logging(name='AuthClient', level=Logging.DEBUG) + + +class AuthClient(object): + """Google Auth client""" + + def __init__( + self, + config: AuthConfig = None, + serviceName: str = None, + scopes: list = None, + version: str = None, + **kwargs, + ): + + self.config = config or AuthConfig( + serviceName=serviceName, + scopes=scopes, + version=version, + **kwargs + ) + + def __repr__(self): + return f'{self.__dict__}' + + @classmethod + def execute(cls, func): + return func.execute() + + def _is_connected(func): + @functools.wraps(func) + def wrapped(self, *args, **kwargs): + if self.authenticate(): + return func(self, *args, **kwargs) + + return wrapped + + def authenticate(self) -> bool: + """authenticate with credentials""" + + try: + return self.authenticate_oauth() + except: + pass + + try: + return self.authenticate_service_account() + except: + pass + + return False + + def authenticate_oauth(self) -> bool: + """authenticate web token""" + + creds = self.config.Credentials + refresh_token = creds.refresh_token + + if refresh_token: + try: + creds.refresh(google.auth.transport.requests.Request()) + log.info(f'token refresh success') + return True + except Exception as e: + log.error(msg=f'token refresh failed: {e}', enable_traceback=False) + + else: + # TODO: add google flow() authentication here + log.info(f'flow login success') + return True + + return False + + def authenticate_service_account(self) -> bool: + """authenticate service account""" + return True + + def is_connected(self) -> bool: + """Check if authenticated to make requests""" + return self.authenticate() + + def service( + self, + serviceName: str = None, + version: str = None, + http=None, + discoveryServiceUrl=None, + developerKey=None, + model=None, + requestBuilder=None, + credentials=None, + cache_discovery=True, + cache=None, + client_options=None, + adc_cert_path=None, + adc_key_path=None, + num_retries=1, + static_discovery=None, + always_use_jwt_access=False, + **kwargs + ) -> googleapiclient.discovery.build: + return googleapiclient.discovery.build( + serviceName=serviceName or self.config.serviceName, + version=version or self.config.version, + http=http, + discoveryServiceUrl=discoveryServiceUrl, + developerKey=developerKey, + model=model, + requestBuilder=requestBuilder or googleapiclient.http.HttpRequest, + credentials=credentials or self.config.Credentials, + cache_discovery=cache_discovery, + cache=cache, + client_options=client_options, + adc_cert_path=adc_cert_path, + adc_key_path=adc_key_path, + num_retries=num_retries, + static_discovery=static_discovery, + always_use_jwt_access=always_use_jwt_access, + **kwargs, + ) diff --git a/automon/integrations/google/auth/config.py b/automon/integrations/google/auth/config.py new file mode 100644 index 00000000..a7f1a972 --- /dev/null +++ b/automon/integrations/google/auth/config.py @@ -0,0 +1,120 @@ +import os +import base64 + +import google.auth.crypt +import google.oauth2.credentials +import google.oauth2.service_account + +from google.auth.transport.requests import Request +from google_auth_oauthlib.flow import InstalledAppFlow + +from automon.log import Logging +from automon.helpers import environ + +log = Logging(name='AuthConfig', level=Logging.DEBUG) + + +class AuthConfig(object): + """Google Auth config""" + + def __init__( + self, + serviceName: str = None, + scopes: list = None, + version: str = None, + ): + self.serviceName = serviceName or 'servicemanagement' + self.scopes = scopes or ['https://www.googleapis.com/auth/cloud-platform.read-only'] + self.version = version or 'v1' + + def __repr__(self): + return f'{self.__dict__}' + + @property + def Credentials(self): + """return Google Credentials object""" + try: + if self.CredentialsFile(): + return self.CredentialsFile() + except: + pass + + try: + if self.CredentialsInfo(): + return self.CredentialsInfo() + except: + pass + + try: + if self.CredentialsServiceAccountFile(): + return self.CredentialsServiceAccountFile() + except: + pass + + try: + if self.CredentialsServiceAccountInfo(): + return self.CredentialsServiceAccountInfo() + except: + pass + + log.error(f'Missing credentials', enable_traceback=False) + + @property + def _GOOGLE_CREDENTIALS(self): + """env var GOOGLE_CREDENTIALS""" + return environ('GOOGLE_CREDENTIALS') + + @property + def _GOOGLE_CREDENTIALS_BASE64(self): + """env var GOOGLE_CREDENTIALS_BASE64""" + return environ('GOOGLE_CREDENTIALS_BASE64') + + def CredentialsFile(self) -> google.oauth2.credentials.Credentials: + """return Credentials object for web auth from file""" + if self._GOOGLE_CREDENTIALS: + if os.path.exists(self._GOOGLE_CREDENTIALS): + return google.oauth2.credentials.Credentials.from_authorized_user_file( + self._GOOGLE_CREDENTIALS + ) + + def CredentialsInfo(self) -> google.oauth2.credentials.Credentials: + """return Credentials object for web auth from dict""" + if self._GOOGLE_CREDENTIALS_BASE64: + return google.oauth2.credentials.Credentials.from_authorized_user_info( + self.base64_to_dict() + ) + + def CredentialsServiceAccountFile(self) -> google.oauth2.service_account.Credentials: + """return Credentials object for service account from file""" + if self._GOOGLE_CREDENTIALS: + if os.path.exists(self._GOOGLE_CREDENTIALS): + return google.oauth2.service_account.Credentials.from_service_account_file( + self._GOOGLE_CREDENTIALS + ) + + def CredentialsServiceAccountInfo(self) -> google.oauth2.service_account.Credentials: + """return Credentials object for service account from dict""" + if self._GOOGLE_CREDENTIALS_BASE64: + return google.oauth2.service_account.Credentials.from_service_account_info( + self.base64_to_dict() + ) + + def base64_to_dict(self, base64: str = None): + """convert credential json to dict""" + if not base64 and self._GOOGLE_CREDENTIALS_BASE64: + base64 = self._GOOGLE_CREDENTIALS_BASE64 + + return base64.decode(base64) + + def file_to_base64(self, path: str = None): + """convert file to base64""" + if not path and self._GOOGLE_CREDENTIALS: + path = self._GOOGLE_CREDENTIALS + + with open(path, 'rb') as f: + return base64.b64encode(f.read()).decode() + + def is_ready(self): + """return True if configured""" + if self.Credentials: + return True diff --git a/automon/integrations/google/auth/tests/__init__.py b/automon/integrations/google/auth/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/automon/integrations/google/auth/tests/test_config_Credentials.py b/automon/integrations/google/auth/tests/test_config_Credentials.py new file mode 100644 index 00000000..fcfb4f5d --- /dev/null +++ b/automon/integrations/google/auth/tests/test_config_Credentials.py @@ -0,0 +1,14 @@ +import unittest + +from automon.integrations.google.auth import AuthConfig + + +class MyTestCase(unittest.TestCase): + def test_something(self): + test = AuthConfig() + if test.Credentials: + self.assertTrue(test.Credentials) + + +if __name__ == '__main__': + unittest.main() diff --git a/automon/integrations/google/auth/tests/test_google_auth.py b/automon/integrations/google/auth/tests/test_google_auth.py new file mode 100644 index 00000000..a9dfec6b --- /dev/null +++ b/automon/integrations/google/auth/tests/test_google_auth.py @@ -0,0 +1,16 @@ +import unittest + +from automon.integrations.google.auth import AuthClient + + +class MyTestCase(unittest.TestCase): + def test_authenticate(self): + test = AuthClient() + # scopes = ['https://www.googleapis.com/auth/contacts.readonly'] + # client = AuthClient(serviceName='people', scopes=scopes) + if test.authenticate(): + self.assertTrue(test.authenticate()) + + +if __name__ == '__main__': + unittest.main() From c1412981c6cffe577d0372b72f47c0caf2a68f0f Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Fri, 4 Aug 2023 05:10:06 +0800 Subject: [PATCH 19/62] add beautifulsoupWrapper --- .../beautifulsoupWrapper/__init__.py | 1 + .../beautifulsoupWrapper/client.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 automon/integrations/beautifulsoupWrapper/__init__.py create mode 100644 automon/integrations/beautifulsoupWrapper/client.py diff --git a/automon/integrations/beautifulsoupWrapper/__init__.py b/automon/integrations/beautifulsoupWrapper/__init__.py new file mode 100644 index 00000000..37058512 --- /dev/null +++ b/automon/integrations/beautifulsoupWrapper/__init__.py @@ -0,0 +1 @@ +from .client import BeautifulSoupClient diff --git a/automon/integrations/beautifulsoupWrapper/client.py b/automon/integrations/beautifulsoupWrapper/client.py new file mode 100644 index 00000000..a520bd54 --- /dev/null +++ b/automon/integrations/beautifulsoupWrapper/client.py @@ -0,0 +1,24 @@ +from bs4 import BeautifulSoup + +from automon.log import Logging + +log = Logging(name='BeautifulSoupClient', level=Logging.DEBUG) + + +class BeautifulSoupClient(object): + + def __init__(self, bs: BeautifulSoup = None): + self.bs = bs + + def read_markup(self, markup: str, features: str = 'lxml'): + """read markup with beautifulsoup""" + try: + self.bs = BeautifulSoup( + markup=markup or self.markup, + features=features + ) + log.info(f'read_markup success ({len(markup)} B)') + except Exception as e: + log.error(f'read_markup failed ({len(markup)} B): {e}') + + return self From e5b80966ff44b767773ae0ab8f305f07831e19fd Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Fri, 4 Aug 2023 05:41:21 +0800 Subject: [PATCH 20/62] add scrapyWrapper --- automon/integrations/scrapyWrapper/__init__.py | 1 + automon/integrations/scrapyWrapper/client.py | 14 ++++++++++++++ requirements.txt | 3 +++ 3 files changed, 18 insertions(+) create mode 100644 automon/integrations/scrapyWrapper/__init__.py create mode 100644 automon/integrations/scrapyWrapper/client.py diff --git a/automon/integrations/scrapyWrapper/__init__.py b/automon/integrations/scrapyWrapper/__init__.py new file mode 100644 index 00000000..42d0cbab --- /dev/null +++ b/automon/integrations/scrapyWrapper/__init__.py @@ -0,0 +1 @@ +from .client import ScrapyClient diff --git a/automon/integrations/scrapyWrapper/client.py b/automon/integrations/scrapyWrapper/client.py new file mode 100644 index 00000000..634ae6d6 --- /dev/null +++ b/automon/integrations/scrapyWrapper/client.py @@ -0,0 +1,14 @@ +import scrapy + +from automon.log import Logging + +log = Logging(name='ScrapyClient', level=Logging.DEBUG) + + +class ScrapyClient(object): + + def Selector(self, text: str): + return scrapy.selector.Selector(text=text) + + def xpath(self, text: str, xpath: str): + return self.Selector(text=text).xpath(xpath).get() diff --git a/requirements.txt b/requirements.txt index 64ea3bb8..cffb2319 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,6 +49,9 @@ selenium>=3.141.0 # sentry.io sentry-sdk>=1.5.1 +# scrapy +Scrapy>=2.9.0 + # slack slackclient>=2.9.3 From 06ea8e8f0df4fe634f05a2cdb20d57300a717536 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Fri, 4 Aug 2023 06:01:01 +0800 Subject: [PATCH 21/62] requests: update properties --- automon/integrations/requestsWrapper/client.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/automon/integrations/requestsWrapper/client.py b/automon/integrations/requestsWrapper/client.py index 880ebf22..86c69cff 100644 --- a/automon/integrations/requestsWrapper/client.py +++ b/automon/integrations/requestsWrapper/client.py @@ -28,6 +28,10 @@ def __init__(self, url: str = None, data: dict = None, headers: dict = None, def __repr__(self): return f'{self.__dict__}' + def __len__(self): + if self.content: + len(self.content) + def _log_result(self): if self.results.status_code == 200: msg = f'{self.results.status_code} ' \ @@ -60,9 +64,14 @@ def _params(self, url, data, headers): @property def content(self): - if self.results is not None: + if self.results: return self.results.content + @property + def text(self): + if self.results: + return self.results.text + def delete(self, url: str = None, data: dict = None, From c3bc40064e4433012ba83306c3e5572e30e9f0fe Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 02:23:40 +0800 Subject: [PATCH 22/62] selenium: change waiting for element to 3 retries --- automon/integrations/seleniumWrapper/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index d8059e59..112151ae 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -279,7 +279,7 @@ def wait_for( self, value: str or list, by: By = By.XPATH, - retries: int = 30, + retries: int = 3, **kwargs) -> str or False: """wait for something""" retry = 1 From e086695621e8c3c8b6cff31f6ec1702fdde9def8 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 02:24:06 +0800 Subject: [PATCH 23/62] selenium: raise exception if webdriver is not set --- automon/integrations/seleniumWrapper/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automon/integrations/seleniumWrapper/config.py b/automon/integrations/seleniumWrapper/config.py index 0f9da585..7341c289 100644 --- a/automon/integrations/seleniumWrapper/config.py +++ b/automon/integrations/seleniumWrapper/config.py @@ -29,4 +29,4 @@ def webdriver(self): if self.set_webdriver.webdriver: return self.set_webdriver.webdriver else: - log.error('driver not set. configure a driver first') + log.error('driver not set. configure a driver first', raise_exception=True) From 9b5f62ac108c5d380593bee43b5dc5a80357a02d Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 02:26:09 +0800 Subject: [PATCH 24/62] selenium: use Request for return code status this way the status feels more useful, and it is tied to the RequestsWrapper, so any `requests` methods can be used to further troubleshoot --- automon/integrations/seleniumWrapper/browser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 112151ae..37fe319f 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -13,6 +13,7 @@ from automon.helpers.dates import Dates from automon.helpers.sleeper import Sleeper from automon.helpers.sanitation import Sanitation +from automon.integrations.requestsWrapper import RequestsClient from .config import SeleniumConfig from .browser_types import SeleniumBrowserType @@ -164,7 +165,7 @@ def get(self, url: str, **kwargs) -> bool: """get url""" try: self.webdriver.get(url, **kwargs) - self.status = 'OK' + self.status = RequestsClient(url=url).results msg = f'GET {self.status} {self.webdriver.current_url}' if kwargs: @@ -172,7 +173,7 @@ def get(self, url: str, **kwargs) -> bool: log.debug(msg) return True except Exception as e: - self.status = f'ERROR {url}' + self.status = RequestsClient(url=url).results msg = f'GET {self.status}: {e}' log.error(msg, enable_traceback=False) From 4d79231c835707e5ca33ff14a4f8a713b8a65a90 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 02:26:33 +0800 Subject: [PATCH 25/62] selenium: remove and depreciate properties removed `self.browser` removed `self.type` --- automon/integrations/seleniumWrapper/browser.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 37fe319f..ea99c57d 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -38,11 +38,6 @@ def __repr__(self): return f'{self.webdriver.name} {self.status} {self.webdriver.current_url} {self.window_size}' return f'{self.webdriver}' - @property - def browser(self): - """alias to webdriver""" - return self.webdriver - @property def by(self) -> By: """Set of supported locator strategies""" @@ -66,9 +61,9 @@ def keys(self): """Set of special keys codes""" return selenium.webdriver.common.keys.Keys - @property - def type(self) -> SeleniumBrowserType: - return SeleniumBrowserType(self.config) + # @property + # def type(self) -> SeleniumBrowserType: + # return SeleniumBrowserType(self.config) @property def url(self): From a8ed955961d645eab5f775c284e11981521bac34 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 03:18:02 +0800 Subject: [PATCH 26/62] selenium: now gets all available logs from webdriver.log_types --- automon/integrations/seleniumWrapper/browser.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index ea99c57d..239f8a66 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -52,9 +52,17 @@ def webdriver(self): return self.config.webdriver @property - def get_log(self, log_type: str = 'browser') -> list: + def get_log(self) -> list: """Gets the log for a given log type""" - return self.webdriver.get_log(log_type) + logs = [] + for log_type in self.webdriver.log_types: + logs.append( + { + log_type: self.webdriver.get_log(log_type) + } + ) + + return logs @property def keys(self): From 2293d5239b16d5571537b016ad519a7e0a85027c Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 03:18:38 +0800 Subject: [PATCH 27/62] selenium: fix chromedriver path missing updated property --- automon/integrations/seleniumWrapper/config_webdriver_chrome.py | 1 + 1 file changed, 1 insertion(+) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index ef7948ca..533c2f19 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -263,6 +263,7 @@ def update_paths(self): if self.chromedriver: if not self._path_updated: os.environ['PATH'] = f"{os.getenv('PATH')}:{self._chromedriver}" + self._path_updated = True def quit(self): """quit From 8c5aeaa5bb165a8a7b0a967ec1569814b4565065 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 03:19:02 +0800 Subject: [PATCH 28/62] selenium: update typing --- automon/integrations/seleniumWrapper/config_webdriver_chrome.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index 533c2f19..77b1c7a0 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -219,7 +219,7 @@ def in_sandbox_disabled(self): self.disable_sandbox() return self - def run(self): + def run(self) -> selenium.webdriver.Chrome: log.info(f'starting {self}') try: if self.chromedriver: From b1819205228a65f4d9b3bb611d1e7beeb079f91b Mon Sep 17 00:00:00 2001 From: Eric <58240560+naisanzaa@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:36:06 -0700 Subject: [PATCH 29/62] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c5431af8..c10f2df2 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ [![master](https://github.com/TheShellLand/automonisaur/actions/workflows/python37.yml/badge.svg)](https://github.com/TheShellLand/automonisaur/actions/workflows/python37.yml) [![master](https://github.com/TheShellLand/automonisaur/actions/workflows/python36.yml/badge.svg)](https://github.com/TheShellLand/automonisaur/actions/workflows/python36.yml) -[![Downloads](https://pepy.tech/badge/automonisaur)](https://pepy.tech/project/automonisaur) -[![Downloads](https://pepy.tech/badge/automonisaur/month)](https://pepy.tech/project/automonisaur) -[![Downloads](https://pepy.tech/badge/automonisaur/week)](https://pepy.tech/project/automonisaur) +[![Downloads](https://static.pepy.tech/badge/automonisaur)](https://pepy.tech/project/automonisaur) +[![Downloads](https://static.pepy.tech/badge/automonisaur/month)](https://pepy.tech/project/automonisaur) +[![Downloads](https://static.pepy.tech/badge/automonisaur/week)](https://pepy.tech/project/automonisaur) [//]: # ([![codecov](https://codecov.io/gh/TheShellLand/automonisaur/branch/master/graph/badge.svg)](https://codecov.io/gh/TheShellLand/automonisaur)) From 51d372650f478f71dd743dda689aca2d0b70f4aa Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 04:25:33 +0800 Subject: [PATCH 30/62] selenium: fix wait_for not iterating through values --- .../integrations/seleniumWrapper/browser.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 239f8a66..0d2c097a 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -290,14 +290,17 @@ def wait_for( while True: try: if isinstance(value, list): - for each in value: - self.find_element( - by=by, - value=each, - **kwargs) - value = each - log.debug(f'found {by}: {value}') - return value + values = value + for value in values: + try: + self.find_element( + by=by, + value=value, + **kwargs) + log.debug(f'found {by}: {value}') + return value + except: + log.error(f'{by} not found: {value}', enable_traceback=False) else: self.find_element( by=by, @@ -308,7 +311,7 @@ def wait_for( except Exception as error: log.error(f'waiting for {by}: {value}, {error}', enable_traceback=False) - Sleeper.seconds(f'wait for', round(retry / 2)) + Sleeper.seconds(f'wait for', 1) retry += 1 From 02eed86499f2c826fd263254b0872eb0dbbd5350 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 03:42:28 +0800 Subject: [PATCH 31/62] selenium: update config --- automon/integrations/seleniumWrapper/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/automon/integrations/seleniumWrapper/config.py b/automon/integrations/seleniumWrapper/config.py index 7341c289..a7c4642e 100644 --- a/automon/integrations/seleniumWrapper/config.py +++ b/automon/integrations/seleniumWrapper/config.py @@ -16,17 +16,17 @@ def __init__(self): def __repr__(self): return f'{self.driver}' - @property - def set_webdriver(self): - return self._webdriver_wrapper - @property def driver(self): return self.webdriver + @property + def set_webdriver(self): + return self._webdriver_wrapper + @property def webdriver(self): if self.set_webdriver.webdriver: return self.set_webdriver.webdriver else: - log.error('driver not set. configure a driver first', raise_exception=True) + log.error('driver not set. configure a driver first', enable_traceback=False) From 21f47ae0ea78c634fec1c6c194b878368e89f184 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 03:42:54 +0800 Subject: [PATCH 32/62] selenium: raise exception when ConfigWebdriver fails --- automon/integrations/seleniumWrapper/config_webdriver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/automon/integrations/seleniumWrapper/config_webdriver.py b/automon/integrations/seleniumWrapper/config_webdriver.py index 8507959f..b16c79c2 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver.py +++ b/automon/integrations/seleniumWrapper/config_webdriver.py @@ -77,7 +77,10 @@ def Firefox(self): def run(self): """run webdriver""" - return self.webdriver_wrapper.run() + try: + return self.webdriver_wrapper.run() + except Exception as e: + log.error(f'failed to run: {e}', raise_exception=True) def start(self): """alias to run""" From e9deb40f12bad426e9e5b12eebcfc056a983804e Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 03:43:41 +0800 Subject: [PATCH 33/62] selenium: update to selenium 4 --- .../seleniumWrapper/config_webdriver_chrome.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index 77b1c7a0..ab2b3152 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -17,6 +17,7 @@ def __init__(self): self._webdriver = None self._chrome_options = selenium.webdriver.ChromeOptions() self._chromedriver = environ('SELENIUM_CHROMEDRIVER_PATH') + self._ChromeService = None self._path_updated = None self.update_paths() @@ -40,6 +41,10 @@ def chrome_options_arg(self): def chromedriver(self): return self._chromedriver + @property + def ChromeService(self): + return self._ChromeService + @property def webdriver(self) -> selenium.webdriver.Chrome: return self._webdriver @@ -223,14 +228,19 @@ def run(self) -> selenium.webdriver.Chrome: log.info(f'starting {self}') try: if self.chromedriver: - self._webdriver = selenium.webdriver.Chrome(executable_path=self.chromedriver, - options=self.chrome_options) + self._ChromeService = selenium.webdriver.ChromeService( + executable_path=self.chromedriver + ) + self._webdriver = selenium.webdriver.Chrome( + service=self._ChromeService, + options=self.chrome_options + ) return self.webdriver self._webdriver = selenium.webdriver.Chrome(options=self.chrome_options) return self.webdriver except Exception as e: - log.error(f'Browser not set. {e}', enable_traceback=False) + log.error(f'Browser not set. {e}', raise_exception=True) def set_chromedriver(self, chromedriver: str): self._chromedriver = chromedriver From 4660f897dbe2576a65fe2383f7e28ae294e74e01 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 07:00:55 +0800 Subject: [PATCH 34/62] selenium: raise exception if missing driver --- automon/integrations/seleniumWrapper/browser.py | 1 - automon/integrations/seleniumWrapper/config.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 0d2c097a..971f632c 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -88,7 +88,6 @@ def _is_running(func) -> functools.wraps: def wrapped(self, *args, **kwargs): if self.webdriver is not None: return func(self, *args, **kwargs) - log.error(f'Browser is not set!', enable_traceback=False) return False return wrapped diff --git a/automon/integrations/seleniumWrapper/config.py b/automon/integrations/seleniumWrapper/config.py index a7c4642e..67dd31ae 100644 --- a/automon/integrations/seleniumWrapper/config.py +++ b/automon/integrations/seleniumWrapper/config.py @@ -29,4 +29,4 @@ def webdriver(self): if self.set_webdriver.webdriver: return self.set_webdriver.webdriver else: - log.error('driver not set. configure a driver first', enable_traceback=False) + log.debug('waiting for driver') From 6b596f5478c7ebeda1cc854fa9db61f24495fe85 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 07:17:45 +0800 Subject: [PATCH 35/62] selenium: better logging --- .../integrations/seleniumWrapper/browser.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 971f632c..b0541ca7 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -155,12 +155,16 @@ def find_element( by: By.ID = By.ID, **kwargs): """find element""" - return self.webdriver.find_element(value=value, by=by, **kwargs) + element = self.webdriver.find_element(value=value, by=by, **kwargs) + log.info(f'found element: {self.url} {element.text}') + return element @_is_running def find_xpath(self, value: str, by: By = By.XPATH, **kwargs): """find xpath""" - return self.find_element(value=value, by=by, **kwargs) + xpath = self.find_element(value=value, by=by, **kwargs) + log.info(f'found xpath: {self.url} {xpath.text}') + return xpath @_is_running def get(self, url: str, **kwargs) -> bool: @@ -296,26 +300,26 @@ def wait_for( by=by, value=value, **kwargs) - log.debug(f'found {by}: {value}') + log.debug(f'waiting for {by}: {self.url} {value}') return value except: - log.error(f'{by} not found: {value}', enable_traceback=False) + log.error(f'{by} not found: {self.url} {value}', enable_traceback=False) else: self.find_element( by=by, value=value, **kwargs) - log.debug(f'found {by}: {value}') + log.debug(f'waiting for {by}: {self.url} {value}') return value except Exception as error: - log.error(f'waiting for {by}: {value}, {error}', + log.error(f'not found {by}: {self.url} {value}, {error}', enable_traceback=False) Sleeper.seconds(f'wait for', 1) retry += 1 if retry > retries: - log.error(f'max wait reached', enable_traceback=False) + log.error(f'max wait reached: {self.url}', enable_traceback=False) break return False From 882994ab86ea56c19cad1d7e507eaaeb5fa706d0 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 15:41:58 -0400 Subject: [PATCH 36/62] selenium: add set_locale --- .../integrations/seleniumWrapper/config_webdriver_chrome.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index ab2b3152..4bd622fb 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -247,6 +247,10 @@ def set_chromedriver(self, chromedriver: str): self.update_paths() return self + def set_locale(self, locale: str = 'en'): + self.chrome_options.add_argument(f"--lang={locale}") + return self + def set_user_agent(self, user_agent: str): self.chrome_options.add_argument(f"user-agent={user_agent}") return self From 67a64ec1ecd64dde155010afbb946dc3787deaea Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 22:49:09 -0400 Subject: [PATCH 37/62] selenium: add enable_translate --- .../seleniumWrapper/config_webdriver_chrome.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index 4bd622fb..bd8dd365 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -60,6 +60,7 @@ def disable_certificate_verification(self): def disable_extensions(self): self.chrome_options.add_argument("--disable-extensions") + return self def disable_infobars(self): self.chrome_options.add_argument("--disable-infobars") @@ -112,6 +113,17 @@ def enable_maximized(self): self.chrome_options.add_argument('--start-maximized') return self + def enable_translate(self, native_language: str = 'en'): + prefs = { + "translate_whitelists": {"your native language": native_language}, + "translate": {"enabled": "True"} + } + self.chrome_options.add_experimental_option( + name="prefs", + value=prefs, + ) + return self + def close(self): """close From e3627ad153abb052eefb60cad34f41974ff19217 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 22:49:19 -0400 Subject: [PATCH 38/62] selenium: add set_locale_experimental --- .../integrations/seleniumWrapper/config_webdriver_chrome.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index bd8dd365..d10a4bfd 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -263,6 +263,12 @@ def set_locale(self, locale: str = 'en'): self.chrome_options.add_argument(f"--lang={locale}") return self + def set_locale_experimental(self, locale: str = 'en-US'): + self.chrome_options.add_experimental_option( + name='prefs', + value={'intl.accept_languages': locale}) + return self + def set_user_agent(self, user_agent: str): self.chrome_options.add_argument(f"user-agent={user_agent}") return self From 8d38299dc26fff515ecac049cbfa087d281f0df3 Mon Sep 17 00:00:00 2001 From: Eric <58240560+naisanzaa@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:36:06 -0700 Subject: [PATCH 39/62] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c5431af8..c10f2df2 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ [![master](https://github.com/TheShellLand/automonisaur/actions/workflows/python37.yml/badge.svg)](https://github.com/TheShellLand/automonisaur/actions/workflows/python37.yml) [![master](https://github.com/TheShellLand/automonisaur/actions/workflows/python36.yml/badge.svg)](https://github.com/TheShellLand/automonisaur/actions/workflows/python36.yml) -[![Downloads](https://pepy.tech/badge/automonisaur)](https://pepy.tech/project/automonisaur) -[![Downloads](https://pepy.tech/badge/automonisaur/month)](https://pepy.tech/project/automonisaur) -[![Downloads](https://pepy.tech/badge/automonisaur/week)](https://pepy.tech/project/automonisaur) +[![Downloads](https://static.pepy.tech/badge/automonisaur)](https://pepy.tech/project/automonisaur) +[![Downloads](https://static.pepy.tech/badge/automonisaur/month)](https://pepy.tech/project/automonisaur) +[![Downloads](https://static.pepy.tech/badge/automonisaur/week)](https://pepy.tech/project/automonisaur) [//]: # ([![codecov](https://codecov.io/gh/TheShellLand/automonisaur/branch/master/graph/badge.svg)](https://codecov.io/gh/TheShellLand/automonisaur)) From 74974c19c406a4354336e55d87b77e7276800638 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Thu, 17 Aug 2023 22:58:31 -0400 Subject: [PATCH 40/62] update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index c10f2df2..53424170 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,20 @@ Github issues and feature requests welcomed. ### Integrations - airport +- beautifulsoup - elasticsearch +- facebook groups - flask +- google auth api - google people api +- google sheets api - instagram - logging - minio - neo4j - nmap - requests +- scrapy - selenium - sentryio - slack From 4221efbf5619a01b0bc359cc3822f2d42a982e2e Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Sat, 19 Aug 2023 09:48:19 -0400 Subject: [PATCH 41/62] selenium: set logs to debug --- automon/integrations/seleniumWrapper/browser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index b0541ca7..ff8ed56c 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -156,14 +156,14 @@ def find_element( **kwargs): """find element""" element = self.webdriver.find_element(value=value, by=by, **kwargs) - log.info(f'found element: {self.url} {element.text}') + log.debug(f'found element: {self.url} {element.text}') return element @_is_running def find_xpath(self, value: str, by: By = By.XPATH, **kwargs): """find xpath""" xpath = self.find_element(value=value, by=by, **kwargs) - log.info(f'found xpath: {self.url} {xpath.text}') + log.debug(f'found xpath: {self.url} {xpath.text}') return xpath @_is_running From c2c28c7c342f103f7b45f82d18b26a3c3c056025 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Sat, 19 Aug 2023 09:50:11 -0400 Subject: [PATCH 42/62] sleeper: set logs to debug --- automon/helpers/sleeper.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/automon/helpers/sleeper.py b/automon/helpers/sleeper.py index 15e631f5..82f6479d 100644 --- a/automon/helpers/sleeper.py +++ b/automon/helpers/sleeper.py @@ -3,7 +3,7 @@ from automon.log import Logging -log = Logging('Sleeper', level=Logging.INFO) +log = Logging(name='Sleeper', level=Logging.INFO) class Sleeper: @@ -14,10 +14,10 @@ def seconds(caller: object or str, seconds: int) -> time.sleep: sleep = seconds if sleep < 2: - log.info(f'[{Sleeper.seconds.__name__}] ' + log.debug(f'[{Sleeper.seconds.__name__}] ' f'[{caller}] sleeping for {sleep} second') else: - log.info(f'[{Sleeper.seconds.__name__}] ' + log.debug(f'[{Sleeper.seconds.__name__}] ' f'[{caller}] sleeping for {sleep} seconds') return time.sleep(sleep) @@ -25,7 +25,7 @@ def seconds(caller: object or str, seconds: int) -> time.sleep: def minute(caller: object or str, sleep: int = 60) -> time.sleep: """Sleep for a minute""" - log.info(f'[{Sleeper.minute.__name__}] ' + log.debug(f'[{Sleeper.minute.__name__}] ' f'[{caller}] sleeping for {sleep} seconds') return time.sleep(sleep) @@ -35,7 +35,7 @@ def within_a_minute(caller, sleep: int = None): sleep = sleep if isinstance(sleep, int) else \ random.choice(range(1, 1 * 60)) - log.info(f'[{Sleeper.within_a_minute.__name__}] ' + log.debug(f'[{Sleeper.within_a_minute.__name__}] ' f'[{caller}] sleeping for {sleep} seconds') return time.sleep(sleep) @@ -44,7 +44,7 @@ def minutes(caller, minutes: int): """Sleep for this many minutes""" sleep = minutes * 60 - log.info(f'[{Sleeper.minutes.__name__}] ' + log.debug(f'[{Sleeper.minutes.__name__}] ' f'[{caller}] sleeping for {sleep} minutes') return time.sleep(sleep) @@ -54,7 +54,7 @@ def hour(caller, hour: int = 1): sleep = hour if not hour else random.choice( range(1, hour * 60 * 60)) - log.info(f'[{Sleeper.hour.__name__}] ' + log.debug(f'[{Sleeper.hour.__name__}] ' f'[{caller}] sleeping for {sleep} seconds') return time.sleep(sleep) @@ -63,7 +63,7 @@ def hours(caller, hours): """Sleep for this many hours""" sleep = hours * 60 * 60 - log.info(f'[{Sleeper.hours.__name__}] ' + log.debug(f'[{Sleeper.hours.__name__}] ' f'[{caller}] sleeping for {hours} hours') return time.sleep(sleep) @@ -73,7 +73,7 @@ def day(caller, hours: int = 24): sleep = hours if not hours else random.choice( range(1, hours * 60 * 60)) - log.info(f'[{Sleeper.day.__name__}] ' + log.debug(f'[{Sleeper.day.__name__}] ' f'[{caller}] sleeping for {sleep} seconds') return time.sleep(sleep) @@ -82,7 +82,7 @@ def daily(caller, hours: int = 24): """Sleep for one day""" sleep = hours if not hours else hours * 60 * 60 - log.info(f'[{Sleeper.daily.__name__}] ' + log.debug(f'[{Sleeper.daily.__name__}] ' f'[{caller}] sleeping for {sleep} seconds') return time.sleep(sleep) @@ -92,6 +92,6 @@ def time_range(caller, seconds: int): """ sleep = seconds if not seconds else random.choice( range(1, seconds)) - log.info(f'[{Sleeper.time_range.__name__}] ' + log.debug(f'[{Sleeper.time_range.__name__}] ' f'[{caller}] sleeping for {sleep} seconds') return time.sleep(sleep) From 5c5953f91d9314d294d29a7f39389018bed250b2 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Sun, 20 Aug 2023 02:34:10 +0800 Subject: [PATCH 43/62] selenium: fix $PATH too long --- .../integrations/seleniumWrapper/config_webdriver_chrome.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index d10a4bfd..1aec76fa 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -19,7 +19,6 @@ def __init__(self): self._chromedriver = environ('SELENIUM_CHROMEDRIVER_PATH') self._ChromeService = None - self._path_updated = None self.update_paths() self._window_size = set_window_size() @@ -293,9 +292,8 @@ def stop_client(self): def update_paths(self): if self.chromedriver: - if not self._path_updated: + if self.chromedriver not in os.getenv('PATH'): os.environ['PATH'] = f"{os.getenv('PATH')}:{self._chromedriver}" - self._path_updated = True def quit(self): """quit From b4cf93511e1a3dcf15f31eb15f1255a67371ae2d Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 16 Aug 2023 03:30:43 +0800 Subject: [PATCH 44/62] facebook: add group info --- automon/integrations/facebook/__init__.py | 1 + automon/integrations/facebook/groups.py | 345 ++++++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 automon/integrations/facebook/__init__.py create mode 100644 automon/integrations/facebook/groups.py diff --git a/automon/integrations/facebook/__init__.py b/automon/integrations/facebook/__init__.py new file mode 100644 index 00000000..718b9b6b --- /dev/null +++ b/automon/integrations/facebook/__init__.py @@ -0,0 +1 @@ +from .groups import FacebookGroups diff --git a/automon/integrations/facebook/groups.py b/automon/integrations/facebook/groups.py new file mode 100644 index 00000000..c00cd453 --- /dev/null +++ b/automon/integrations/facebook/groups.py @@ -0,0 +1,345 @@ +import datetime + +from automon.log import Logging +from automon.integrations.seleniumWrapper import SeleniumBrowser + +log = Logging(name='FacebookGroups', level=Logging.DEBUG) + + +class FacebookGroups(object): + xpath_about = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[3]/div/div/div/div/div/div/div[1]/div/div/div/div/div[2]/a[1]/div[1]/span', + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[3]/div/div/div/div/div/div/div[1]/div/div/div/div/div[1]/div[1]/span', + ] + xpath_popup_close = [ + '/html/body/div[1]/div/div[1]/div/div[5]/div/div/div[1]/div/div[2]/div/div/div/div[1]/div/i', + ] + xpath_content_unavailble = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div/div/div[1]/div[2]/div[1]/span', + ] + xpath_creation_date = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div/div[2]/div/div/div[4]/div/div/div/div/div/div[3]/div/div/div/div/div/div[2]/div/div[3]/div/div/div[2]/div/div/span', + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[3]/div/div/div/div/div/div[2]/div/div[3]/div/div/div[2]/div/div/span', + ] + xpath_history = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[1]/div/div/div/div/div/div[2]/div[4]/div/div/div[2]/div/div[2]/span/span', + ] + xpath_title = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[1]/div[2]/div/div/div/div/div[1]/div/div/div/div/div/div[1]/h1/span/a', + ] + xpath_members = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div/div[2]/div/div/div[4]/div/div/div/div/div/div[3]/div/div/div/div/div/div[2]/div/div[2]/div/div/div[2]/div/div[1]/span', + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[3]/div/div/div/div/div/div[2]/div/div[2]/div/div/div[2]/div/div[1]/span', + ] + xpath_posts_today = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[3]/div/div/div/div/div/div[2]/div/div[1]/div/div/div[2]/div/div[1]/span', + ] + xpath_posts_monthly = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[3]/div/div/div/div/div/div[2]/div/div[1]/div/div/div[2]/div/div[2]/span', + ] + xpath_privacy = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[1]/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div/div[1]/span/span', + ] + xpath_privacy_details = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[1]/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div/div[2]/span/span', + ] + xpath_visible = [ + '/html/body/div[1]/div/div[1]/div/div[3]/div/div/div/div[1]/div[1]/div[4]/div/div/div/div/div/div[1]/div/div/div/div/div/div[2]/div[3]/div/div/div[2]/div/div[2]/span/span', + ] + + def __init__(self, url: str = None): + """Facebook Groups object + + Depends on Selenium""" + self._content_unavailable = None + self._creation_date = None + self._creation_date_timestamp = None + self._history = None + self._members = None + self._members_count = None + self._posts_monthly = None + self._posts_monthly_count = None + self._posts_today = None + self._posts_today_count = None + self._privacy = None + self._privacy_details = None + self._title = None + self._url = url + self._visible = None + + self._browser = None + + def __repr__(self): + return f'{self.__dict__}' + + @property + def content_unavailable(self): + """This content isn't available right now""" + if not self._browser: + self.start() + + if not self._content_unavailable: + try: + xpath_content_unavailble = self._browser.wait_for_xpath(self.xpath_content_unavailble) + self._content_unavailable = self._browser.find_xpath(xpath_content_unavailble).text + except Exception as e: + log.error(f"can't get content message {self.url}: {e}", enable_traceback=False) + + return self._content_unavailable + + @property + def creation_date(self): + if not self._browser: + self.start() + + if not self._creation_date: + try: + xpath_creation_date = self._browser.wait_for_xpath(self.xpath_creation_date) + self._creation_date = self._browser.find_xpath(xpath_creation_date).text + except Exception as e: + log.error(f"can't get creation date {self.url}: {e}", enable_traceback=False) + + return self._creation_date + + @property + def creation_date_timestamp(self): + if self._creation_date: + # TODO: convert date to datetime timestamp + return self._creation_date_timestamp + + @property + def history(self): + if not self._browser: + self.start() + + if not self._history: + try: + xpath_history = self._browser.wait_for_xpath(self.xpath_history) + self._history = self._browser.find_xpath(xpath_history).text + except Exception as e: + log.error(f"can't get history {self.url}: {e}", enable_traceback=False) + + return self._history + + @property + def members(self): + if not self._browser: + self.start() + + if not self._members: + try: + xpath_members = self._browser.wait_for_xpath(self.xpath_members) + self._members = self._browser.find_xpath(xpath_members).text + # TODO: need to clean up string from members and remove bad chars + except Exception as e: + log.error(f"can't get member count {self.url}: {e}", enable_traceback=False) + + return self._members + + @property + def members_count(self): + if not self._browser: + self.start() + + if self._members: + count = [x for x in self._members] + count = [x for x in count if x in [str(x) for x in range(0, 10)]] + if count: + self._members_count = int(''.join(count)) if count else 0 + + return self._members_count + + @property + def posts_monthly(self): + if not self._browser: + self.start() + + if not self._posts_monthly: + try: + xpath_monthly_posts = self._browser.wait_for_xpath(self.xpath_posts_monthly) + self._posts_monthly = self._browser.find_xpath(xpath_monthly_posts).text + except Exception as e: + print(f"can't get monthly posts {self.url}: {e}") + + return self._posts_monthly + + @property + def posts_monthly_count(self): + if not self._browser: + self.start() + + if self._posts_monthly: + count = [x for x in self._posts_monthly] + count = [x for x in count if x in [str(x) for x in range(0, 10)]] + if count: + self._posts_monthly_count = int(''.join(count)) if count else 0 + + return self._posts_monthly_count + + @property + def posts_today(self): + if not self._browser: + self.start() + + if not self._posts_today: + try: + xpath_posts_today = self._browser.wait_for_xpath(self.xpath_posts_today) + self._posts_today = self._browser.find_xpath(xpath_posts_today).text + except Exception as e: + log.error(f"can't get today's posts {self.url}: {e}", enable_traceback=False) + + return self._posts_today + + @property + def posts_today_count(self): + if not self._browser: + self.start() + + if self.posts_today: + count = [x for x in self.posts_today] + count = [x for x in count if x in [str(x) for x in range(0, 10)]] + if count: + self._posts_today_count = int(''.join(count)) if count else 0 + + return self._posts_today_count + + @property + def privacy(self): + if not self._browser: + self.start() + + if not self._privacy: + try: + xpath_privacy = self._browser.wait_for_xpath(self.xpath_privacy) + self._privacy = self._browser.find_xpath(xpath_privacy).text + except Exception as e: + log.error(f"can't get privacy {self.url}: {e}", enable_traceback=False) + + return self._privacy + + @property + def privacy_details(self): + if not self._browser: + self.start() + + if not self._privacy_details: + try: + xpath_privacy_details = self._browser.wait_for_xpath(self.xpath_privacy_details) + self._privacy_details = self._browser.find_xpath(xpath_privacy_details).text + except Exception as e: + log.error(f"can't get privacy details {self.url}: {e}", enable_traceback=False) + + return self._privacy_details + + @property + def title(self) -> str: + if not self._browser: + self.start() + + if not self._title: + try: + xpath_title = self._browser.wait_for_xpath(self.xpath_title) + self._title = self._browser.find_xpath(xpath_title).text + except Exception as e: + log.error(f"can't get title {self.url}: {e}", enable_traceback=False) + + return self._title + + @property + def url(self) -> str: + return self._url + + @property + def visible(self) -> str: + if not self._browser: + self.start() + + if not self._visible: + try: + xpath_visible = self._browser.wait_for_xpath(self.xpath_visible) + self._visible = self._browser.find_xpath(xpath_visible).text + except Exception as e: + log.error(f"can't get visible {self.url}: {e}", enable_traceback=False) + + return self._visible + + def get(self, url: str = None) -> bool: + """get url""" + if not self._browser: + self.start() + + if not url and not self.url: + raise Exception(f"missing url") + + return self._browser.get(url=url or self.url) + + def get_about(self): + url = f'{self.url}/about' + return self.get(url=url) + + def run(self): + """run selenium browser""" + if self._browser: + return self._browser.run() + + def restart(self): + """quit and start new instance of selenium""" + if self._browser: + self.quit() + return self.start() + + def start(self, headless: bool = True): + """start new instance of selenium""" + self._browser = SeleniumBrowser() + + if headless: + self._browser.config.set_webdriver.Chrome().in_headless().set_locale_experimental() + else: + self._browser.config.set_webdriver.Chrome().set_locale_experimental() + + return self._browser.run() + + def stop(self): + """alias to quit""" + return self.quit() + + def to_dict(self): + self.content_unavailable + self.creation_date + self.creation_date_timestamp + self.history + self.members + self.members_count + self.posts_monthly + self.posts_monthly_count + self.posts_today + self.posts_today_count + self.privacy + self.privacy_details + self.title + self.url + self.visible + + return dict( + content_unavailable=self._content_unavailable, + creation_date=self._creation_date, + creation_date_timestamp=self._creation_date_timestamp, + history=self._history, + members=self._members, + members_count=self._members_count, + posts_monthly=self._posts_monthly, + posts_monthly_count=self._posts_monthly_count, + posts_today=self._posts_today, + posts_today_count=self._posts_today_count, + privacy=self._privacy, + privacy_details=self._privacy_details, + title=self._title, + url=self._url, + visible=self._visible, + status=self._browser.status, + ) + + def quit(self): + """quit selenium""" + if self._browser: + return self._browser.quit() From c7789674f4ae2f3b0a137d16755fd0ee3fefeb79 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 21 Aug 2023 08:37:53 +0800 Subject: [PATCH 45/62] selenium: update request and status --- automon/integrations/seleniumWrapper/browser.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index ff8ed56c..e7402b21 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -25,13 +25,13 @@ class SeleniumBrowser(object): config: SeleniumConfig webdriver: selenium.webdriver - status: str + status: int def __init__(self, config: SeleniumConfig = None): """A selenium wrapper""" self._config = config or SeleniumConfig() - self.status = '' + self.request = None def __repr__(self): if self.url: @@ -69,6 +69,11 @@ def keys(self): """Set of special keys codes""" return selenium.webdriver.common.keys.Keys + @property + def status(self): + if self.request is not None: + return self.request.results.status_code + # @property # def type(self) -> SeleniumBrowserType: # return SeleniumBrowserType(self.config) @@ -171,7 +176,7 @@ def get(self, url: str, **kwargs) -> bool: """get url""" try: self.webdriver.get(url, **kwargs) - self.status = RequestsClient(url=url).results + self.request = RequestsClient(url=url) msg = f'GET {self.status} {self.webdriver.current_url}' if kwargs: @@ -179,7 +184,7 @@ def get(self, url: str, **kwargs) -> bool: log.debug(msg) return True except Exception as e: - self.status = RequestsClient(url=url).results + self.request = RequestsClient(url=url) msg = f'GET {self.status}: {e}' log.error(msg, enable_traceback=False) From 7e2046db89f517ee1ffca4511357acd00774c086 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 11:25:29 -0400 Subject: [PATCH 46/62] google: rename auth and config --- automon/integrations/google/auth/__init__.py | 4 ++-- automon/integrations/google/auth/client.py | 10 +++++----- automon/integrations/google/auth/config.py | 2 +- .../google/auth/tests/test_config_Credentials.py | 4 ++-- .../integrations/google/auth/tests/test_google_auth.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/automon/integrations/google/auth/__init__.py b/automon/integrations/google/auth/__init__.py index 293d82bd..a96ebe64 100644 --- a/automon/integrations/google/auth/__init__.py +++ b/automon/integrations/google/auth/__init__.py @@ -1,2 +1,2 @@ -from .client import AuthClient -from .config import AuthConfig +from .client import GoogleAuthClient +from .config import GoogleAuthConfig diff --git a/automon/integrations/google/auth/client.py b/automon/integrations/google/auth/client.py index b8453e9a..d4fa08c0 100644 --- a/automon/integrations/google/auth/client.py +++ b/automon/integrations/google/auth/client.py @@ -5,24 +5,24 @@ from automon.log import Logging -from .config import AuthConfig +from .config import GoogleAuthConfig -log = Logging(name='AuthClient', level=Logging.DEBUG) +log = Logging(name='GoogleAuthClient', level=Logging.DEBUG) -class AuthClient(object): +class GoogleAuthClient(object): """Google Auth client""" def __init__( self, - config: AuthConfig = None, + config: GoogleAuthConfig = None, serviceName: str = None, scopes: list = None, version: str = None, **kwargs, ): - self.config = config or AuthConfig( + self.config = config or GoogleAuthConfig( serviceName=serviceName, scopes=scopes, version=version, diff --git a/automon/integrations/google/auth/config.py b/automon/integrations/google/auth/config.py index a7f1a972..631a4b48 100644 --- a/automon/integrations/google/auth/config.py +++ b/automon/integrations/google/auth/config.py @@ -14,7 +14,7 @@ log = Logging(name='AuthConfig', level=Logging.DEBUG) -class AuthConfig(object): +class GoogleAuthConfig(object): """Google Auth config""" def __init__( diff --git a/automon/integrations/google/auth/tests/test_config_Credentials.py b/automon/integrations/google/auth/tests/test_config_Credentials.py index fcfb4f5d..48c2e3c0 100644 --- a/automon/integrations/google/auth/tests/test_config_Credentials.py +++ b/automon/integrations/google/auth/tests/test_config_Credentials.py @@ -1,11 +1,11 @@ import unittest -from automon.integrations.google.auth import AuthConfig +from automon.integrations.google.auth import GoogleAuthConfig class MyTestCase(unittest.TestCase): def test_something(self): - test = AuthConfig() + test = GoogleAuthConfig() if test.Credentials: self.assertTrue(test.Credentials) diff --git a/automon/integrations/google/auth/tests/test_google_auth.py b/automon/integrations/google/auth/tests/test_google_auth.py index a9dfec6b..57d0ab8b 100644 --- a/automon/integrations/google/auth/tests/test_google_auth.py +++ b/automon/integrations/google/auth/tests/test_google_auth.py @@ -1,11 +1,11 @@ import unittest -from automon.integrations.google.auth import AuthClient +from automon.integrations.google.auth import GoogleAuthClient class MyTestCase(unittest.TestCase): def test_authenticate(self): - test = AuthClient() + test = GoogleAuthClient() # scopes = ['https://www.googleapis.com/auth/contacts.readonly'] # client = AuthClient(serviceName='people', scopes=scopes) if test.authenticate(): From 7a4a065f7a8e9339e02ba2adce6238635f90e58b Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 11:47:05 -0400 Subject: [PATCH 47/62] google: fix auth check --- automon/integrations/google/auth/client.py | 4 +++- automon/integrations/google/auth/config.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/automon/integrations/google/auth/client.py b/automon/integrations/google/auth/client.py index d4fa08c0..a479679b 100644 --- a/automon/integrations/google/auth/client.py +++ b/automon/integrations/google/auth/client.py @@ -82,7 +82,9 @@ def authenticate_oauth(self) -> bool: def authenticate_service_account(self) -> bool: """authenticate service account""" - return True + if self.config.Credentials: + return True + return False def is_connected(self) -> bool: """Check if authenticated to make requests""" diff --git a/automon/integrations/google/auth/config.py b/automon/integrations/google/auth/config.py index 631a4b48..c160473f 100644 --- a/automon/integrations/google/auth/config.py +++ b/automon/integrations/google/auth/config.py @@ -1,4 +1,5 @@ import os +import json import base64 import google.auth.crypt @@ -11,7 +12,7 @@ from automon.log import Logging from automon.helpers import environ -log = Logging(name='AuthConfig', level=Logging.DEBUG) +log = Logging(name='GoogleAuthConfig', level=Logging.DEBUG) class GoogleAuthConfig(object): @@ -57,7 +58,7 @@ def Credentials(self): except: pass - log.error(f'Missing credentials', enable_traceback=False) + log.error(f'Missing GOOGLE_CREDENTIALS or GOOGLE_CREDENTIALS_BASE64', enable_traceback=False) @property def _GOOGLE_CREDENTIALS(self): @@ -99,12 +100,15 @@ def CredentialsServiceAccountInfo(self) -> google.oauth2.service_account.Credent self.base64_to_dict() ) - def base64_to_dict(self, base64: str = None): + def base64_to_dict(self, base64_str: str = None) -> dict: """convert credential json to dict""" - if not base64 and self._GOOGLE_CREDENTIALS_BASE64: - base64 = self._GOOGLE_CREDENTIALS_BASE64 + if not base64_str and not self._GOOGLE_CREDENTIALS_BASE64: + raise Exception(f'Missing GOOGLE_CREDENTIALS_BASE6') - return base64.decode(base64) + base64_str = base64_str or self._GOOGLE_CREDENTIALS_BASE64 + return json.loads( + base64.b64decode(base64_str) + ) def file_to_base64(self, path: str = None): """convert file to base64""" From f6aab5a9b14c72e6b57780c917c23e86ac9f7786 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 11:54:23 -0400 Subject: [PATCH 48/62] google: rename classes --- automon/integrations/google/__init__.py | 6 +++--- automon/integrations/google/gmail/__init__.py | 4 ++-- automon/integrations/google/gmail/v1/__init__.py | 4 ++-- automon/integrations/google/gmail/v1/client.py | 8 ++++---- automon/integrations/google/gmail/v1/config.py | 4 ++-- automon/integrations/google/people/__init__.py | 4 ++-- automon/integrations/google/people/client.py | 16 ++++++++-------- automon/integrations/google/people/config.py | 4 ++-- automon/integrations/google/people/person.py | 2 +- automon/integrations/google/people/results.py | 2 +- .../google/people/tests/test_google_contacts.py | 4 ++-- .../people/tests/test_google_contacts_neo4j.py | 4 ++-- automon/integrations/google/people/urls.py | 4 ++-- .../google/tests/test_google_contacts.py | 4 ++-- .../google/tests/test_google_contacts_neo4j.py | 4 ++-- 15 files changed, 37 insertions(+), 37 deletions(-) diff --git a/automon/integrations/google/__init__.py b/automon/integrations/google/__init__.py index dc84b86a..63c280a3 100644 --- a/automon/integrations/google/__init__.py +++ b/automon/integrations/google/__init__.py @@ -1,3 +1,3 @@ -from .auth import AuthClient -from .gmail import GmailClientV1 -from .people import PeopleClient +from .auth import GoogleAuthClient +from .gmail import GoogleGmailClient +from .people import GooglePeopleClient diff --git a/automon/integrations/google/gmail/__init__.py b/automon/integrations/google/gmail/__init__.py index 37d7a424..f11c7114 100644 --- a/automon/integrations/google/gmail/__init__.py +++ b/automon/integrations/google/gmail/__init__.py @@ -1,2 +1,2 @@ -from .v1 import GmailClient as GmailClientV1 -from .v1 import GmailConfig as GmailConfigV1 +from .v1 import GoogleGmailClient +from .v1 import GoogleGmailConfig diff --git a/automon/integrations/google/gmail/v1/__init__.py b/automon/integrations/google/gmail/v1/__init__.py index 063d8da6..c26f7cff 100644 --- a/automon/integrations/google/gmail/v1/__init__.py +++ b/automon/integrations/google/gmail/v1/__init__.py @@ -1,2 +1,2 @@ -from .client import GmailClient -from .config import GmailConfig +from .client import GoogleGmailClient +from .config import GoogleGmailConfig diff --git a/automon/integrations/google/gmail/v1/client.py b/automon/integrations/google/gmail/v1/client.py index b9579176..347eb1ba 100644 --- a/automon/integrations/google/gmail/v1/client.py +++ b/automon/integrations/google/gmail/v1/client.py @@ -1,12 +1,12 @@ from automon.integrations.requestsWrapper import RequestsClient -from .config import GmailConfig +from .config import GoogleGmailConfig -class GmailClient: +class GoogleGmailClient: - def __init__(self, api_key: str = None, user: str = None, password: str = None, config: GmailConfig = None): - self.config = config or GmailConfig(user=user, password=password, api_key=api_key) + def __init__(self, api_key: str = None, user: str = None, password: str = None, config: GoogleGmailConfig = None): + self.config = config or GoogleGmailConfig(user=user, password=password, api_key=api_key) self.endpoint = self.config.endpoint self.userId = self.config.userId diff --git a/automon/integrations/google/gmail/v1/config.py b/automon/integrations/google/gmail/v1/config.py index f5413c22..fecff96d 100644 --- a/automon/integrations/google/gmail/v1/config.py +++ b/automon/integrations/google/gmail/v1/config.py @@ -2,10 +2,10 @@ from automon.log import Logging -log = Logging(name='GmailConfig', level=Logging.DEBUG) +log = Logging(name='GoogleGmailConfig', level=Logging.DEBUG) -class GmailConfig: +class GoogleGmailConfig: def __init__(self, endpoint: str = None, api_key: str = None, user: str = None, diff --git a/automon/integrations/google/people/__init__.py b/automon/integrations/google/people/__init__.py index 458c9651..a802b592 100644 --- a/automon/integrations/google/people/__init__.py +++ b/automon/integrations/google/people/__init__.py @@ -1,2 +1,2 @@ -from .client import PeopleClient -from .config import PeopleConfig +from .client import GooglePeopleClient +from .config import GooglePeopleConfig diff --git a/automon/integrations/google/people/client.py b/automon/integrations/google/people/client.py index 0c781a31..a5e9f6fd 100644 --- a/automon/integrations/google/people/client.py +++ b/automon/integrations/google/people/client.py @@ -9,20 +9,20 @@ from automon.log import Logging -from .urls import PeopleUrls -from .config import PeopleConfig +from .urls import GooglePeopleUrls +from .config import GooglePeopleConfig from .results import ConnectionsResults -log = Logging(name='PeopleClient', level=Logging.DEBUG) +log = Logging(name='GooglePeopleClient', level=Logging.DEBUG) -class PeopleClient: +class GooglePeopleClient: def __init__(self, client_id: str = None, client_secret: str = None, - config: PeopleConfig = None): + config: GooglePeopleConfig = None): """Google People API Client""" - self.config = config or PeopleConfig( + self.config = config or GooglePeopleConfig( client_id=client_id, client_secret=client_secret ) @@ -132,10 +132,10 @@ def list_connections( """ if not resourceName: - resourceName = PeopleUrls().resourceName() + resourceName = GooglePeopleUrls().resourceName() if not personFields: - personFields = PeopleUrls().personFields_toStr() + personFields = GooglePeopleUrls().personFields_toStr() return self._list( resourceName=resourceName, diff --git a/automon/integrations/google/people/config.py b/automon/integrations/google/people/config.py index 0cc13a32..96c5048f 100644 --- a/automon/integrations/google/people/config.py +++ b/automon/integrations/google/people/config.py @@ -13,10 +13,10 @@ from automon.log import Logging from automon.helpers import environ -log = Logging(name='PeopleConfig', level=Logging.DEBUG) +log = Logging(name='GooglePeopleConfig', level=Logging.DEBUG) -class PeopleConfig: +class GooglePeopleConfig: def __init__(self, token=None, diff --git a/automon/integrations/google/people/person.py b/automon/integrations/google/people/person.py index c3a6662b..f5b1f362 100644 --- a/automon/integrations/google/people/person.py +++ b/automon/integrations/google/people/person.py @@ -2,7 +2,7 @@ from automon.log import Logging -log = Logging(level=Logging.DEBUG) +log = Logging(name='GooglePeople', level=Logging.DEBUG) class AgeRange(Enum): diff --git a/automon/integrations/google/people/results.py b/automon/integrations/google/people/results.py index f5e18c38..2f573fe8 100644 --- a/automon/integrations/google/people/results.py +++ b/automon/integrations/google/people/results.py @@ -2,7 +2,7 @@ from .person import Person -log = Logging(name='PeopleResults', level=Logging.DEBUG) +log = Logging(name='GooglePeopleResults', level=Logging.DEBUG) class ConnectionsResults: diff --git a/automon/integrations/google/people/tests/test_google_contacts.py b/automon/integrations/google/people/tests/test_google_contacts.py index a5c18459..d3755a71 100644 --- a/automon/integrations/google/people/tests/test_google_contacts.py +++ b/automon/integrations/google/people/tests/test_google_contacts.py @@ -1,8 +1,8 @@ import unittest -from automon.integrations.google import PeopleClient +from automon.integrations.google import GooglePeopleClient -c = PeopleClient() +c = GooglePeopleClient() class TestClient(unittest.TestCase): diff --git a/automon/integrations/google/people/tests/test_google_contacts_neo4j.py b/automon/integrations/google/people/tests/test_google_contacts_neo4j.py index 15cf1c3d..9c5ba56a 100644 --- a/automon/integrations/google/people/tests/test_google_contacts_neo4j.py +++ b/automon/integrations/google/people/tests/test_google_contacts_neo4j.py @@ -1,9 +1,9 @@ import unittest -from automon.integrations.google import PeopleClient +from automon.integrations.google import GooglePeopleClient from automon.integrations.neo4jWrapper import Neo4jClient -c = PeopleClient() +c = GooglePeopleClient() n = Neo4jClient() diff --git a/automon/integrations/google/people/urls.py b/automon/integrations/google/people/urls.py index 6c05c5ea..fb80ea40 100644 --- a/automon/integrations/google/people/urls.py +++ b/automon/integrations/google/people/urls.py @@ -1,9 +1,9 @@ from automon.log import Logging -log = Logging(name='PeopleUrls', level=Logging.ERROR) +log = Logging(name='GooglePeopleUrls', level=Logging.ERROR) -class PeopleUrls: +class GooglePeopleUrls: PEOPLE_API = 'https://people.googleapis.com' API_VER = 'v1' BASE_URL = f'{PEOPLE_API}/{API_VER}' diff --git a/automon/integrations/google/tests/test_google_contacts.py b/automon/integrations/google/tests/test_google_contacts.py index a5c18459..d3755a71 100644 --- a/automon/integrations/google/tests/test_google_contacts.py +++ b/automon/integrations/google/tests/test_google_contacts.py @@ -1,8 +1,8 @@ import unittest -from automon.integrations.google import PeopleClient +from automon.integrations.google import GooglePeopleClient -c = PeopleClient() +c = GooglePeopleClient() class TestClient(unittest.TestCase): diff --git a/automon/integrations/google/tests/test_google_contacts_neo4j.py b/automon/integrations/google/tests/test_google_contacts_neo4j.py index 15cf1c3d..9c5ba56a 100644 --- a/automon/integrations/google/tests/test_google_contacts_neo4j.py +++ b/automon/integrations/google/tests/test_google_contacts_neo4j.py @@ -1,9 +1,9 @@ import unittest -from automon.integrations.google import PeopleClient +from automon.integrations.google import GooglePeopleClient from automon.integrations.neo4jWrapper import Neo4jClient -c = PeopleClient() +c = GooglePeopleClient() n = Neo4jClient() From 3cbf231f696002666c9e301503e78cfd01c1eceb Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 26 Jul 2023 00:27:25 +0800 Subject: [PATCH 49/62] sheets: add client and config --- .../integrations/google/sheets/__init__.py | 2 + automon/integrations/google/sheets/client.py | 123 +++++++++ automon/integrations/google/sheets/config.py | 27 ++ .../google/sheets/tests/__init__.py | 0 .../google/sheets/tests/test_google_sheets.py | 234 ++++++++++++++++++ .../sheets/tests/test_google_sheets_AUDIT.py | 59 +++++ 6 files changed, 445 insertions(+) create mode 100644 automon/integrations/google/sheets/__init__.py create mode 100644 automon/integrations/google/sheets/client.py create mode 100644 automon/integrations/google/sheets/config.py create mode 100644 automon/integrations/google/sheets/tests/__init__.py create mode 100644 automon/integrations/google/sheets/tests/test_google_sheets.py create mode 100644 automon/integrations/google/sheets/tests/test_google_sheets_AUDIT.py diff --git a/automon/integrations/google/sheets/__init__.py b/automon/integrations/google/sheets/__init__.py new file mode 100644 index 00000000..0f5f8d55 --- /dev/null +++ b/automon/integrations/google/sheets/__init__.py @@ -0,0 +1,2 @@ +from .client import GoogleSheetsClient +from .config import GoogleSheetsConfig diff --git a/automon/integrations/google/sheets/client.py b/automon/integrations/google/sheets/client.py new file mode 100644 index 00000000..27087263 --- /dev/null +++ b/automon/integrations/google/sheets/client.py @@ -0,0 +1,123 @@ +from automon.log import Logging +from automon.integrations.google.auth import GoogleAuthClient + +from .config import GoogleSheetsConfig + +log = Logging(name='GoogleSheetsClient', level=Logging.DEBUG) + + +class Fields: + hyperlink: str = 'sheets/data/rowData/values/hyperlink' + + +class ValueInputOption: + USER_ENTERED: str = 'USER_ENTERED' + RAW: str = 'RAW' + + +class GoogleSheetsClient(GoogleAuthClient): + """Google Sheets client""" + + spreadsheetId: str + worksheet: str + range: str + config: GoogleSheetsConfig + + def __init__( + self, + spreadsheetId: str = None, + worksheet: str = None, + range: str = 'A:Z', + config: GoogleSheetsConfig = None, + **kwargs + ): + super().__init__() + self.config = config or GoogleSheetsConfig( + spreadsheetId=spreadsheetId, + **kwargs + ) + + self.worksheet = worksheet + self.range = range + + self.response = None + + @property + def values(self): + if self.response: + try: + return self.response['values'] + except Exception as e: + pass + + def spreadsheets(self): + """spreadsheet service""" + return self.service().spreadsheets() + + def get( + self, + spreadsheetId: str = None, + ranges: str = None, + includeGridData: bool = False, + fields: Fields or str = None, + **kwargs, + ): + try: + self.response = self.spreadsheets().get( + spreadsheetId=spreadsheetId or self.config.spreadsheetId, + ranges=ranges or self.range, + includeGridData=includeGridData, + fields=fields, + **kwargs, + ).execute() + except Exception as e: + log.error(f'{e}', enable_traceback=False) + + return self + + def get_values( + self, + spreadsheetId: str = None, + range: str = None, + **kwargs, + ): + try: + self.response = self.spreadsheets().values().get( + spreadsheetId=spreadsheetId or self.config.spreadsheetId, + range=range or f'{self.worksheet}!{self.range}', + **kwargs, + ).execute() + except Exception as e: + log.error(f'{e}', enable_traceback=False) + + return self + + def list(self): + # list(pageSize=1).execute() + return + + def update( + self, + spreadsheetId: str = None, + range: str = None, + valueInputOption: ValueInputOption = ValueInputOption.USER_ENTERED, + values: list = None, + ): + try: + + body = { + 'values': values + } + + result = self.spreadsheets().values().update( + spreadsheetId=spreadsheetId or self.config.spreadsheetId, + range=range or self.range, + valueInputOption=valueInputOption, + body=body + ).execute() + + print(f"{result.get('updatedCells')} cells updated.") + return result + except Exception as error: + print(f"An error occurred: {error}") + return error diff --git a/automon/integrations/google/sheets/config.py b/automon/integrations/google/sheets/config.py new file mode 100644 index 00000000..f4f99606 --- /dev/null +++ b/automon/integrations/google/sheets/config.py @@ -0,0 +1,27 @@ +from automon.log import Logging +from automon.helpers.osWrapper import environ +from automon.integrations.google.auth import GoogleAuthConfig + +log = Logging(name='SheetsConfig', level=Logging.DEBUG) + + +class GoogleSheetsConfig(GoogleAuthConfig): + """Google Sheets config""" + + def __init__( + self, + spreadsheetId: str = None, + ): + super().__init__() + + self.serviceName = 'sheets' + self.scopes = [ + 'https://www.googleapis.com/auth/drive', + 'https://www.googleapis.com/auth/drive.file', + 'https://www.googleapis.com/auth/drive.readonly', + 'https://www.googleapis.com/auth/spreadsheets', + 'https://www.googleapis.com/auth/spreadsheets.readonly', + ] + self.version = 'v4' + + self.spreadsheetId = spreadsheetId or environ('GOOGLE_SHEET_ID') diff --git a/automon/integrations/google/sheets/tests/__init__.py b/automon/integrations/google/sheets/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/automon/integrations/google/sheets/tests/test_google_sheets.py b/automon/integrations/google/sheets/tests/test_google_sheets.py new file mode 100644 index 00000000..126aeee6 --- /dev/null +++ b/automon/integrations/google/sheets/tests/test_google_sheets.py @@ -0,0 +1,234 @@ +import datetime +import logging + +import pandas as pd +import numpy as np +import tracemalloc + +import unittest + +import automon +from automon.integrations.google.sheets import GoogleSheetsClient +from automon.integrations.facebook import FacebookGroups +from automon import Logging + +logging.getLogger('SeleniumBrowser').setLevel(logging.CRITICAL) +logging.getLogger('FacebookGroups').setLevel(logging.CRITICAL) +logging.getLogger('ConfigChrome').setLevel(logging.ERROR) +logging.getLogger('RequestsClient').setLevel(logging.INFO) + +tracemalloc.start() + +log = Logging(level=Logging.INFO) + + +def get_facebook_info(url: str): + if not url: + return {} + + group = FacebookGroups( + url=url_cleaner(url=url) + ) + # group.start(headless=False) + group.start(headless=True) + group.get_about() + # if not group.privacy_details: + # close = group._browser.wait_for(group.xpath_popup_close) + # group._browser.action_click(close) + # about = group._browser.wait_for(group.xpath_about) + # group._browser.action_click(about) + + return group.to_dict() + + +def url_cleaner(url: str): + if url[-1] == '/': + url = url[:-1] + return url + + +class MyTestCase(unittest.TestCase): + def test_authenticate(self): + spreadsheetId = '1isrvjU0DaRijEztByQuT9u40TaCOCwdaLAXgGmKHap8' + test = GoogleSheetsClient( + spreadsheetId=spreadsheetId, + worksheet='Automated Count WIP', + ) + + if not test.authenticate(): + return + + def merge_urls(): + test.get( + ranges='AUDIT list Shelley!A:Z', + fields="sheets/data/rowData/values/hyperlink", + ) + + data = test.response['sheets'][0]['data'][0]['rowData'] + # expand nested data + links = [] + for x in data: + if x: + links.append( + x['values'][0]['hyperlink'] + ) + + df_Shelley = pd.DataFrame(data=links, columns=['url']) + + test.get() + test.get_values( + range='Automated Count WIP!A:Z' + ) + + sheet_values = test.values + sheet_columns = sheet_values[0] + sheet_data = sheet_values[1:] + + df = pd.DataFrame(data=sheet_data, columns=sheet_columns) + df = df.dropna(subset=['url']) + + # merge both lists or urls + df = pd.merge(df, df_Shelley, how='outer', on='url') + df = df.drop_duplicates(subset=['url'], keep='first') + return df + + def batch_processing(index: int, df: pd.DataFrame): + df_results = df['url'].dropna().apply( + lambda url: get_facebook_info(url=url) + ) + df_results = pd.DataFrame(df_results.tolist()) + + todays_date = datetime.datetime.now().date() + + df = df.reset_index() + df = df.drop('index', axis=1) + + # create columns + df[f'url'] = df_results['url'] + df[f'{todays_date}'] = df_results['members_count'] + df[f'title'] = df_results['title'] + df[f'content_unavailable'] = df_results['content_unavailable'] + df[f'creation_date'] = df_results['creation_date'] + df[f'creation_date_timestamp'] = df_results['creation_date_timestamp'] + df[f'history'] = df_results['history'] + df[f'members_count'] = df_results['members_count'] + df[f'posts_monthly_count'] = df_results['posts_monthly_count'] + df[f'posts_today_count'] = df_results['posts_today_count'] + df[f'privacy'] = df_results['privacy'] + df[f'visible'] = df_results['visible'] + + # set dtype to Int32 + df[f'{todays_date}'] = df[f'{todays_date}'].astype('Int32') + df[f'creation_date_timestamp'] = df[f'creation_date_timestamp'].astype('Int32') + df[f'members_count'] = df[f'members_count'].astype('Int32') + df[f'posts_monthly_count'] = df[f'posts_monthly_count'].astype('Int32') + df[f'posts_today_count'] = df[f'posts_today_count'].astype('Int32') + + # order columns + columns = [ + 'url', + 'title', + 'creation_date', + 'creation_date_timestamp', + 'history', + 'privacy', + 'visible', + 'content_unavailable', + 'posts_monthly_count', + 'posts_today_count', + 'members_count', + ] + + # add all other dates + df_columns = df.columns.tolist() + columns.extend( + [x for x in df_columns if x not in columns] + ) + + # finally add today's date + if f'{todays_date}' not in columns: + columns.append( + f'{todays_date}', + ) + + df = df.loc[:, columns] + df = df.fillna(np.nan).replace([np.nan], [None]) + + sheet_index = index + 2 + + update_columns = test.update( + range=f'Automated Count WIP!A1:Z', + values=[columns], + ) + + update = test.update( + range=f'Automated Count WIP!A{sheet_index}:Z', + values=[x for x in df.values.tolist()] + ) + + log.info( + f'{[x for x in df.values.tolist()]}' + ) + + return df + + def memory_profiler(): + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics("lineno") + + df_memory_profile = pd.DataFrame([ + dict(size_B=stat.size, count=stat.count, file=stat.traceback._frames[0][0], + file_line=stat.traceback._frames[0][1]) for stat in top_stats + ]) + df_memory_profile.sort_values(by='size_B', ascending=False) + df_memory_profile['size_KB'] = df_memory_profile['size_B'].apply( + lambda B: round(B / 1024) + ) + df_memory_profile['size_MB'] = df_memory_profile['size_KB'].apply( + lambda KB: round(KB / 1024) + ) + cols = df_memory_profile.columns.tolist() + cols.sort() + df_memory_profile = df_memory_profile.loc[:, cols] + + log.debug( + f"total memory used: {df_memory_profile['size_MB'].sum()} MB; " + f'most memory used: ' + f'{df_memory_profile.iloc[0].to_dict()}' + ) + + return df_memory_profile + + # start processing + test.get_values( + range='Automated Count WIP!A:Z' + ) + + sheet_values = test.values + sheet_columns = sheet_values[0] + sheet_data = sheet_values[1:] + + df = pd.DataFrame(data=sheet_data, columns=sheet_columns) + df = df.dropna(subset=['url']) + + BATCH_SIZE = 1 + + i = 0 + 6146 + i = 0 + while i < len(df): + df_batch = df[i:i + BATCH_SIZE] + try: + df_batch = batch_processing(index=i, df=df_batch) + df_memory = memory_profiler() + except Exception as e: + df_memory = memory_profiler() + pass + i += 1 + + pass + + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/automon/integrations/google/sheets/tests/test_google_sheets_AUDIT.py b/automon/integrations/google/sheets/tests/test_google_sheets_AUDIT.py new file mode 100644 index 00000000..4160e3c7 --- /dev/null +++ b/automon/integrations/google/sheets/tests/test_google_sheets_AUDIT.py @@ -0,0 +1,59 @@ +import datetime +import pandas as pd + +import unittest + +from automon.integrations.google.sheets import GoogleSheetsClient +from automon.integrations.facebook import FacebookGroups + + +def get_facebook_info(url: str): + group = FacebookGroups() + # group.start(headless=False) + group.start(headless=True) + group.get(url=url) + if not group.privacy_details: + close = group._browser.wait_for(group.xpath_popup_close) + group._browser.action_click(close) + about = group._browser.wait_for(group.xpath_about) + group._browser.action_click(about) + + return group.to_dict() + + +class MyTestCase(unittest.TestCase): + def test_authenticate(self): + spreadsheetId = '1isrvjU0DaRijEztByQuT9u40TaCOCwdaLAXgGmKHap8' + test = GoogleSheetsClient( + spreadsheetId=spreadsheetId, + worksheet='AUDIT list Shelley', + range='AUDIT list Shelley!A:B' + ) + + if not test.authenticate(): + return + + test.get_values( + range='AUDIT list Shelley!A:Z', + ) + test.get( + ranges='AUDIT list Shelley!A:Z', + fields="sheets/data/rowData/values/hyperlink", + ) + + data = test.response['sheets'][0]['data'][0]['rowData'] + # expand nested data + links = [] + for x in data: + if x: + links.append( + x['values'][0]['hyperlink'] + ) + + df = pd.DataFrame(links) + + pass + + +if __name__ == '__main__': + unittest.main() From 9bf0ba7707a764604bab45d551a0efa910008b55 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 12:32:23 -0400 Subject: [PATCH 50/62] 0.3.1 Change log: add scrapyWrapper add beautifulsoupWrapper add google/auth add google/sheets google: rename classes google: fix auth check google: rename auth and config facebook: add group info sleeper: set logs to debug selenium: update request and status selenium: fix $PATH too long selenium: set logs to debug selenium: add set_locale_experimental selenium: add enable_translate selenium: add set_locale selenium: better logging selenium: raise exception if missing driver selenium: update to selenium 4 selenium: raise exception when ConfigWebdriver fails selenium: update config selenium: fix wait_for not iterating through values selenium: update typing selenium: fix chromedriver path missing updated property selenium: now gets all available logs from webdriver.log_types selenium: remove and depreciate properties selenium: use Request for return code status selenium: raise exception if webdriver is not set selenium: change waiting for element to 3 retries requests: update properties --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bcc543bc..8e473511 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.3.0", + version="0.3.1", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From d88671eb71936e7ca3a8eadb2d16af15f9864c3c Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 13:20:15 -0400 Subject: [PATCH 51/62] selenium: fix tests --- automon/integrations/seleniumWrapper/config_webdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automon/integrations/seleniumWrapper/config_webdriver.py b/automon/integrations/seleniumWrapper/config_webdriver.py index b16c79c2..e80c973a 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver.py +++ b/automon/integrations/seleniumWrapper/config_webdriver.py @@ -80,7 +80,7 @@ def run(self): try: return self.webdriver_wrapper.run() except Exception as e: - log.error(f'failed to run: {e}', raise_exception=True) + log.error(f'failed to run: {e}', enable_traceback=False) def start(self): """alias to run""" From 8042ecb46ef558b33c4def5a282cea46ab8a60a1 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 13:28:20 -0400 Subject: [PATCH 52/62] 0.3.2 Change log: selenium: fix tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e473511..bfc6917f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.3.1", + version="0.3.2", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From 449ac31ce6da9d64a9d7b2ad8a41fc05f9ffba68 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 13:32:15 -0400 Subject: [PATCH 53/62] scrapy: fix for python36 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cffb2319..1bf2a99c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ selenium>=3.141.0 sentry-sdk>=1.5.1 # scrapy -Scrapy>=2.9.0 +Scrapy>=2.6.0 # slack slackclient>=2.9.3 From 1614b5e40310eae9ef2c378ecb5a03b1ede18707 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 18:23:05 -0400 Subject: [PATCH 54/62] 0.3.3 Change log: scrapy: fix for python36 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bfc6917f..a398167f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.3.2", + version="0.3.3", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From a01bae04027812423c03ca2b09481b3caf07e55c Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 19:06:54 -0400 Subject: [PATCH 55/62] sheets: fix no results when worksheet = None --- automon/integrations/google/sheets/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automon/integrations/google/sheets/client.py b/automon/integrations/google/sheets/client.py index 27087263..8269ee4d 100644 --- a/automon/integrations/google/sheets/client.py +++ b/automon/integrations/google/sheets/client.py @@ -26,7 +26,7 @@ class GoogleSheetsClient(GoogleAuthClient): def __init__( self, spreadsheetId: str = None, - worksheet: str = None, + worksheet: str = '', range: str = 'A:Z', config: GoogleSheetsConfig = None, **kwargs From ce7dd06f6e8331fa8efe836c72cc8a8e5ab1963c Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Tue, 22 Aug 2023 19:51:52 -0400 Subject: [PATCH 56/62] 0.3.4 Change log: sheets: fix no results when worksheet = None --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a398167f..f290106a 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.3.3", + version="0.3.4", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From 212b29cb7899184820a1931cd6d77e18eee7a7b2 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 23 Aug 2023 13:24:54 -0400 Subject: [PATCH 57/62] selenium: update warnings --- .../config_webdriver_chrome.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index 1aec76fa..44e319c6 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -53,7 +53,7 @@ def window_size(self): return self._window_size def disable_certificate_verification(self): - warnings.warn('Certificates are not verified', Warning) + log.warn('Certificates are not verified') self.chrome_options.add_argument('--ignore-certificate-errors') return self @@ -79,12 +79,12 @@ def disable_sandbox(self): return self def disable_shm(self): - warnings.warn('Disabled shm will use disk I/O, and will be slow', Warning) + log.warn('Disabled shm will use disk I/O, and will be slow') self.chrome_options.add_argument('--disable-dev-shm-usage') return self def enable_bigshm(self): - warnings.warn('Big shm not yet implemented', Warning) + log.warn('Big shm not yet implemented') return self def enable_defaults(self): @@ -151,8 +151,10 @@ def in_headless_sandboxed(self): """Headless Chrome with sandbox enabled """ - warnings.warn('Docker does not support sandbox option') - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + log.warn( + 'Docker does not support sandbox option. ' + 'Default shm size is 64m, which will cause chrome driver to crash.' + ) self.enable_defaults() self.enable_headless() @@ -162,7 +164,7 @@ def in_headless_sandbox_disabled(self): """Headless Chrome with sandbox disabled """ - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + log.warn('Default shm size is 64m, which will cause chrome driver to crash.') self.enable_defaults() self.enable_headless() @@ -173,7 +175,7 @@ def in_headless_sandbox_disabled_certificate_unverified(self): """Headless Chrome with sandbox disabled with no certificate verification """ - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + log.warn('Default shm size is 64m, which will cause chrome driver to crash.') self.enable_defaults() self.enable_headless() @@ -195,7 +197,7 @@ def in_headless_sandbox_disabled_bigshm(self): """Headless Chrome with sandbox disabled """ - warnings.warn('Larger shm option is not implemented', Warning) + log.warn('Larger shm option is not implemented') self.enable_defaults() self.enable_headless() @@ -219,8 +221,10 @@ def in_sandbox(self): """Chrome with sandbox enabled """ - warnings.warn('Docker does not support sandbox option') - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + log.warn( + 'Docker does not support sandbox option. ' + 'Default shm size is 64m, which will cause chrome driver to crash.' + ) self.enable_defaults() return self @@ -229,7 +233,7 @@ def in_sandbox_disabled(self): """Chrome with sandbox disabled """ - warnings.warn('Default shm size is 64m, which will cause chrome driver to crash.', Warning) + log.warn('Default shm size is 64m, which will cause chrome driver to crash.') self.enable_defaults() self.disable_sandbox() From a6f89627adb54f536b5c4c81971bdcb7b500eb44 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 23 Aug 2023 13:26:06 -0400 Subject: [PATCH 58/62] sheets: update tests --- .../google/sheets/tests/test_google_sheets.py | 331 +++++++++--------- 1 file changed, 171 insertions(+), 160 deletions(-) diff --git a/automon/integrations/google/sheets/tests/test_google_sheets.py b/automon/integrations/google/sheets/tests/test_google_sheets.py index 126aeee6..ec6b08c1 100644 --- a/automon/integrations/google/sheets/tests/test_google_sheets.py +++ b/automon/integrations/google/sheets/tests/test_google_sheets.py @@ -1,16 +1,22 @@ import datetime import logging +import automon +import tracemalloc import pandas as pd import numpy as np -import tracemalloc -import unittest - -import automon -from automon.integrations.google.sheets import GoogleSheetsClient -from automon.integrations.facebook import FacebookGroups from automon import Logging +from automon.integrations.facebook import FacebookGroups +from automon.integrations.google.sheets import GoogleSheetsClient + +logging.getLogger('google_auth_httplib2').setLevel(logging.ERROR) +logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) +logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR) +logging.getLogger('urllib3.connectionpool').setLevel(logging.ERROR) +logging.getLogger('selenium.webdriver.common.service').setLevel(logging.ERROR) +logging.getLogger('selenium.webdriver.remote.remote_connection').setLevel(logging.ERROR) +logging.getLogger('selenium.webdriver.common.selenium_manager').setLevel(logging.ERROR) logging.getLogger('SeleniumBrowser').setLevel(logging.CRITICAL) logging.getLogger('FacebookGroups').setLevel(logging.CRITICAL) @@ -21,6 +27,10 @@ log = Logging(level=Logging.INFO) +sheets_client = GoogleSheetsClient( + worksheet='Automated Count WIP', +) + def get_facebook_info(url: str): if not url: @@ -32,11 +42,6 @@ def get_facebook_info(url: str): # group.start(headless=False) group.start(headless=True) group.get_about() - # if not group.privacy_details: - # close = group._browser.wait_for(group.xpath_popup_close) - # group._browser.action_click(close) - # about = group._browser.wait_for(group.xpath_about) - # group._browser.action_click(about) return group.to_dict() @@ -47,188 +52,194 @@ def url_cleaner(url: str): return url -class MyTestCase(unittest.TestCase): - def test_authenticate(self): - spreadsheetId = '1isrvjU0DaRijEztByQuT9u40TaCOCwdaLAXgGmKHap8' - test = GoogleSheetsClient( - spreadsheetId=spreadsheetId, - worksheet='Automated Count WIP', - ) - - if not test.authenticate(): - return +def merge_urls(): + sheets_client.get( + ranges='AUDIT list Shelley!A:Z', + fields="sheets/data/rowData/values/hyperlink", + ) - def merge_urls(): - test.get( - ranges='AUDIT list Shelley!A:Z', - fields="sheets/data/rowData/values/hyperlink", + data = sheets_client.response['sheets'][0]['data'][0]['rowData'] + # expand nested data + links = [] + for x in data: + if x: + links.append( + x['values'][0]['hyperlink'] ) - data = test.response['sheets'][0]['data'][0]['rowData'] - # expand nested data - links = [] - for x in data: - if x: - links.append( - x['values'][0]['hyperlink'] - ) + df_Shelley = pd.DataFrame(data=links, columns=['url']) - df_Shelley = pd.DataFrame(data=links, columns=['url']) + sheets_client.get() + sheets_client.get_values( + range='Automated Count WIP!A:Z' + ) - test.get() - test.get_values( - range='Automated Count WIP!A:Z' - ) + sheet_values = sheets_client.values + sheet_columns = sheet_values[0] + sheet_data = sheet_values[1:] - sheet_values = test.values - sheet_columns = sheet_values[0] - sheet_data = sheet_values[1:] + df = pd.DataFrame(data=sheet_data, columns=sheet_columns) + df = df.dropna(subset=['url']) - df = pd.DataFrame(data=sheet_data, columns=sheet_columns) - df = df.dropna(subset=['url']) + # merge both lists or urls + df = pd.merge(df, df_Shelley, how='outer', on='url') + df = df.drop_duplicates(subset=['url'], keep='first') + return df - # merge both lists or urls - df = pd.merge(df, df_Shelley, how='outer', on='url') - df = df.drop_duplicates(subset=['url'], keep='first') - return df - def batch_processing(index: int, df: pd.DataFrame): - df_results = df['url'].dropna().apply( - lambda url: get_facebook_info(url=url) - ) - df_results = pd.DataFrame(df_results.tolist()) - - todays_date = datetime.datetime.now().date() - - df = df.reset_index() - df = df.drop('index', axis=1) - - # create columns - df[f'url'] = df_results['url'] - df[f'{todays_date}'] = df_results['members_count'] - df[f'title'] = df_results['title'] - df[f'content_unavailable'] = df_results['content_unavailable'] - df[f'creation_date'] = df_results['creation_date'] - df[f'creation_date_timestamp'] = df_results['creation_date_timestamp'] - df[f'history'] = df_results['history'] - df[f'members_count'] = df_results['members_count'] - df[f'posts_monthly_count'] = df_results['posts_monthly_count'] - df[f'posts_today_count'] = df_results['posts_today_count'] - df[f'privacy'] = df_results['privacy'] - df[f'visible'] = df_results['visible'] - - # set dtype to Int32 - df[f'{todays_date}'] = df[f'{todays_date}'].astype('Int32') - df[f'creation_date_timestamp'] = df[f'creation_date_timestamp'].astype('Int32') - df[f'members_count'] = df[f'members_count'].astype('Int32') - df[f'posts_monthly_count'] = df[f'posts_monthly_count'].astype('Int32') - df[f'posts_today_count'] = df[f'posts_today_count'].astype('Int32') - - # order columns - columns = [ - 'url', - 'title', - 'creation_date', - 'creation_date_timestamp', - 'history', - 'privacy', - 'visible', - 'content_unavailable', - 'posts_monthly_count', - 'posts_today_count', - 'members_count', - ] - - # add all other dates - df_columns = df.columns.tolist() - columns.extend( - [x for x in df_columns if x not in columns] - ) +def batch_processing(index: int, df: pd.DataFrame): + df_results = df['url'].dropna().apply( + lambda url: get_facebook_info(url=url) + ) + df_results = pd.DataFrame(df_results.tolist()) + + df = df.reset_index() + df = df.drop('index', axis=1) + + todays_date = datetime.datetime.now().date() + monthly = f'{todays_date.year}-{todays_date.month}' + + # create columns + df[f'url'] = df_results['url'] + df[f'{monthly}'] = df_results['members_count'] + df[f'last_updated'] = monthly + df[f'title'] = df_results['title'] + df[f'content_unavailable'] = df_results['content_unavailable'] + df[f'creation_date'] = df_results['creation_date'] + df[f'creation_date_timestamp'] = df_results['creation_date_timestamp'] + df[f'history'] = df_results['history'] + df[f'members_count'] = df_results['members_count'] + df[f'posts_monthly_count'] = df_results['posts_monthly_count'] + df[f'posts_today_count'] = df_results['posts_today_count'] + df[f'privacy'] = df_results['privacy'] + df[f'visible'] = df_results['visible'] + + # set dtype to Int32 + df[f'{monthly}'] = df[f'{monthly}'].astype('Int32') + df[f'creation_date_timestamp'] = df[f'creation_date_timestamp'].astype('Int32') + df[f'members_count'] = df[f'members_count'].astype('Int32') + df[f'posts_monthly_count'] = df[f'posts_monthly_count'].astype('Int32') + df[f'posts_today_count'] = df[f'posts_today_count'].astype('Int32') + + # order columns + columns = [ + 'url', + 'title', + 'creation_date', + 'creation_date_timestamp', + 'history', + 'privacy', + 'visible', + 'content_unavailable', + 'last_updated', + 'posts_monthly_count', + 'posts_today_count', + 'members_count', + ] + + # add all other dates + df_columns = df.columns.tolist() + columns.extend( + [x for x in df_columns if x not in columns] + ) + + # finally add today's date + if f'{monthly}' not in columns: + columns.append( + f'{monthly}', + ) - # finally add today's date - if f'{todays_date}' not in columns: - columns.append( - f'{todays_date}', - ) + df = df.loc[:, columns] + df = df.fillna(np.nan).replace([np.nan], [None]) - df = df.loc[:, columns] - df = df.fillna(np.nan).replace([np.nan], [None]) + sheet_index = index + 2 - sheet_index = index + 2 + update_columns = sheets_client.update( + range=f'Automated Count WIP!A1:Z', + values=[columns], + ) - update_columns = test.update( - range=f'Automated Count WIP!A1:Z', - values=[columns], - ) + update = sheets_client.update( + range=f'Automated Count WIP!A{sheet_index}:Z', + values=[x for x in df.values.tolist()] + ) - update = test.update( - range=f'Automated Count WIP!A{sheet_index}:Z', - values=[x for x in df.values.tolist()] - ) + log.info( + f'{[x for x in df.values.tolist()]}' + ) - log.info( - f'{[x for x in df.values.tolist()]}' - ) + return df - return df - def memory_profiler(): - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics("lineno") +def memory_profiler(): + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics("lineno") - df_memory_profile = pd.DataFrame([ - dict(size_B=stat.size, count=stat.count, file=stat.traceback._frames[0][0], - file_line=stat.traceback._frames[0][1]) for stat in top_stats - ]) - df_memory_profile.sort_values(by='size_B', ascending=False) - df_memory_profile['size_KB'] = df_memory_profile['size_B'].apply( - lambda B: round(B / 1024) - ) - df_memory_profile['size_MB'] = df_memory_profile['size_KB'].apply( - lambda KB: round(KB / 1024) - ) - cols = df_memory_profile.columns.tolist() - cols.sort() - df_memory_profile = df_memory_profile.loc[:, cols] - - log.debug( - f"total memory used: {df_memory_profile['size_MB'].sum()} MB; " - f'most memory used: ' - f'{df_memory_profile.iloc[0].to_dict()}' - ) + df_memory_profile = pd.DataFrame([ + dict(size_B=stat.size, count=stat.count, file=stat.traceback._frames[0][0], + file_line=stat.traceback._frames[0][1]) for stat in top_stats + ]) + df_memory_profile.sort_values(by='size_B', ascending=False) + df_memory_profile['size_KB'] = df_memory_profile['size_B'].apply( + lambda B: round(B / 1024) + ) + df_memory_profile['size_MB'] = df_memory_profile['size_KB'].apply( + lambda KB: round(KB / 1024) + ) + cols = df_memory_profile.columns.tolist() + cols.sort() + df_memory_profile = df_memory_profile.loc[:, cols] + + log.debug( + f"total memory used: {df_memory_profile['size_MB'].sum()} MB; " + f'most memory used: ' + f'{df_memory_profile.iloc[0].to_dict()}' + ) - return df_memory_profile + return df_memory_profile - # start processing - test.get_values( - range='Automated Count WIP!A:Z' - ) - sheet_values = test.values - sheet_columns = sheet_values[0] - sheet_data = sheet_values[1:] +def main(): + if not sheets_client.authenticate(): + return + + # start processing + sheets_client.get_values( + range='Automated Count WIP!A:Z' + ) + + sheet_values = sheets_client.values + sheet_columns = sheet_values[0] + sheet_data = sheet_values[1:] + + df = pd.DataFrame(data=sheet_data, columns=sheet_columns) + df = df.dropna(subset=['url']) + + todays_date = datetime.datetime.now().date() + last_updated = f'{todays_date.year}-{todays_date.month}' - df = pd.DataFrame(data=sheet_data, columns=sheet_columns) - df = df.dropna(subset=['url']) + BATCH_SIZE = 1 - BATCH_SIZE = 1 + i = 0 + while i < len(df): + df_batch = df[i:i + BATCH_SIZE] + + # skip if last_updated is in current month + if not df_batch['last_updated'].iloc[0] == last_updated: - i = 0 + 6146 - i = 0 - while i < len(df): - df_batch = df[i:i + BATCH_SIZE] try: df_batch = batch_processing(index=i, df=df_batch) df_memory = memory_profiler() except Exception as e: df_memory = memory_profiler() pass - i += 1 - pass + i += 1 pass + pass + if __name__ == '__main__': - unittest.main() + main() From cedcb441f8580ccb0371c0df2ff71eebd011ce55 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 11 Sep 2023 20:02:32 +0900 Subject: [PATCH 59/62] selenium: depreciate config.driver --- automon/integrations/seleniumWrapper/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automon/integrations/seleniumWrapper/config.py b/automon/integrations/seleniumWrapper/config.py index 67dd31ae..0277897f 100644 --- a/automon/integrations/seleniumWrapper/config.py +++ b/automon/integrations/seleniumWrapper/config.py @@ -16,9 +16,9 @@ def __init__(self): def __repr__(self): return f'{self.driver}' - @property - def driver(self): - return self.webdriver + # @property + # def driver(self): + # return self.webdriver @property def set_webdriver(self): From e9d7be50a7063600cb690fe01b6c9e258ba7926f Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Mon, 11 Sep 2023 20:02:57 +0900 Subject: [PATCH 60/62] selenium: fix status from request --- automon/integrations/seleniumWrapper/browser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index e7402b21..481d3b2b 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -72,7 +72,10 @@ def keys(self): @property def status(self): if self.request is not None: - return self.request.results.status_code + try: + return self.request.results.status_code + except: + pass # @property # def type(self) -> SeleniumBrowserType: From 67d6e1f218369a64ee72f044e04076854fd0c0c8 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 20 Sep 2023 17:43:42 +0900 Subject: [PATCH 61/62] 0.3.5 Change log: selenium: fix status from request selenium: depreciate config.driver selenium: update warnings sheets: update tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f290106a..f171f051 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="automonisaur", - version="0.3.4", + version="0.3.5", author="naisanza", author_email="naisanza@gmail.com", description="Core libraries for automonisaur", From 1002b6376955bd6b51d5e262528e8a6d27726ff1 Mon Sep 17 00:00:00 2001 From: naisanzaa Date: Wed, 20 Sep 2023 18:02:27 +0900 Subject: [PATCH 62/62] ci: update to docker/setup-buildx-action@v3 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b006b34..efd20006 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Build docker run: docker build . --tag ${{ env.IMAGE_NAME }} @@ -61,7 +61,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Build docker run: docker build . --tag ${{ env.IMAGE_NAME }} - name: Run tests in docker