Dev

.github/workflows/python-app.yaml

@@ -2,9 +2,6 @@ name: Python application
 
 on:
   push:
-    branches:
-      - dev
-      - main
     paths:
       - '**.py'
       - '**.yml'
@@ -20,7 +17,7 @@ jobs:
     - name: Install Chrome
       run: |
         wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-        wget https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/119.0.6045.105/linux64/chromedriver-linux64.zip
+        wget -O chromedriver-linux64.zip `curl -s https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json | jq -r '.channels.Stable.downloads.chromedriver[0].url'`
         sudo rm -rf /usr/bin/chromedriver
         sudo mkdir /usr/bin/chromedriver
         sudo unzip chromedriver-linux64.zip -d /usr/bin/chromedriver

config.yaml

@@ -1,7 +1,6 @@
 { "proxy":
     {"http": "socks5://127.0.0.1:7890", "https": "socks5://127.0.0.1:7890" }, "max_threads": 1,
-  "header": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"}
-,"user_data_dir": "/media/Data/Project/twitter_user_tweet_crawler/twitter_user_tweet_crawler/userdata",
+  "header": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"},
   "inject_js": "/media/Data/Project/twitter_user_tweet_crawler/script.js",
   "save": "/media/Data/Project/twitter_user_tweet_crawler/output",
   "user": "s_nample"

test.py

@@ -10,13 +10,12 @@ def get_tweet():
     set_work_directory(Path(__file__).absolute().parent)
     config.load({"proxy": {"http": "socks5://127.0.0.1:7890", "https": "socks5://127.0.0.1:7890"}, "max_threads": 2,
                  "header": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, "
-                                          "like Gecko) "},
-                 "user_data_dir": "/media/Data/Project/twitter_user_tweet_crawler/twitter_user_tweet_crawler/userdata"
+                                          "like Gecko) "}
                     , "inject_js": "/media/Data/Project/twitter_user_tweet_crawler/script.js",
                  "save": "/media/Data/Project/twitter_user_tweet_crawler/output/", }
                 )
     from twitter_user_tweet_crawler.tweet import Tweet
-    Path(config.save / 'res').mkdir(parents=True, exist_ok=True)
+    (Path(config.save) / 'res').mkdir(parents=True, exist_ok=True)
     browser = get_browser()
     browser.get('https://twitter.com/404')
     cookie: list[dict]

tests/CI.py

@@ -5,28 +5,25 @@
 from selenium.webdriver.chrome.webdriver import WebDriver
 from selenium.webdriver.common.by import By
 
-from twitter_user_tweet_crawler.tweet import Tweet
+from twitter_user_tweet_crawler.browser import get_browser
+from twitter_user_tweet_crawler.util.config import config
 
-
-def get_browser() -> WebDriver:
-    chrome_options = webdriver.ChromeOptions()
-    chrome_options.add_argument('--blink-settings=imagesEnabled=false')
-    chrome_options.add_argument('--disable-remote-fonts')
-    chrome_options.add_argument('--disable-gpu')
-    chrome_options.add_argument('--no-sandbox')
-    chrome_options.add_argument('--headless')
-    driver = webdriver.Chrome(options=chrome_options)
-    return driver
+config.load({"proxy": None, "max_threads": 2,
+             "header": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, "
+                                      "like Gecko) "}
+                , "inject_js": "script.js",
+             }
+            )
 
 
 class TestCase(unittest.TestCase):
     def test_spider(self):
-        browser = get_browser()
+        browser = get_browser(headless=True)
         browser.get('https://twitter.com/_CASTSTATION/status/1697029186777706544')
         sleep(20)
-        element = browser.find_element(By.XPATH,"//*[contains(text(), '{}')]".format('miku miku oo ee oo'))
+        element = browser.find_element(By.XPATH, "//*[contains(text(), '{}')]".format('miku miku oo ee oo'))
         browser.save_screenshot('debug.png')
-        self.assertIn('miku miku oo ee oo',element.get_attribute('innerHTML'))
+        self.assertIn('miku miku oo ee oo', element.get_attribute('innerHTML'))
 
 
 if __name__ == '__main__':

twitter_user_tweet_crawler/__main__.py

@@ -39,7 +39,7 @@ def get_items_need_handle():
         return driver.find_elements(*selector)
 
     selector = (By.XPATH, '//*/div[2]/div/div[3]/a[@role="link"]')
-    Path(config.save / 'res').mkdir(exist_ok=True, parents=True)
+    (Path(config.save) / 'res').mkdir(exist_ok=True, parents=True)
 
     driver = get_browser()
 

twitter_user_tweet_crawler/browser.py

@@ -14,11 +14,6 @@ def get_browser(headless: bool = False, id=None) -> WebDriver:
     chrome_options.add_argument('--disable-gpu')
     chrome_options.add_argument('--no-sandbox')
     chrome_options.add_argument('--window-size=1200x600"')
-
-    if not id:
-        chrome_options.add_argument(f'user-data-dir={config["user_data_dir"] + "/" + str(browsers := browsers + 1)}')
-    else:
-        chrome_options.add_argument(f'user-data-dir={config["user_data_dir"] + "/" + str(id)}')
     if headless:
         chrome_options.add_argument('--headless')
     driver = webdriver.Chrome(options=chrome_options)