Merge branch 'master' into add_youtube_downloader

.github/workflows/ci.yml

@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Build docker
         run: docker build . --tag ${{ env.IMAGE_NAME }}
 
@@ -61,7 +61,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Build docker
         run: docker build . --tag ${{ env.IMAGE_NAME }}
       - name: Run tests in docker

README.md

@@ -18,9 +18,9 @@
 [![master](https://github.com/TheShellLand/automonisaur/actions/workflows/python37.yml/badge.svg)](https://github.com/TheShellLand/automonisaur/actions/workflows/python37.yml)
 [![master](https://github.com/TheShellLand/automonisaur/actions/workflows/python36.yml/badge.svg)](https://github.com/TheShellLand/automonisaur/actions/workflows/python36.yml)
 
-[![Downloads](https://pepy.tech/badge/automonisaur)](https://pepy.tech/project/automonisaur)
-[![Downloads](https://pepy.tech/badge/automonisaur/month)](https://pepy.tech/project/automonisaur)
-[![Downloads](https://pepy.tech/badge/automonisaur/week)](https://pepy.tech/project/automonisaur)
+[![Downloads](https://static.pepy.tech/badge/automonisaur)](https://pepy.tech/project/automonisaur)
+[![Downloads](https://static.pepy.tech/badge/automonisaur/month)](https://pepy.tech/project/automonisaur)
+[![Downloads](https://static.pepy.tech/badge/automonisaur/week)](https://pepy.tech/project/automonisaur)
 
 [//]: # ([![codecov](https://codecov.io/gh/TheShellLand/automonisaur/branch/master/graph/badge.svg)](https://codecov.io/gh/TheShellLand/automonisaur))
 
@@ -35,15 +35,20 @@ Github issues and feature requests welcomed.
 ### Integrations
 
 - airport
+- beautifulsoup
 - elasticsearch
+- facebook groups
 - flask
+- google auth api
 - google people api
+- google sheets api
 - instagram
 - logging
 - minio
 - neo4j
 - nmap
 - requests
+- scrapy
 - selenium
 - sentryio
 - slack

automon/helpers/sleeper.py

@@ -3,7 +3,7 @@
 
 from automon.log import Logging
 
-log = Logging('Sleeper', level=Logging.INFO)
+log = Logging(name='Sleeper', level=Logging.INFO)
 
 
 class Sleeper:
@@ -14,18 +14,18 @@ def seconds(caller: object or str, seconds: int) -> time.sleep:
 
         sleep = seconds
         if sleep < 2:
-            log.info(f'[{Sleeper.seconds.__name__}] '
+            log.debug(f'[{Sleeper.seconds.__name__}] '
                      f'[{caller}] sleeping for {sleep} second')
         else:
-            log.info(f'[{Sleeper.seconds.__name__}] '
+            log.debug(f'[{Sleeper.seconds.__name__}] '
                      f'[{caller}] sleeping for {sleep} seconds')
         return time.sleep(sleep)
 
     @staticmethod
     def minute(caller: object or str, sleep: int = 60) -> time.sleep:
         """Sleep for a minute"""
 
-        log.info(f'[{Sleeper.minute.__name__}] '
+        log.debug(f'[{Sleeper.minute.__name__}] '
                  f'[{caller}] sleeping for {sleep} seconds')
         return time.sleep(sleep)
 
@@ -35,7 +35,7 @@ def within_a_minute(caller, sleep: int = None):
 
         sleep = sleep if isinstance(sleep, int) else \
             random.choice(range(1, 1 * 60))
-        log.info(f'[{Sleeper.within_a_minute.__name__}] '
+        log.debug(f'[{Sleeper.within_a_minute.__name__}] '
                  f'[{caller}] sleeping for {sleep} seconds')
         return time.sleep(sleep)
 
@@ -44,7 +44,7 @@ def minutes(caller, minutes: int):
         """Sleep for this many minutes"""
 
         sleep = minutes * 60
-        log.info(f'[{Sleeper.minutes.__name__}] '
+        log.debug(f'[{Sleeper.minutes.__name__}] '
                  f'[{caller}] sleeping for {sleep} minutes')
         return time.sleep(sleep)
 
@@ -54,7 +54,7 @@ def hour(caller, hour: int = 1):
 
         sleep = hour if not hour else random.choice(
             range(1, hour * 60 * 60))
-        log.info(f'[{Sleeper.hour.__name__}] '
+        log.debug(f'[{Sleeper.hour.__name__}] '
                  f'[{caller}] sleeping for {sleep} seconds')
         return time.sleep(sleep)
 
@@ -63,7 +63,7 @@ def hours(caller, hours):
         """Sleep for this many hours"""
 
         sleep = hours * 60 * 60
-        log.info(f'[{Sleeper.hours.__name__}] '
+        log.debug(f'[{Sleeper.hours.__name__}] '
                  f'[{caller}] sleeping for {hours} hours')
         return time.sleep(sleep)
 
@@ -73,7 +73,7 @@ def day(caller, hours: int = 24):
 
         sleep = hours if not hours else random.choice(
             range(1, hours * 60 * 60))
-        log.info(f'[{Sleeper.day.__name__}] '
+        log.debug(f'[{Sleeper.day.__name__}] '
                  f'[{caller}] sleeping for {sleep} seconds')
         return time.sleep(sleep)
 
@@ -82,7 +82,7 @@ def daily(caller, hours: int = 24):
         """Sleep for one day"""
 
         sleep = hours if not hours else hours * 60 * 60
-        log.info(f'[{Sleeper.daily.__name__}] '
+        log.debug(f'[{Sleeper.daily.__name__}] '
                  f'[{caller}] sleeping for {sleep} seconds')
         return time.sleep(sleep)
 
@@ -92,6 +92,6 @@ def time_range(caller, seconds: int):
         """
         sleep = seconds if not seconds else random.choice(
             range(1, seconds))
-        log.info(f'[{Sleeper.time_range.__name__}] '
+        log.debug(f'[{Sleeper.time_range.__name__}] '
                  f'[{caller}] sleeping for {sleep} seconds')
         return time.sleep(sleep)

automon/integrations/beautifulsoupWrapper/__init__.py

@@ -0,0 +1 @@
+from .client import BeautifulSoupClient

automon/integrations/beautifulsoupWrapper/client.py

@@ -0,0 +1,24 @@
+from bs4 import BeautifulSoup
+
+from automon.log import Logging
+
+log = Logging(name='BeautifulSoupClient', level=Logging.DEBUG)
+
+
+class BeautifulSoupClient(object):
+
+    def __init__(self, bs: BeautifulSoup = None):
+        self.bs = bs
+
+    def read_markup(self, markup: str, features: str = 'lxml'):
+        """read markup with beautifulsoup"""
+        try:
+            self.bs = BeautifulSoup(
+                markup=markup or self.markup,
+                features=features
+            )
+            log.info(f'read_markup success ({len(markup)} B)')
+        except Exception as e:
+            log.error(f'read_markup failed ({len(markup)} B): {e}')
+
+        return self

automon/integrations/facebook/__init__.py

@@ -0,0 +1 @@
+from .groups import FacebookGroups