diff --git a/automon/helpers/assertions.py b/automon/helpers/assertions.py index 1781727b..bbc82ad4 100755 --- a/automon/helpers/assertions.py +++ b/automon/helpers/assertions.py @@ -1,9 +1,11 @@ import re from ast import literal_eval -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) -log = Logging(__name__, Logging.DEBUG) def make_tuple(obj: str) -> tuple: diff --git a/automon/helpers/asyncioWrapper/loop.py b/automon/helpers/asyncioWrapper/loop.py index e223f368..4b1a2e8a 100644 --- a/automon/helpers/asyncioWrapper/loop.py +++ b/automon/helpers/asyncioWrapper/loop.py @@ -1,9 +1,9 @@ import asyncio -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) def get_event_loop(): diff --git a/automon/helpers/cpu.py b/automon/helpers/cpu.py index 925cbdac..6a6f09c9 100644 --- a/automon/helpers/cpu.py +++ b/automon/helpers/cpu.py @@ -1,16 +1,17 @@ import psutil -from automon.log import Logging +from automon import log -log = Logging('cpu', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) def cpu_usage(max_cpu_percentage=80): """Limit max cpu usage """ if psutil.cpu_percent() < max_cpu_percentage: - log.debug(f'[{cpu_usage.__name__}] {psutil.cpu_percent()}%') + logger.debug(f'[{cpu_usage.__name__}] {psutil.cpu_percent()}%') return True else: - log.debug(f'[{cpu_usage.__name__}] {psutil.cpu_percent()}%') + logger.debug(f'[{cpu_usage.__name__}] {psutil.cpu_percent()}%') return False diff --git a/automon/helpers/grok/__init__.py b/automon/helpers/grok/__init__.py index ef93f577..797d6791 100644 --- a/automon/helpers/grok/__init__.py +++ b/automon/helpers/grok/__init__.py @@ -2,12 +2,14 @@ import warnings import pandas as pd -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class Grok: def __init__(self): - self._log = Logging(name=Grok.__name__, level=Logging.DEBUG) p = 'logstash-patterns-core/patterns' l = f'{os.path.join(os.path.split(os.path.realpath(__file__))[0])}' @@ -49,7 +51,7 @@ def expanded_dict(self, patterns: list) -> pd.DataFrame: # if k not in big_dict.keys(): # big_dict[k] = v # else: - # self._log.info(f'skipping existing, {k}') + # logger.info(f'skipping existing, {k}') df = pd.DataFrame(pd.concat(patterns)) return df @@ -202,7 +204,6 @@ class GrokLegacy: 'LOGLEVEL'] = '([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)' def __init__(self): - self._log = Logging(GrokLegacy.__name__, Logging.DEBUG) self.url = 'https://raw.githubusercontent.com/logstash-plugins/logstash-patterns-core/master/patterns/grok-patterns' # self.file = f'{os.path.split(os.path.realpath(__file__))[0]}/grok-patterns.txt' @@ -223,7 +224,7 @@ def __init__(self): # else: # section.append(line) # - # self._log.debug(line) + # logger.debug(line) # build dict from file diff --git a/automon/helpers/nest_asyncioWrapper/client.py b/automon/helpers/nest_asyncioWrapper/client.py index ff0b29bd..5378a487 100644 --- a/automon/helpers/nest_asyncioWrapper/client.py +++ b/automon/helpers/nest_asyncioWrapper/client.py @@ -2,15 +2,17 @@ import logging import nest_asyncio -from automon.log import Logging +from automon import log -logging.getLogger("asyncio").setLevel(Logging.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) + +log.logging.getLogger("asyncio").setLevel(log.ERROR) class AsyncStarter: def __init__(self) -> asyncio.get_event_loop: - self._log = Logging(name=AsyncStarter.__name__, level=Logging.DEBUG) self.event_loop = asyncio.get_event_loop() self.maxqueue = 1000 diff --git a/automon/helpers/networking.py b/automon/helpers/networking.py index 02c80913..3b335133 100644 --- a/automon/helpers/networking.py +++ b/automon/helpers/networking.py @@ -1,9 +1,10 @@ import socket from urllib.parse import urlparse -from automon.log import Logging +from automon import log -log = Logging(name='Networking', level=Logging.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class Networking: @@ -23,10 +24,10 @@ def check_connection(url, timeout: int = 1): s.settimeout(timeout) s.connect((host, port)) s.close() - log.debug(f'SUCCESS {url}') + logger.debug(f'SUCCESS {url}') return True except Exception as e: - log.error(f'{url} {e}', enable_traceback=False) + logger.error(f'{url} {e}') return False @staticmethod diff --git a/automon/helpers/sleeper.py b/automon/helpers/sleeper.py index 58b39968..400d9a64 100644 --- a/automon/helpers/sleeper.py +++ b/automon/helpers/sleeper.py @@ -15,7 +15,7 @@ def seconds(seconds: int) -> time.sleep: """Sleep for this many seconds""" sleep = seconds - log.debug(f'{sleep}') + logger.debug(f'{sleep}') return time.sleep(sleep) @staticmethod diff --git a/automon/helpers/subprocessWrapper/run.py b/automon/helpers/subprocessWrapper/run.py index 82b6a3be..30316027 100644 --- a/automon/helpers/subprocessWrapper/run.py +++ b/automon/helpers/subprocessWrapper/run.py @@ -3,11 +3,11 @@ from pprint import pprint from subprocess import PIPE -from automon.log import logger +from automon import log from automon.helpers.dates import Dates -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Run: @@ -77,7 +77,7 @@ def run(self, command: str = None, if sanitize_command: command = self._command(self.command) - log.debug(f'[command] {command}') + logger.debug(f'[command] {command}') try: if inBackground: @@ -116,17 +116,17 @@ def run(self, command: str = None, self.returncode = self.call.returncode if self.stdout: - log.debug(f'[stdout] {stdout}') + logger.debug(f'[stdout] {stdout}') if self.stderr: - log.error(f'[stderr] {stderr}') + logger.error(f'[stderr] {stderr}') if self.returncode == 0: return True except Exception as e: self.stderr = f'{e}'.encode() - log.error(f'{e}') + logger.error(f'{e}') return False @@ -139,7 +139,7 @@ def _command(self, command: str) -> list: for arg in split_command: if '|' in arg: - log.warning(f'Pipes are not supported! {split_command}') + logger.warning(f'Pipes are not supported! {split_command}') return self.command diff --git a/automon/helpers/tests/test_runner.py b/automon/helpers/subprocessWrapper/tests/test_runner.py similarity index 100% rename from automon/helpers/tests/test_runner.py rename to automon/helpers/subprocessWrapper/tests/test_runner.py diff --git a/automon/integrations/beautifulsoupWrapper/client.py b/automon/integrations/beautifulsoupWrapper/client.py index a520bd54..cd330f6e 100644 --- a/automon/integrations/beautifulsoupWrapper/client.py +++ b/automon/integrations/beautifulsoupWrapper/client.py @@ -1,8 +1,9 @@ from bs4 import BeautifulSoup -from automon.log import Logging +from automon import log -log = Logging(name='BeautifulSoupClient', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class BeautifulSoupClient(object): @@ -17,8 +18,8 @@ def read_markup(self, markup: str, features: str = 'lxml'): markup=markup or self.markup, features=features ) - log.info(f'read_markup success ({len(markup)} B)') + logger.info(f'read_markup success ({len(markup)} B)') except Exception as e: - log.error(f'read_markup failed ({len(markup)} B): {e}') + logger.error(f'read_markup failed ({len(markup)} B): {e}') return self diff --git a/automon/integrations/cryptocurrency/accounting.py b/automon/integrations/cryptocurrency/accounting.py index bccd2c35..726c4fc1 100644 --- a/automon/integrations/cryptocurrency/accounting.py +++ b/automon/integrations/cryptocurrency/accounting.py @@ -1,16 +1,18 @@ import os -from automon.log import Logging +from automon import log from .robinhood import Robinhood from .coinbase import Coinbase from .other import Other +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class CryptoAccounting: def __init__(self, csvs: str = None): - self._log = Logging(name=CryptoAccounting.__name__, level=Logging.DEBUG) self.accounts = [] @@ -28,17 +30,17 @@ def __init__(self, csvs: str = None): self.auto_detect(file_path) def auto_detect(self, csv): - self._log.debug(f'supported exchanges: {self.supported_exchanges}') + logger.debug(f'supported exchanges: {self.supported_exchanges}') for exchange in self.supported_exchanges: - self._log.debug(f'reading exchange: {exchange}') + logger.debug(f'reading exchange: {exchange}') x = exchange(csv) if x.is_match: - self._log.debug(f'exchange matched: {exchange}') + logger.debug(f'exchange matched: {exchange}') if x not in self.accounts: - self._log.info(f'added {x}') + logger.info(f'added {x}') self.accounts.append(x) return True - self._log.debug(f'already added: {x}') + logger.debug(f'already added: {x}') else: o = Other(csv) if o not in self.accounts: diff --git a/automon/integrations/cryptocurrency/coinbase.py b/automon/integrations/cryptocurrency/coinbase.py index 49367b1d..efa5c1aa 100644 --- a/automon/integrations/cryptocurrency/coinbase.py +++ b/automon/integrations/cryptocurrency/coinbase.py @@ -1,10 +1,12 @@ -from automon.log import Logging +from automon import log from automon.integrations.datascience import Pandas +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class Coinbase: def __init__(self, csv): - self._log = Logging(name=Coinbase.__name__, level=Logging.DEBUG) self.csv = None self.df = None diff --git a/automon/integrations/cryptocurrency/other.py b/automon/integrations/cryptocurrency/other.py index 00783904..3a4e85bd 100644 --- a/automon/integrations/cryptocurrency/other.py +++ b/automon/integrations/cryptocurrency/other.py @@ -1,11 +1,12 @@ -from automon.log import Logging +from automon import log from automon.integrations.datascience import Pandas +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class Other: def __init__(self, csv): - self._log = Logging(name=Other.__name__, level=Logging.DEBUG) - self.csv = csv self.df = OtherCSV(csv).df @@ -14,15 +15,14 @@ def __repr__(self): def __eq__(self, other): if self.df.equals(other.df): - self._log.debug(f'same {other}') + logger.debug(f'same {other}') return True - self._log.debug(f'different {other}') + logger.debug(f'different {other}') return False class OtherCSV: def __init__(self, csv): - self._log = Logging(name=OtherCSV.__name__, level=Logging.DEBUG) self.csv = csv self.df = Pandas().read_csv(csv) @@ -33,7 +33,7 @@ def __repr__(self): def __eq__(self, other): if isinstance(other, OtherCSV): if self.df == other.df: - self._log.debug(f'same {other}') + logger.debug(f'same {other}') return True - self._log.debug(f'different {other}') + logger.debug(f'different {other}') return False diff --git a/automon/integrations/cryptocurrency/robinhood.py b/automon/integrations/cryptocurrency/robinhood.py index 558c0775..0550112d 100644 --- a/automon/integrations/cryptocurrency/robinhood.py +++ b/automon/integrations/cryptocurrency/robinhood.py @@ -1,10 +1,12 @@ -from automon.log import Logging +from automon import log from automon.integrations.datascience import Pandas +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class Robinhood: def __init__(self, csv): - self._log = Logging(name=Robinhood.__name__, level=Logging.DEBUG) self.csv = None self.df = None @@ -13,7 +15,7 @@ def __init__(self, csv): r = RobinhoodCSV(csv) if r.matches: - self._log.info(f'matched {r}') + logger.info(f'matched {r}') self.is_match = True self.csv = csv self.df = r.df @@ -39,14 +41,13 @@ def __init__(self, csv): ASSET NAME,RECEIVED DATE,COST BASIS(USD),DATE SOLD,PROCEEDS """ - self._log = Logging(name=RobinhoodCSV.__name__, level=Logging.DEBUG) self.csv = csv self.df = None self.matches = None if 'Provider: Robinhood Crypto LLC' in Pandas().read_csv(csv): - self._log.debug(f'matched {csv}') + logger.debug(f'matched {csv}') self.matches = True with open(csv) as f: @@ -60,7 +61,7 @@ def __repr__(self): def __eq__(self, other): if isinstance(other, RobinhoodCSV): if self.df == other.df: - self._log.debug(f'same {other}') + logger.debug(f'same {other}') return True - self._log.debug(f'different {other}') + logger.debug(f'different {other}') return False diff --git a/automon/integrations/datascience/pandas/dataframe.py b/automon/integrations/datascience/pandas/dataframe.py index 6bd282fd..62f064fb 100644 --- a/automon/integrations/datascience/pandas/dataframe.py +++ b/automon/integrations/datascience/pandas/dataframe.py @@ -1,10 +1,12 @@ import pandas -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) def DataFrame(*args, **kwargs) -> pandas.DataFrame: - log = Logging('DataFrame', level=Logging.ERROR) df = pandas.DataFrame(*args, **kwargs) - log.debug(df) + logger.debug(df) return df diff --git a/automon/integrations/datascience/pandas/pandas.py b/automon/integrations/datascience/pandas/pandas.py index 228162c0..dedd09e7 100644 --- a/automon/integrations/datascience/pandas/pandas.py +++ b/automon/integrations/datascience/pandas/pandas.py @@ -4,16 +4,18 @@ from io import StringIO from time import time as epoch_time -from automon.log import Logging +from automon import log from .series import Series from .dataframe import DataFrame +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class Pandas: def __init__(self): - self._log = Logging(name=Pandas.__name__, level=Logging.DEBUG) self.df = None self.csv_name = None @@ -31,14 +33,14 @@ def read_csv(self, file: str or StringIO, delimiter: str = None, **kwargs) -> pa self.csv_name = file self.df = pandas.read_csv(file, delimiter=delimiter, **kwargs) - self._log.info(f'imported {file}') + logger.info(f'imported {file}') return self.df def csv_from_string(self, csv, delimiter: str = None, **kwargs) -> pandas.read_csv: """read csv from string""" self.df = self.read_csv(StringIO(csv), delimiter=delimiter, **kwargs) - self._log.info(f'imported csv string {len(csv) / 1024 / 1024} MB') + logger.info(f'imported csv string {len(csv) / 1024 / 1024} MB') return self.df def export_csv(self, file: str = None, overwrite: bool = False, diff --git a/automon/integrations/datascience/pandas/series.py b/automon/integrations/datascience/pandas/series.py index d25b217e..5d9030c4 100644 --- a/automon/integrations/datascience/pandas/series.py +++ b/automon/integrations/datascience/pandas/series.py @@ -1,10 +1,12 @@ import pandas -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) def Series(*args, **kwargs) -> pandas.Series: - log = Logging('Series', level=Logging.ERROR) s = pandas.Series(*args, **kwargs) - log.debug(s) + logger.debug(s) return s diff --git a/automon/integrations/elasticsearch/cleanup.py b/automon/integrations/elasticsearch/cleanup.py index 6018f0b6..7c08b0af 100644 --- a/automon/integrations/elasticsearch/cleanup.py +++ b/automon/integrations/elasticsearch/cleanup.py @@ -1,7 +1,8 @@ -from automon.log import Logging +from automon import log from automon.integrations.elasticsearch.client import ElasticsearchClient -log = Logging(__name__, Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Cleanup: diff --git a/automon/integrations/elasticsearch/client.py b/automon/integrations/elasticsearch/client.py index f8244852..ade3d87f 100644 --- a/automon/integrations/elasticsearch/client.py +++ b/automon/integrations/elasticsearch/client.py @@ -5,10 +5,13 @@ from requests.auth import HTTPBasicAuth from elasticsearch import Elasticsearch -from automon.log import Logging +from automon import log from .config import ElasticsearchConfig from automon.helpers.sanitation import Sanitation +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class ElasticsearchClient(Elasticsearch): def __init__(self, host: str = None, @@ -31,8 +34,6 @@ def __init__(self, host: str = None, :param config: class ElasticsearchConfig """ - self._log = Logging(ElasticsearchClient.__name__, Logging.DEBUG) - self.config = config or ElasticsearchConfig( host=host, cloud_id=cloud_id, @@ -63,11 +64,11 @@ def _client(self): http_auth=self.config.http_auth, verify_certs=self.config.verify_certs ) - self._log.info(f'Connected to elasticsearch: {client}') + logger.info(f'Connected to elasticsearch: {client}') return client except Exception as e: - self._log.error(f'Cannot connect to elasticsearch: {self.config.ELASTICSEARCH_HOST}, {e}', enable_traceback=False) + logger.error(f'Cannot connect to elasticsearch: {self.config.ELASTICSEARCH_HOST}, {e}') return False @@ -78,7 +79,7 @@ def connected(self): if self.client.ping(): return True except Exception as e: - self._log.error(f'{e}', enable_traceback=False) + logger.error(f'{e}') return False def create_document(self, doc: dict, index: str = 'default', id: str = None): @@ -101,10 +102,10 @@ def create_document(self, doc: dict, index: str = 'default', id: str = None): r = self.results self.client.indices.refresh(index=index) self.success.append({'doc': doc, 'index': index, 'id': id, 'result': r}) - self._log.debug(f'created document: {index} {id} {doc}') + logger.debug(f'created document: {index} {id} {doc}') return True except Exception as e: - self._log.error(f'Create document failed: {e}') + logger.error(f'Create document failed: {e}') self.errors.append({'index': index, 'doc': doc, 'id': id, 'error': e}) return False @@ -115,11 +116,11 @@ def delete_index(self, index: str, **kwargs): r = self.results self.success.append({'delete index': index, 'result': r}) - self._log.debug(f'deleted index: {index}') + logger.debug(f'deleted index: {index}') return True except Exception as e: self.errors.append({'index': index, 'error': e}) - self._log.error(f'Delete index failed: {e}') + logger.error(f'Delete index failed: {e}') return False @@ -130,10 +131,10 @@ def delete_document(self, index: str, id: str = None): r = self.results self.success.append({'index': index, 'id': id, 'result': r}) - self._log.debug(f'deleted document: {index} {id}') + logger.debug(f'deleted document: {index} {id}') return True except Exception as e: - self._log.error(f'Delete document failed: {e}') + logger.error(f'Delete document failed: {e}') self.errors.append({'index': index, 'id': id, 'error': e}) return False @@ -145,19 +146,19 @@ def get_indices(self) -> bool: retrieved_indices = self.results self.indices = retrieved_indices - self._log.info(f'Retrieved {len(retrieved_indices)} indices') + logger.info(f'Retrieved {len(retrieved_indices)} indices') for i in retrieved_indices: info = retrieved_indices.get(i) date = int(info.get('settings').get('index').get('creation_date')) / 1000.0 date = datetime.fromtimestamp(date).strftime("%A, %B %d, %Y %I:%M:%S") - self._log.debug(f'Index: (created: {date})\t{i}') + logger.debug(f'Index: (created: {date})\t{i}') self.success.append({'indices': retrieved_indices}) - self._log.info(f'indices: {len(retrieved_indices)}') + logger.info(f'indices: {len(retrieved_indices)}') return True except Exception as e: - self._log.error(f'Failed to get indices: {e}') + logger.error(f'Failed to get indices: {e}') self.errors.append({'error': e}) return False @@ -168,7 +169,7 @@ def info(self) -> bool: self.results = self.client.info() return True except Exception as e: - self._log.error(f'Failed to get info:{e}') + logger.error(f'Failed to get info:{e}') self.errors.append({'error': e}) return False @@ -180,11 +181,11 @@ def ping(self): if self.connected(): try: self.client.ping() - self._log.debug(f'Ping successful') + logger.debug(f'Ping successful') return True except Exception as e: self.errors.append({'error': e}) - self._log.error(f'Ping failed: {e}') + logger.error(f'Ping failed: {e}') return False @@ -204,7 +205,7 @@ def rest(self, url: str) -> requests: return r except Exception as e: - self._log.error(f'REST request failed: {e}') + logger.error(f'REST request failed: {e}') self.errors.append({'url': url, 'error': e}) return False @@ -219,10 +220,10 @@ def search(self, search: dict = None, index: str = 'default'): r = self.results self.success.append({'search': search, 'index': index, 'result': r}) - self._log.debug(f'search :{search} {index}, result {r}') + logger.debug(f'search :{search} {index}, result {r}') return True except Exception as e: - self._log.error(f'Search failed: {e}') + logger.error(f'Search failed: {e}') self.errors.append({'search': search, 'index': index, 'error': e}) return False @@ -237,7 +238,7 @@ def search_summary(self, **kwargs): print(f'{hit.get("_source")}') self.success.append({'result': res}) - self._log.debug(f'search summary {res}') + logger.debug(f'search summary {res}') return True @@ -251,13 +252,13 @@ def search_summary(self, **kwargs): # num_indices = len(retrieved_indices) # # if not num_indices: - # self._log.debug(f'No indices found') + # logger.debug(f'No indices found') # return False # - # self._log.info(f'Search found {num_indices} indices') + # logger.info(f'Search found {num_indices} indices') # # for index in retrieved_indices: - # self._log.debug(index) + # logger.debug(index) # # # TODO: Find a way to undo index deletions # # One way could be to rename the indices and store a link to the new @@ -291,17 +292,17 @@ def search_indices(self, index_pattern): retrieved_indices = self.results num_indices = len(retrieved_indices) - self._log.info(f'Search found {num_indices} indices') + logger.info(f'Search found {num_indices} indices') self.success.append({'index pattern': index_pattern, 'result': retrieved_indices}) - self._log.debug(f'search indices: {index_pattern}') + logger.debug(f'search indices: {index_pattern}') return True except elasticsearch.exceptions.NotFoundError as e: - self._log.error( + logger.error( f"You provided the index pattern '{index_pattern}', but returned no results") self.errors.append({'index pattern': index_pattern, 'error': e}) except Exception as e: - self._log.error(f'Failed to search indices: {e}') + logger.error(f'Failed to search indices: {e}') self.errors.append({'index pattern': index_pattern, 'error': e}) return False diff --git a/automon/integrations/elasticsearch/config.py b/automon/integrations/elasticsearch/config.py index 8e86977c..f0ed55a3 100644 --- a/automon/integrations/elasticsearch/config.py +++ b/automon/integrations/elasticsearch/config.py @@ -1,9 +1,12 @@ import os import logging -from automon.log import Logging +from automon import log from automon.helpers.sanitation import Sanitation as S +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + logging.getLogger('elasticsearch').setLevel(logging.ERROR) logging.getLogger('urllib3.connectionpool').setLevel(logging.ERROR) @@ -23,8 +26,6 @@ def __init__(self, host: str = None, proxy=None): """elasticsearch config""" - self._log = Logging(ElasticsearchConfig.__name__, Logging.DEBUG) - # hosts self.ELASTICSEARCH_HOST = host or os.getenv('ELASTICSEARCH_HOSTS') self.ELASTICSEARCH_CLOUD_ID = cloud_id or os.getenv('ELASTICSEARCH_CLOUD_ID') @@ -56,7 +57,7 @@ def __repr__(self): def __eq__(self, other): if not isinstance(other, ElasticsearchConfig): - self._log.warn(f'Not implemented') + logger.warn(f'Not implemented') return NotImplemented return self.ELASTICSEARCH_HOST == other.ELASTICSEARCH_HOST diff --git a/automon/integrations/elasticsearch/jvm.py b/automon/integrations/elasticsearch/jvm.py index cf38f399..d8b580cb 100644 --- a/automon/integrations/elasticsearch/jvm.py +++ b/automon/integrations/elasticsearch/jvm.py @@ -1,14 +1,16 @@ import json -from automon.log import Logging +from automon import log from automon.integrations.elasticsearch.metrics import Cluster from automon.integrations.elasticsearch.config import ElasticsearchConfig from automon.integrations.elasticsearch.client import ElasticsearchClient +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class ElasticsearchJvmMonitor: def __init__(self, config: ElasticsearchConfig = None): - self._log = Logging(ElasticsearchJvmMonitor.__name__, Logging.DEBUG) self._config = config if isinstance(config, ElasticsearchConfig) else ElasticsearchConfig() self._client = ElasticsearchClient(config) if isinstance(config, ElasticsearchConfig) else ElasticsearchClient() @@ -23,7 +25,7 @@ def _get_all_stats(self): request_json = request.text return json.loads(request_json) except Exception as e: - self._log.error(f'Failed to get all stats: {e}') + logger.error(f'Failed to get all stats: {e}') return False return False @@ -33,7 +35,7 @@ def _get_all_jvm_metrics(self): try: return Cluster(self._get_all_stats()) except Exception as e: - self._log.error(f'Failed to get jvm metrics: {e}') + logger.error(f'Failed to get jvm metrics: {e}') return False @@ -42,13 +44,13 @@ def read_file(self, file): with open(file, 'rb') as stats: return json.load(stats) except Exception as e: - self._log.error(f'Failed to read file: {e}') + logger.error(f'Failed to read file: {e}') def get_metrics(self): if self._client.connected(): try: return self._get_all_jvm_metrics() except Exception as e: - self._log.error(f'Failed to get metrics: {e}') + logger.error(f'Failed to get metrics: {e}') return False diff --git a/automon/integrations/elasticsearch/snapshots.py b/automon/integrations/elasticsearch/snapshots.py index 79f6e5a2..0acfb7d9 100644 --- a/automon/integrations/elasticsearch/snapshots.py +++ b/automon/integrations/elasticsearch/snapshots.py @@ -1,15 +1,16 @@ import json # import requests -from automon.log import Logging +from automon import log from automon.integrations.elasticsearch.client import ElasticsearchClient from automon.integrations.elasticsearch.config import ElasticsearchConfig +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class Snapshot: def __init__(self, snapshot: dict): - self._log = Logging(Snapshot.__name__, Logging.DEBUG) - self._snapshot = snapshot self.id = snapshot.get('id') self.status = snapshot.get('status') @@ -25,7 +26,7 @@ def __init__(self, snapshot: dict): def __eq__(self, other): if not isinstance(other, Snapshot): - self._log.warning(f'{other} != Snapshot') + logger.warning(f'{other} != Snapshot') return NotImplemented return self._snapshot == other._snapshot @@ -34,7 +35,6 @@ def __eq__(self, other): class ElasticsearchSnapshotMonitor: def __init__(self, elasticsearch_repository: str = 'found-snapshots', snapshots_prefix: str = '', config: ElasticsearchConfig = ElasticsearchConfig()): - self._log = Logging(ElasticsearchSnapshotMonitor.__name__, Logging.DEBUG) self._config = config if config == ElasticsearchConfig else ElasticsearchConfig() self._client = ElasticsearchClient(config=self._config) @@ -56,7 +56,7 @@ def _get_all_snapshots(self) -> bool: url = f'{endpoint}/_cat/snapshots/{self.repository}?format=json' # url = f'{endpoint}/_snapshot/{self.repository}?format=json' - self._log.info('Downloading snapshots list') + logger.info('Downloading snapshots list') request = self._client.rest(url) content = request.text @@ -67,12 +67,12 @@ def _get_all_snapshots(self) -> bool: return False def _process_snapshots(self, snapshots: dict) -> bool: - self._log.info('Processing snapshots') + logger.info('Processing snapshots') try: self.total_snapshots = list(snapshots).__len__() - self._log.info(f'{self.total_snapshots} snapshots') + logger.info(f'{self.total_snapshots} snapshots') for snapshot in snapshots: @@ -93,12 +93,12 @@ def _process_snapshots(self, snapshots: dict) -> bool: return True except Exception as e: - self._log.error(f'Unable to get snapshots: {e}') + logger.error(f'Unable to get snapshots: {e}') self.error = SnapshotError(snapshots) return False def read_file(self, file_path): - self._log.info('Reading snapshots from file') + logger.info('Reading snapshots from file') with open(file_path, 'rb') as snapshots: snapshots = json.load(snapshots) @@ -106,7 +106,7 @@ def read_file(self, file_path): self._process_snapshots(snapshots) def check_snapshots(self): - self._log.info('Checking snapshots') + logger.info('Checking snapshots') return self._get_all_snapshots() @@ -154,7 +154,6 @@ def check_snapshots(self): class SnapshotError: def __init__(self, error: dict): - self._log = Logging(SnapshotError.__name__, Logging.DEBUG) self.error = error.get('error') @@ -172,5 +171,5 @@ def __init__(self, error: dict): def __eq__(self, other): if isinstance(other, SnapshotError): return self.error == other.error - self._log.warning(NotImplemented) + logger.warning(NotImplemented) return NotImplemented diff --git a/automon/integrations/facebook/groups.py b/automon/integrations/facebook/groups.py index 3a1a1f08..3f4bbd74 100644 --- a/automon/integrations/facebook/groups.py +++ b/automon/integrations/facebook/groups.py @@ -2,13 +2,13 @@ import datetime import statistics -from automon.log import logger +from automon import log from automon.helpers.sleeper import Sleeper from automon.integrations.seleniumWrapper import SeleniumBrowser from automon.integrations.seleniumWrapper.config_webdriver_chrome import ChromeWrapper -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class FacebookGroups(object): @@ -87,11 +87,11 @@ def content_unavailable(self): try: xpath_content_unavailble = self._browser.wait_for_xpath(self._xpath_content_unavailble) content_unavailable = self._browser.find_xpath(xpath_content_unavailble).text - log.debug(content_unavailable) + logger.debug(content_unavailable) return content_unavailable except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -104,11 +104,11 @@ def creation_date(self): try: xpath_creation_date = self._browser.wait_for_xpath(self._xpath_creation_date) creation_date = self._browser.find_xpath(xpath_creation_date).text - log.debug(creation_date) + logger.debug(creation_date) return creation_date except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -123,18 +123,18 @@ def creation_date_timestamp(self): def current_rate_too_fast(self): if self.average_rate() == 0 or len(self._rate_counter) < 2: - log.info(False) + logger.info(False) return False if self.average_rate() < self.rate_per_minute(): - log.info(True) + logger.info(True) return True return False def rate_per_minute(self) -> int: rate = int(60 / self._rate_per_minute) - log.info(str(dict( + logger.info(str(dict( seconds=rate, ))) return rate @@ -142,7 +142,7 @@ def rate_per_minute(self) -> int: def average_rate(self): if self._rate_counter: rate = int(statistics.mean(self._rate_counter)) - log.info(str(dict( + logger.info(str(dict( seconds=rate, ))) return rate @@ -153,11 +153,11 @@ def history(self): try: xpath_history = self._browser.wait_for_xpath(self._xpath_history) history = self._browser.find_xpath(xpath_history).text - log.debug(history) + logger.debug(history) return history except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -173,11 +173,11 @@ def temporarily_blocked(self): temporarily_blocked = self._browser.find_xpath( xpath_temporarily_blocked ).text - log.debug(temporarily_blocked) + logger.debug(temporarily_blocked) return temporarily_blocked except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -190,12 +190,12 @@ def members(self): try: xpath_members = self._browser.wait_for_xpath(self._xpath_members) members = self._browser.find_xpath(xpath_members).text - log.debug(members) + logger.debug(members) return members # TODO: need to clean up string from members and remove bad chars except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -211,7 +211,7 @@ def members_count(self): if count: members_count = int(''.join(count)) if count else 0 - log.debug(members_count) + logger.debug(members_count) return members_count def must_login(self): @@ -222,11 +222,11 @@ def must_login(self): must_login = self._browser.find_xpath( xpath_must_login ).text - log.debug(must_login) + logger.debug(must_login) return must_login except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -239,11 +239,11 @@ def posts_monthly(self): try: xpath_monthly_posts = self._browser.wait_for_xpath(self._xpath_posts_monthly) posts_monthly = self._browser.find_xpath(xpath_monthly_posts).text - log.debug(posts_monthly) + logger.debug(posts_monthly) return posts_monthly except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -259,7 +259,7 @@ def posts_monthly_count(self): if count: posts_monthly_count = int(''.join(count)) if count else 0 - log.debug(posts_monthly_count) + logger.debug(posts_monthly_count) return posts_monthly_count def posts_today(self): @@ -267,11 +267,11 @@ def posts_today(self): try: xpath_posts_today = self._browser.wait_for_xpath(self._xpath_posts_today) posts_today = self._browser.find_xpath(xpath_posts_today).text - log.debug(posts_today) + logger.debug(posts_today) return posts_today except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -287,7 +287,7 @@ def posts_today_count(self): if count: posts_today_count = int(''.join(count)) if count else 0 - log.debug(posts_today_count) + logger.debug(posts_today_count) return posts_today_count def privacy(self): @@ -295,11 +295,11 @@ def privacy(self): try: xpath_privacy = self._browser.wait_for_xpath(self._xpath_privacy) privacy = self._browser.find_xpath(xpath_privacy).text - log.debug(privacy) + logger.debug(privacy) return privacy except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -312,11 +312,11 @@ def privacy_details(self): try: xpath_privacy_details = self._browser.wait_for_xpath(self._xpath_privacy_details) privacy_details = self._browser.find_xpath(xpath_privacy_details).text - log.debug(privacy_details) + logger.debug(privacy_details) return privacy_details except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -329,11 +329,11 @@ def title(self) -> str: try: xpath_title = self._browser.wait_for_xpath(self._xpath_title) title = self._browser.find_xpath(xpath_title).text - log.debug(title) + logger.debug(title) return title except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -359,11 +359,11 @@ def visible(self) -> str: try: xpath_visible = self._browser.wait_for_xpath(self._xpath_visible) visible = self._browser.find_xpath(xpath_visible).text - log.debug(visible) + logger.debug(visible) return visible except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, message=message, session=session, @@ -395,7 +395,7 @@ def get(self, url: str) -> bool: start = datetime.datetime.now().timestamp() result = self._browser.get(url=url) - log.info(str(dict( + logger.info(str(dict( url=url, result=result, ))) @@ -404,7 +404,7 @@ def get(self, url: str) -> bool: end = datetime.datetime.now().timestamp() seconds_elapsed = int(end - start) - log.info(str(dict( + logger.info(str(dict( seconds_elapsed=seconds_elapsed, result=result, ))) @@ -421,7 +421,7 @@ def get_about(self, rate_limiting: bool = True): else: result = self.get(url=url) - log.info(str(dict( + logger.info(str(dict( url=url, result=result, ))) @@ -448,7 +448,7 @@ def get_with_rate_limiter( self._rate_counter.append(self._wait_between_retries) Sleeper.seconds(seconds=self._wait_between_retries) - log.error(str(dict( + logger.error(str(dict( url=url, retry=retry, retries=retries, @@ -459,12 +459,12 @@ def get_with_rate_limiter( result = self.get(url=url) self.screenshot() - log.info(f'{result}') + logger.info(f'{result}') return result retry = retry + 1 - log.error(f'{url}') + logger.error(f'{url}') self.screenshot_error() return result @@ -475,7 +475,7 @@ def rate_limit_decrease(self, multiplier: int = 0.75): if self._wait_between_retries == 0: self._wait_between_retries = 1 - log.info(str(dict( + logger.info(str(dict( before=before, after=self._wait_between_retries, multiplier=multiplier, @@ -486,7 +486,7 @@ def rate_limit_increase(self, multiplier: int = 2): before = self._wait_between_retries self._wait_between_retries = abs(int(self._wait_between_retries * multiplier)) - log.info(str(dict( + logger.info(str(dict( before=before, after=self._wait_between_retries, multiplier=multiplier, @@ -496,52 +496,52 @@ def rate_limit_increase(self, multiplier: int = 2): def rate_limited(self): """rate limit checker""" if self.current_rate_too_fast(): - log.info(True) + logger.info(True) self.screenshot() return True if self.temporarily_blocked() or self.must_login(): - log.info(True) + logger.info(True) self.screenshot() return True - log.error(False) + logger.error(False) self.screenshot_error() return False def run(self): """run selenium browser""" if self._browser: - log.info(f'{self._browser}') + logger.info(f'{self._browser}') return self._browser.run() def reset_rate_counter(self): self._rate_counter = [] - log.info(self._rate_counter) + logger.info(self._rate_counter) return self._rate_counter def restart(self): """quit and start new instance of selenium""" if self._browser: self.quit() - log.info(f'{self._browser}') + logger.info(f'{self._browser}') return self.start() def screenshot(self, filename: str = 'screenshot.png'): screenshot = self._browser.save_screenshot(filename=filename, folder='.') - log.info(f'{screenshot}') + logger.info(f'{screenshot}') return screenshot def screenshot_error(self): """get error screenshot""" screenshot = self.screenshot(filename='screenshot-error.png') - log.debug(f'{screenshot}') + logger.debug(f'{screenshot}') return screenshot def screenshot_success(self): """get success screenshot""" screenshot = self.screenshot(filename='screenshot-success.png') - log.debug(f'{screenshot}') + logger.debug(f'{screenshot}') return screenshot def set_url(self, url: str) -> str: @@ -567,7 +567,7 @@ def start(self, headless: bool = True, random_user_agent: bool = False, set_user set_user_agent ) - log.info(str(dict( + logger.info(str(dict( browser=self._browser ))) browser = self._browser.run() @@ -600,5 +600,5 @@ def to_dict(self): def quit(self): """quit selenium""" if self._browser: - log.info(f'{self._browser}') + logger.info(f'{self._browser}') return self._browser.quit() diff --git a/automon/integrations/flaskWrapper/boilerplate.py b/automon/integrations/flaskWrapper/boilerplate.py index ee405c88..13eea899 100644 --- a/automon/integrations/flaskWrapper/boilerplate.py +++ b/automon/integrations/flaskWrapper/boilerplate.py @@ -1,15 +1,17 @@ import flask from flask import Flask -from automon.log import Logging +from automon import log from automon.integrations.flaskWrapper.config import FlaskConfig +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class FlaskBoilerplate(object): def __init__(self, import_name: str = __name__, config: FlaskConfig = None, **kwargs): """Wrapper for flask""" - self._log = Logging(FlaskBoilerplate.__name__, Logging.DEBUG) self.Flask = Flask(import_name=import_name, **kwargs) self.config = config or FlaskConfig() diff --git a/automon/integrations/google/auth/client.py b/automon/integrations/google/auth/client.py index a479679b..190219dd 100644 --- a/automon/integrations/google/auth/client.py +++ b/automon/integrations/google/auth/client.py @@ -3,11 +3,12 @@ import googleapiclient.discovery import google.auth.transport.requests -from automon.log import Logging +from automon import log from .config import GoogleAuthConfig -log = Logging(name='GoogleAuthClient', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GoogleAuthClient(object): @@ -68,14 +69,14 @@ def authenticate_oauth(self) -> bool: if refresh_token: try: creds.refresh(google.auth.transport.requests.Request()) - log.info(f'token refresh success') + logger.info(f'token refresh success') return True except Exception as e: - log.error(msg=f'token refresh failed: {e}', enable_traceback=False) + logger.error(msg=f'token refresh failed: {e}') else: # TODO: add google flow() authentication here - log.info(f'flow login success') + logger.info(f'flow login success') return True return False diff --git a/automon/integrations/google/auth/config.py b/automon/integrations/google/auth/config.py index c160473f..18cc16cd 100644 --- a/automon/integrations/google/auth/config.py +++ b/automon/integrations/google/auth/config.py @@ -9,10 +9,11 @@ from google.auth.transport.requests import Request from google_auth_oauthlib.flow import InstalledAppFlow -from automon.log import Logging +from automon import log from automon.helpers import environ -log = Logging(name='GoogleAuthConfig', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GoogleAuthConfig(object): @@ -58,7 +59,7 @@ def Credentials(self): except: pass - log.error(f'Missing GOOGLE_CREDENTIALS or GOOGLE_CREDENTIALS_BASE64', enable_traceback=False) + logger.error(f'Missing GOOGLE_CREDENTIALS or GOOGLE_CREDENTIALS_BASE64') @property def _GOOGLE_CREDENTIALS(self): diff --git a/automon/integrations/google/gmail/v1/config.py b/automon/integrations/google/gmail/v1/config.py index fecff96d..766c9e0c 100644 --- a/automon/integrations/google/gmail/v1/config.py +++ b/automon/integrations/google/gmail/v1/config.py @@ -1,8 +1,9 @@ from os import getenv -from automon.log import Logging +from automon import log -log = Logging(name='GoogleGmailConfig', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GoogleGmailConfig: diff --git a/automon/integrations/google/people/client.py b/automon/integrations/google/people/client.py index a5e9f6fd..a046c2b5 100644 --- a/automon/integrations/google/people/client.py +++ b/automon/integrations/google/people/client.py @@ -7,13 +7,14 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from automon.log import Logging +from automon import log from .urls import GooglePeopleUrls from .config import GooglePeopleConfig from .results import ConnectionsResults -log = Logging(name='GooglePeopleClient', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GooglePeopleClient: @@ -78,7 +79,7 @@ def authenticate(self) -> bool: creds.refresh(Request()) return True except Exception as e: - log.error(msg=f'authentication failed {e}', raise_exception=False) + logger.error(msg=f'authentication failed {e}', raise_exception=False) else: flow = InstalledAppFlow.from_client_config( diff --git a/automon/integrations/google/people/config.py b/automon/integrations/google/people/config.py index 8cdc408e..d1d6e88d 100644 --- a/automon/integrations/google/people/config.py +++ b/automon/integrations/google/people/config.py @@ -10,10 +10,11 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from automon.log import Logging +from automon import log from automon.helpers import environ -log = Logging(name='GooglePeopleConfig', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GooglePeopleConfig: @@ -191,7 +192,7 @@ def oauth_dict(self) -> dict: ) } - log.warning(f'Missing client_type') + logger.warning(f'Missing client_type') return False def from_authorized_user_file(self, file: str) -> Credentials: @@ -204,7 +205,7 @@ def isReady(self): if self.oauth_dict(): return True - log.warning(f'config is not ready') + logger.warning(f'config is not ready') return False def load_oauth(self, oauth: dict) -> Credentials: @@ -219,7 +220,7 @@ def load_oauth(self, oauth: dict) -> Credentials: return self.update(oauth) else: - log.error(msg=f'Unsupported or not an Oauth token. {oauth}', raise_exception=True) + logger.error(msg=f'Unsupported or not an Oauth token. {oauth}', raise_exception=True) return self.Credentials diff --git a/automon/integrations/google/people/person.py b/automon/integrations/google/people/person.py index 5b9ed17a..c64724e5 100644 --- a/automon/integrations/google/people/person.py +++ b/automon/integrations/google/people/person.py @@ -1,13 +1,14 @@ from enum import Enum -from automon.log import Logging +from automon import log -log = Logging(name='GooglePeople', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class AgeRange(Enum): """Please use person.ageRanges instead""" - log.warning(DeprecationWarning) + logger.warning(DeprecationWarning) AGE_RANGE_UNSPECIFIED = 'AGE_RANGE_UNSPECIFIED' LESS_THAN_EIGHTEEN = 'LESS_THAN_EIGHTEEN' @@ -393,7 +394,7 @@ class Relation(object): class RelationshipInterest(object): - log.warning(DeprecationWarning) + logger.warning(DeprecationWarning) metadata: { FieldMetadata @@ -403,7 +404,7 @@ class RelationshipInterest(object): class RelationshipStatus(object): - log.warning(DeprecationWarning) + logger.warning(DeprecationWarning) metadata: { FieldMetadata @@ -413,7 +414,7 @@ class RelationshipStatus(object): class Residence(object): - log.warning(DeprecationWarning) + logger.warning(DeprecationWarning) metadata: { FieldMetadata @@ -439,7 +440,7 @@ class Skill(object): class Tagline(object): - log.warning(DeprecationWarning) + logger.warning(DeprecationWarning) metadata: { FieldMetadata diff --git a/automon/integrations/google/people/results.py b/automon/integrations/google/people/results.py index 2f573fe8..61454b2b 100644 --- a/automon/integrations/google/people/results.py +++ b/automon/integrations/google/people/results.py @@ -1,8 +1,9 @@ -from automon.log import Logging +from automon import log from .person import Person -log = Logging(name='GooglePeopleResults', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class ConnectionsResults: @@ -36,7 +37,7 @@ def __init__(self, result: dict): self.__dict__.update(result) - log.debug(msg=f'{self}') + logger.debug(msg=f'{self}') def __repr__(self): return f'totalPeople: {self.totalPeople}, totalItems: {self.totalItems}, contacts: {len(self.contacts)}' diff --git a/automon/integrations/google/people/urls.py b/automon/integrations/google/people/urls.py index dff13bb9..eaaea99c 100644 --- a/automon/integrations/google/people/urls.py +++ b/automon/integrations/google/people/urls.py @@ -1,6 +1,7 @@ -from automon.log import Logging +from automon import log -log = Logging(name='GooglePeopleUrls', level=Logging.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GooglePeopleUrls: @@ -52,5 +53,5 @@ def personFields_toStr(self): return ','.join(self.personFields) def resourceName(self) -> str: - log.warning(msg=f'resourceName is deprecieated. Only people/me is valid.') + logger.warning(msg=f'resourceName is deprecieated. Only people/me is valid.') return f'{self.RESOURCE_NAME}/me' diff --git a/automon/integrations/google/sheets/client.py b/automon/integrations/google/sheets/client.py index a65860b7..32f1e9ee 100644 --- a/automon/integrations/google/sheets/client.py +++ b/automon/integrations/google/sheets/client.py @@ -1,10 +1,10 @@ -from automon.log import logger +from automon import log from automon.integrations.google.auth import GoogleAuthClient from .config import GoogleSheetsConfig -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Fields: @@ -67,11 +67,11 @@ def clear( **kwargs, ).execute() - log.info(f'{result}') - log.info(f"{result.get('clearedRange')} cells cleared.") + logger.info(f'{result}') + logger.info(f"{result.get('clearedRange')} cells cleared.") return result except Exception as error: - log.error(f"An error occurred: {error}") + logger.error(f"An error occurred: {error}") return error def spreadsheets(self): @@ -95,9 +95,9 @@ def get( fields=fields, **kwargs, ).execute() - log.info(f'{self.worksheet}!{self.range} ({self.config.spreadsheetId})') + logger.info(f'{self.worksheet}!{self.range} ({self.config.spreadsheetId})') except Exception as e: - log.error(f'{e}') + logger.error(f'{e}') return self @@ -115,19 +115,19 @@ def get_values( **kwargs, ).execute() - log.info(str(dict( + logger.info(str(dict( worksheet=self.worksheet, range=self.range, spreadsheetId=self.config.spreadsheetId, ))) except Exception as e: - log.error(f'{e}') + logger.error(f'{e}') return self def list(self): # list(pageSize=1).execute() - log.warning(f'{NotImplemented}') + logger.warning(f'{NotImplemented}') return def update( @@ -144,7 +144,7 @@ def update( 'values': values } - log.debug(f'{body}') + logger.debug(f'{body}') result = self.spreadsheets().values().update( spreadsheetId=spreadsheetId or self.config.spreadsheetId, @@ -153,9 +153,9 @@ def update( body=body ).execute() - log.info(f'{result}') - log.info(f"{result.get('updatedCells')} cells updated.") + logger.info(f'{result}') + logger.info(f"{result.get('updatedCells')} cells updated.") return result except Exception as error: - log.error(f"An error occurred: {error}") + logger.error(f"An error occurred: {error}") return error diff --git a/automon/integrations/google/sheets/config.py b/automon/integrations/google/sheets/config.py index 5a922283..ae189216 100644 --- a/automon/integrations/google/sheets/config.py +++ b/automon/integrations/google/sheets/config.py @@ -1,9 +1,9 @@ -from automon.log import logger +from automon import log from automon.helpers.osWrapper import environ from automon.integrations.google.auth import GoogleAuthConfig -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GoogleSheetsConfig(GoogleAuthConfig): @@ -27,4 +27,4 @@ def __init__( self.spreadsheetId = spreadsheetId or environ('GOOGLE_SHEET_ID') - log.info(f'{self}') + logger.info(f'{self}') diff --git a/automon/integrations/google/sheets/tests/test_google_sheets.py b/automon/integrations/google/sheets/tests/test_google_sheets.py index 4363cf8b..bbd5a248 100644 --- a/automon/integrations/google/sheets/tests/test_google_sheets.py +++ b/automon/integrations/google/sheets/tests/test_google_sheets.py @@ -6,7 +6,7 @@ import pandas as pd import numpy as np -from automon.log import logger +from automon import log from automon.helpers.sleeper import Sleeper from automon.integrations.facebook import FacebookGroups from automon.integrations.google.sheets import GoogleSheetsClient @@ -26,8 +26,8 @@ tracemalloc.start() -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) SHEET_NAME = 'Automated Count DO NOT EDIT' SHEET_NAME_INGEST = 'URLS TO INGEST' @@ -172,7 +172,7 @@ def batch_processing(sheet_index: int, df: pd.DataFrame): values=[x for x in df.values.tolist()] ) - log.info(f'{sheet_index_df}: {[x for x in df.values.tolist()]}') + logger.info(f'{sheet_index_df}: {[x for x in df.values.tolist()]}') return df @@ -196,7 +196,7 @@ def memory_profiler(): cols.sort() df_memory_profile = df_memory_profile.loc[:, cols] - log.debug( + logger.debug( f"total memory used: {df_memory_profile['size_MB'].sum()} MB; " f'most memory used: ' f'{df_memory_profile.iloc[0].to_dict()}' @@ -247,7 +247,7 @@ def main(): result = sheets_client.clear(range=range) # max 60/min Sleeper.seconds(seconds=1) - log.info(result) + logger.info(result) df = df.drop(duplicate_index) # ingest urls from SHEET_NAME_INGEST @@ -279,14 +279,14 @@ def main(): values=values ) - log.info( + logger.info( f'{index_add_url}: {values}' ) # clear url from ingest sheet range = f'{SHEET_NAME_INGEST}!{ingest_index}:{ingest_index}' clear = sheets_client.clear(range=range) - log.info(f'{clear}') + logger.info(f'{clear}') # start updating for data in df.iterrows(): @@ -298,7 +298,7 @@ def main(): todays_date = datetime.datetime.now().date() last_updated = f'{todays_date.year}-{todays_date.month}' if df_batch['last_updated'].iloc[0] == last_updated: - # log.debug(f'skipping {data_index}, {data_row.to_dict()}') + # logger.debug(f'skipping {data_index}, {data_row.to_dict()}') continue batch_result = batch_processing(sheet_index=data_index, df=df_batch) @@ -308,7 +308,7 @@ def main(): df_memory = memory_profiler() except Exception as e: df_memory = memory_profiler() - log.error(f'{e}') + logger.error(f'{e}') pass diff --git a/automon/integrations/instagram/client.py b/automon/integrations/instagram/client.py index 1155860e..c1028076 100644 --- a/automon/integrations/instagram/client.py +++ b/automon/integrations/instagram/client.py @@ -1,9 +1,9 @@ -from automon.log import logger +from automon import log from .config import InstagramConfig -log = logger.logging.getLogger(__name__) -log.setLevel(logger.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class InstagramClient(object): diff --git a/automon/integrations/instagram/client_browser.py b/automon/integrations/instagram/client_browser.py index d8d9e822..c58e895b 100644 --- a/automon/integrations/instagram/client_browser.py +++ b/automon/integrations/instagram/client_browser.py @@ -1,6 +1,6 @@ import functools -from automon.log import logger +from automon import log from automon.integrations.seleniumWrapper.browser import SeleniumBrowser from automon.integrations.seleniumWrapper.config_webdriver_chrome import ChromeWrapper @@ -11,8 +11,8 @@ from .urls import Urls from .xpaths import XPaths -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class InstagramBrowserClient: @@ -62,7 +62,7 @@ def wrapped(self, *args, **kwargs): def get_page(self, account: str): """ Get page """ - log.debug(f'[get_page] getting {account}') + logger.debug(f'[get_page] getting {account}') page = f'https://instagram.com/{account}' browser = self.authenticated_browser @@ -74,7 +74,7 @@ def get_stories(self, account: str): story = f'https://www.instagram.com/stories/{account}/' num_of_stories = 0 - log.debug(f'[get_stories] {story}') + logger.debug(f'[get_stories] {story}') browser = self.authenticated_browser browser.get(story) @@ -82,7 +82,7 @@ def get_stories(self, account: str): prefix='instagram/' + account) if 'Page Not Found' in browser.browser.title: - log.debug(f'[get_stories] no stories for {account}') + logger.debug(f'[get_stories] no stories for {account}') return num_of_stories Sleeper.seconds(2) @@ -93,7 +93,7 @@ def get_stories(self, account: str): title = browser.browser.title if title == 'Instagram': - log.debug( + logger.debug( ('[get_stories] {} end of stories'.format(account))) raise Exception num_of_stories += 1 @@ -104,7 +104,7 @@ def get_stories(self, account: str): prefix='instagram/' + account) except Exception as error: # TODO: disable browser proxy when done - log.debug(f'[get_stories] done: {account}, {error}') + logger.debug(f'[get_stories] done: {account}, {error}') return num_of_stories def _next_story(self, authenticated_browser): @@ -122,14 +122,14 @@ def _next_story(self, authenticated_browser): browser = authenticated_browser button = browser.browser.find_element_by_xpath(xpath) found_btn = True - log.debug('[next_story] next story') + logger.debug('[next_story] next story') return button.click() except Exception as error: - log.error(f'{error}') + logger.error(f'{error}') if not found_btn: # no more stories. exit - log.debug('[_next_story] no more stories') + logger.debug('[_next_story] no more stories') raise Exception def remove_not_now(self): @@ -157,7 +157,7 @@ def run_stories(self, limit=None): """Run """ - log.debug('[login] {}'.format(self.login)) + logger.debug('[login] {}'.format(self.login)) self.authenticated_browser = self.authenticate() @@ -167,7 +167,7 @@ def run_stories(self, limit=None): # if limit: # # for account in self.following: - # hevlog.logging.debug( + # hevlogger.debug( # '[runrun] [{}] {} session: {}'.format(self.authenticated_browser.browser.name, # self.authenticated_browser.browser.title, # self.authenticated_browser.browser.session_id)) @@ -205,10 +205,10 @@ def authenticate(self): self.remove_not_now() if self.is_authenticated(): - log.info(f'{True}') + logger.info(f'{True}') return True - log.error(f'{False}') + logger.error(f'{False}') return False @_is_running @@ -226,18 +226,18 @@ def is_authenticated(self): self.remove_not_now() profile_picture = self.browser.wait_for_xpath(self.xpaths.profile_picture) if profile_picture: - log.info(f'{True}') + logger.info(f'{True}') return True except Exception as error: - log.error(f'{error}') + logger.error(f'{error}') return False def is_running(self) -> bool: if self.config.is_configured: if self.browser.is_running(): - log.info(f'{True}') + logger.info(f'{True}') return True - log.error(f'{False}') + logger.error(f'{False}') return False @property diff --git a/automon/integrations/instagram/config.py b/automon/integrations/instagram/config.py index 704fdb35..6889576c 100644 --- a/automon/integrations/instagram/config.py +++ b/automon/integrations/instagram/config.py @@ -1,9 +1,9 @@ -from automon.log import logger +from automon import log from automon.helpers.osWrapper.environ import environ -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class InstagramConfig(object): @@ -21,19 +21,19 @@ def __repr__(self): @property def login(self): if not self._login: - log.error(f'INSTAGRAM_LOGIN') + logger.error(f'INSTAGRAM_LOGIN') return self._login @property def password(self): if not self._password: - log.error(f'INSTAGRAM_PASSWORD') + logger.error(f'INSTAGRAM_PASSWORD') return self._password @property def is_configured(self): if self.login and self.password: - log.info(f'{True}') + logger.info(f'{True}') return True - log.error(f'{False}') + logger.error(f'{False}') return False diff --git a/automon/integrations/instagram/stories.py b/automon/integrations/instagram/stories.py index 06f2cc89..f30119fb 100644 --- a/automon/integrations/instagram/stories.py +++ b/automon/integrations/instagram/stories.py @@ -1,15 +1,15 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.action_chains import ActionChains -from automon.log import logger +from automon import log from automon.helpers.sleeper import Sleeper from automon.integrations.seleniumWrapper.config import SeleniumConfig from automon.integrations.seleniumWrapper.browser import SeleniumBrowser from automon.integrations.minioWrapper import MinioClient -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) def authenticate(username, password, minio_client=None, retries=None): @@ -39,7 +39,7 @@ def authenticate(username, password, minio_client=None, retries=None): browser.browser.get(login_page) - log.logging.debug('[authenticate] {}'.format(login_page)) + logger.debug('[authenticate] {}'.format(login_page)) Sleeper.seconds(1) @@ -84,9 +84,9 @@ def authenticate(username, password, minio_client=None, retries=None): if found_pass and found_btn: break else: - log.logging.error('[browser] Authentication failed') + logger.error('[browser] Authentication failed') - log.logging.debug( + logger.debug( '[browser] Found password field: {} Found login button: {}'.format(browser.browser.name, found_pass, found_btn)) @@ -97,7 +97,7 @@ def authenticate(username, password, minio_client=None, retries=None): Sleeper.seconds(5) - log.logging.debug( + logger.debug( '[authenticated browser] [{}] {} session: {}'.format(browser.browser.name, browser.browser.title, browser.browser.session_id)) browser.save_screenshot_to_minio() @@ -114,13 +114,13 @@ def get_stories(authenticated_browser, account): # TODO: check if account exists browser = authenticated_browser - log.logging.debug('[get_stories] {}'.format(story)) + logger.debug('[get_stories] {}'.format(story)) browser.browser.get(story) browser.save_screenshot_to_minio(prefix=account) if 'Page Not Found' in browser.browser.title: - log.logging.debug('[get_stories] no stories for {}'.format(account)) + logger.debug('[get_stories] no stories for {}'.format(account)) return num_of_stories Sleeper.seconds(2) @@ -131,14 +131,14 @@ def get_stories(authenticated_browser, account): title = browser.browser.title if title == 'Instagram': - log.logging.debug(('[get_stories] {} end of stories'.format(account))) + logger.debug(('[get_stories] {} end of stories'.format(account))) raise Exception num_of_stories += 1 Sleeper.seconds(1) browser.save_screenshot_to_minio(prefix=account) except: # TODO: disable browser proxy when done - log.logging.debug('[get_stories] done: {}'.format(account)) + logger.debug('[get_stories] done: {}'.format(account)) return num_of_stories @@ -157,14 +157,14 @@ def next_story(authenticated_browser): browser = authenticated_browser button = browser.browser.find_element_by_xpath(xpath) found_btn = True - log.logging.debug('[next_story] next story') + logger.debug('[next_story] next story') return button.click() except: pass if not found_btn: # no more stories. exit - log.logging.debug('[next_story] no more stories') + logger.debug('[next_story] no more stories') raise Exception @@ -172,7 +172,7 @@ def get_page(authenticated_browser, account): """ Get page """ # TODO: need to download page - log.logging.debug('[get_page] getting {}'.format(account)) + logger.debug('[get_page] getting {}'.format(account)) page = 'https://instagram.com/{}'.format(account) browser = authenticated_browser return browser.browser.get(page) @@ -186,9 +186,9 @@ def run(config): password = instagram_config['login']['password'] accounts = instagram_config['following'] - log.logging.debug('[login] {}'.format(login)) - log.logging.info('Running...') - log.logging.info('[accounts] {}'.format(len(accounts))) + logger.debug('[login] {}'.format(login)) + logger.info('Running...') + logger.info('[accounts] {}'.format(len(accounts))) while True: @@ -208,13 +208,13 @@ def run(config): def runrun(browser, account): - log.logging.debug( + logger.debug( '[runrun] [{}] {} session: {}'.format(browser.browser.name, browser.browser.title, browser.browser.session_id)) num_of_stories = get_stories(browser, account) - log.logging.info('[{}] {} stories'.format(account, num_of_stories)) + logger.info('[{}] {} stories'.format(account, num_of_stories)) # Sleeper.minute('instagram') @@ -229,9 +229,9 @@ def test_run(config): password = instagram_config['login']['password'] accounts = instagram_config['following'] - log.logging.debug('[login] {}'.format(login)) - log.logging.info('Running...') - log.logging.info('[accounts] {}'.format(len(accounts))) + logger.debug('[login] {}'.format(login)) + logger.info('Running...') + logger.info('[accounts] {}'.format(len(accounts))) while True: diff --git a/automon/integrations/ldap/client.py b/automon/integrations/ldap/client.py index 2cfce1fb..3460825f 100755 --- a/automon/integrations/ldap/client.py +++ b/automon/integrations/ldap/client.py @@ -5,7 +5,10 @@ from pandas import DataFrame from subprocess import Popen, PIPE -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class LdapResult(object): @@ -26,7 +29,7 @@ def __repr__(self) -> str: class LdapClient(object): - def __init__(self, log_level=Logging.INFO, **kwargs): + def __init__(self): """run ldap commands :param log_level: @@ -41,8 +44,6 @@ def __init__(self, log_level=Logging.INFO, **kwargs): self.result = None self.results = Queue() - self._log = Logging(name=LdapClient.__name__, level=log_level, **kwargs) - def __repr__(self) -> str: return f'{self.__dict__}' @@ -64,11 +65,11 @@ def ldap(self, query: str = None) -> LdapResult: self.results.put_nowait(self.result) if not result_ldap.df.empty: - # self._log.debug(f'{result_ldap.df.iloc[0]}') - self._log.debug(f'FOUND {query} ({self.results.qsize()})') + # logger.debug(f'{result_ldap.df.iloc[0]}') + logger.debug(f'FOUND {query} ({self.results.qsize()})') # print('o', end='', flush=True) else: - self._log.debug(f'UNKNOWN {query} ({self.results.qsize()})') + logger.debug(f'UNKNOWN {query} ({self.results.qsize()})') # print('.', end='', flush=True) return result_ldap diff --git a/automon/integrations/mac/airport/airport.py b/automon/integrations/mac/airport/airport.py index 76fcbfd1..0ba53b61 100644 --- a/automon/integrations/mac/airport/airport.py +++ b/automon/integrations/mac/airport/airport.py @@ -3,7 +3,7 @@ from bs4 import BeautifulSoup -from automon.log import logger +from automon import log from automon.helpers import Run from automon.helpers import Dates @@ -21,8 +21,8 @@ } -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Airport: @@ -91,10 +91,10 @@ def getinfo(self): def isReady(self): if sys.platform == 'darwin': if os.path.exists(self._airport): - log.debug(f'Airport found! {self._airport}') + logger.debug(f'Airport found! {self._airport}') return True else: - log.error(f'Airport not found! {self._airport}') + logger.error(f'Airport not found! {self._airport}') return False def run(self, args: str = None): @@ -111,12 +111,12 @@ def run(self, args: str = None): self.scan_date = Dates.iso() try: - log.info(command) + logger.info(command) if self._runner.Popen(command=command, text=True): self._scan_output = self._runner.stdout return True except Exception as e: - log.error(e) + logger.error(e) raise (Exception(e)) return False @@ -152,7 +152,7 @@ def scan_channel(self, channel: int = None): def scan_summary(self, channel: int = None, args: str = None, output: bool = True): if self.scan(channel=channel, args=args): if output: - log.info(f'{self._scan_output}') + logger.info(f'{self._scan_output}') return True return False @@ -185,6 +185,6 @@ def scan_xml(self, ssid: str = None, channel: int = None) -> [Ssid]: return True except Exception as e: - log.error(f'Scan not parsed: {e}, {self.scan_cmd}') + logger.error(f'Scan not parsed: {e}, {self.scan_cmd}') return False diff --git a/automon/integrations/mac/airport/scan.py b/automon/integrations/mac/airport/scan.py index 33eb56f1..31991f3d 100644 --- a/automon/integrations/mac/airport/scan.py +++ b/automon/integrations/mac/airport/scan.py @@ -1,11 +1,11 @@ from bs4 import BeautifulSoup -from automon.log import logger +from automon import log from .ssid import Ssid -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class ScanXml: @@ -75,7 +75,7 @@ def ssids(self) -> [Ssid]: bssids = xml.contents[1].contents[0].contents scan = [self._bs2dict(x) for x in bssids] except: - log.error(f'No BSSIDs') + logger.error(f'No BSSIDs') if scan: ssids = [Ssid(ssid) for ssid in scan] @@ -103,5 +103,5 @@ def summary(self) -> dict: summary['SSID'][ssid.SSID] = count summary['SSID'] = {k: v for k, v in sorted(summary['SSID'].items())} - log.info(f'Total SSID: {summary["Total SSID"]}') + logger.info(f'Total SSID: {summary["Total SSID"]}') return summary diff --git a/automon/integrations/mac/airport/ssid.py b/automon/integrations/mac/airport/ssid.py index df4f1500..b3b828ba 100644 --- a/automon/integrations/mac/airport/ssid.py +++ b/automon/integrations/mac/airport/ssid.py @@ -1,7 +1,7 @@ -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Ssid: @@ -59,5 +59,5 @@ def summary(self): f'[noise: {self.NOISE}] ' \ f'[age: {self.AGE}] ' - log.debug(f'{summary}') + logger.debug(f'{summary}') return summary diff --git a/automon/integrations/minioWrapper/assertions.py b/automon/integrations/minioWrapper/assertions.py index 922b1782..7e9cab91 100644 --- a/automon/integrations/minioWrapper/assertions.py +++ b/automon/integrations/minioWrapper/assertions.py @@ -1,6 +1,7 @@ -from automon.log import Logging +from automon import log -log = Logging(name='MinioAssertions', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class MinioAssertions: @@ -9,5 +10,5 @@ class MinioAssertions: def bucket_name(bucket: str) -> bool: if bucket == f'{bucket}'.lower(): return f'{bucket}' - log.warning(msg=f'bucket name "{bucket}" must be lower') + logger.warning(msg=f'bucket name "{bucket}" must be lower') return f'{bucket}'.lower() diff --git a/automon/integrations/minioWrapper/client.py b/automon/integrations/minioWrapper/client.py index 0b070979..83d44971 100644 --- a/automon/integrations/minioWrapper/client.py +++ b/automon/integrations/minioWrapper/client.py @@ -5,14 +5,15 @@ from typing import Optional -from automon.log import Logging +from automon import log from .bucket import Bucket from .object import Object, DeleteObject from .config import MinioConfig from .assertions import MinioAssertions -log = Logging(name='MinioClient', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class MinioClient(object): @@ -57,7 +58,7 @@ def _wrapper(self, *args, **kwargs): # if not self._sessionExpired() or self.client.list_buckets(): return func(self, *args, **kwargs) except Exception as e: - log.error(f'Minio client not connected. {e}') + logger.error(f'Minio client not connected. {e}') return False return _wrapper @@ -68,7 +69,7 @@ def download_object(self, bucket_name: str, file: str): """ bucket_name = MinioAssertions.bucket_name(bucket_name) - log.debug(f'[downloader] Downloading: {bucket_name}/{file.object_name}') + logger.debug(f'[downloader] Downloading: {bucket_name}/{file.object_name}') return self.client.get_object(bucket_name, file.object_name) @_is_connected @@ -82,14 +83,14 @@ def get_bucket(self, bucket_name: str, **kwargs) -> Bucket or None: bucket_index = buckets.index(bucket_name) return buckets[bucket_index] - log.info(msg=f'Get bucket: "{bucket_name}" does not exist') + logger.info(msg=f'Get bucket: "{bucket_name}" does not exist') return @_is_connected def is_connected(self): """Check if MinioClient is connected """ - log.info(f'Minio client connected') + logger.info(f'Minio client connected') return True @_is_connected @@ -99,7 +100,7 @@ def list_buckets(self, **kwargs) -> [Bucket]: buckets = self.client.list_buckets(**kwargs) buckets = [Bucket(x) for x in buckets] - log.info(f'List buckets: {len(buckets)}') + logger.info(f'List buckets: {len(buckets)}') return buckets @_is_connected @@ -126,11 +127,11 @@ def list_objects( if prefix: msg += f' Prefix: "{prefix}"' - log.info(msg) + logger.info(msg) return objects except Exception as error: - log.error(f'failed to list objects. {error}', enable_traceback=False) + logger.error(f'failed to list objects. {error}') return [] @@ -149,7 +150,7 @@ def list_objects_generator( return objects except Exception as e: - log.error(f'failed to list objects. {e}') + logger.error(f'failed to list objects. {e}') return [] @@ -161,11 +162,11 @@ def remove_bucket(self, bucket_name: str, **kwargs) -> bool: try: self.client.remove_bucket(bucket_name, **kwargs) - log.info(f'Removed bucket: "{bucket_name}"') + logger.info(f'Removed bucket: "{bucket_name}"') return True except Exception as e: - log.error(f'Remove bucket: "{bucket_name}" failed. {e}', enable_traceback=False) + logger.error(f'Remove bucket: "{bucket_name}" failed. {e}') return False @@ -178,11 +179,11 @@ def remove_objects(self, bucket_name: str, prefix: str = None, **kwargs) -> bool delete_objects = [DeleteObject(x) for x in objects] if not delete_objects: - log.info(f'Bucket is empty: "{bucket_name}"') + logger.info(f'Bucket is empty: "{bucket_name}"') return True errors = list(self.client.remove_objects(bucket_name, delete_objects, **kwargs)) - log.info(f'Removed {len(delete_objects)} objects in bucket "{bucket_name}"') + logger.info(f'Removed {len(delete_objects)} objects in bucket "{bucket_name}"') if self.list_objects(bucket_name, prefix): return self.remove_objects(bucket_name, prefix=prefix) @@ -196,10 +197,10 @@ def make_bucket(self, bucket_name: str) -> Bucket: bucket_name = MinioAssertions.bucket_name(bucket_name) try: self.client.make_bucket(bucket_name) - log.info(f'Created bucket: "{bucket_name}"') + logger.info(f'Created bucket: "{bucket_name}"') except Exception as e: - log.warning(f'Bucket exists: "{bucket_name}". {e}') + logger.warning(f'Bucket exists: "{bucket_name}". {e}') return self.get_bucket(bucket_name) @@ -213,7 +214,7 @@ def put_object(self, bucket_name: str, object_name: str, data: io.BytesIO, lengt bucket_name = MinioAssertions.bucket_name(bucket_name) length = length or data.getvalue().__len__() - log.debug(f'[{self.put_object.__name__}] Uploading: {object_name}') + logger.debug(f'[{self.put_object.__name__}] Uploading: {object_name}') try: put = self.client.put_object( @@ -224,7 +225,7 @@ def put_object(self, bucket_name: str, object_name: str, data: io.BytesIO, lengt metadata=metadata, sse=sse, progress=progress) - log.info( + logger.info( f'[put_object] Saved to: ' f'{self.config.endpoint}/{bucket_name}/{object_name}' ) @@ -232,7 +233,7 @@ def put_object(self, bucket_name: str, object_name: str, data: io.BytesIO, lengt return put except Exception as e: - log.error( + logger.error( f'[{self.put_object.__name__}] Unable to save: ' f'{self.config.endpoint}/{bucket_name}/{bucket_name} ' f'{e}', @@ -253,5 +254,5 @@ def check_connection(host, port): s.close() return True except Exception as e: - log.error(e, enable_traceback=False) + logger.error(e) return False diff --git a/automon/integrations/minioWrapper/config.py b/automon/integrations/minioWrapper/config.py index 7f6b2ba2..0d05f91b 100644 --- a/automon/integrations/minioWrapper/config.py +++ b/automon/integrations/minioWrapper/config.py @@ -1,10 +1,11 @@ import os import urllib3 -from automon.log import Logging +from automon import log from automon.helpers import environ -log = Logging(name='MinioConfig', level=Logging.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class MinioConfig(object): @@ -34,13 +35,13 @@ def __init__(self, endpoint: str = None, self._http_client = http_client or environ('MINIO_HTTP_CLIENT') if not self.endpoint: - log.warning(f'missing MINIO_ENDPOINT') + logger.warning(f'missing MINIO_ENDPOINT') if not self.access_key: - log.warning(f'missing MINIO_ACCESS_KEY') + logger.warning(f'missing MINIO_ACCESS_KEY') if not self.secret_key: - log.warning(f'missing MINIO_SECRET_KEY') + logger.warning(f'missing MINIO_SECRET_KEY') @property def access_key(self): diff --git a/automon/integrations/neo4jWrapper/client.py b/automon/integrations/neo4jWrapper/client.py index c418d603..f0cd8c93 100644 --- a/automon/integrations/neo4jWrapper/client.py +++ b/automon/integrations/neo4jWrapper/client.py @@ -4,15 +4,15 @@ from neo4j import GraphDatabase from queue import Queue -from automon.log import logger +from automon import log from automon.integrations.neo4jWrapper.cypher import Cypher from .config import Neo4jConfig from .results import Results -logger.logging.getLogger('neo4j').setLevel(logging.ERROR) -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +log.logging.getLogger('neo4j').setLevel(logging.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Neo4jClient: @@ -50,11 +50,11 @@ def _client(self): client = GraphDatabase.driver( uri=self._config.NEO4J_HOST, auth=(self._config.NEO4J_USER, self._config.NEO4J_PASSWORD)) - log.info(f'Connected to neo4j server: {self._config.NEO4J_HOST}') + logger.info(f'Connected to neo4j server: {self._config.NEO4J_HOST}') return client except Exception as e: - log.error(f'Cannot connect to neo4j server: {self._config.NEO4J_HOST}, {e}') + logger.error(f'Cannot connect to neo4j server: {self._config.NEO4J_HOST}, {e}') return False @@ -165,7 +165,7 @@ def merge_node(self, data: dict, label: str = None, **kwargs): cypher = self._Cypher.merge_dict(label=label, data=data) - # log.debug(f'{final_cypher}') + # logger.debug(f'{final_cypher}') return self._send(cypher) @@ -180,12 +180,12 @@ def run(self, cypher=None) -> bool: response = self._session.run(cypher) self.results = Results(response) - log.info(f'cypher: {cypher}') - log.debug(f'Results: {self.results}') + logger.info(f'cypher: {cypher}') + logger.debug(f'Results: {self.results}') return True except Exception as e: - log.error(f"{e}") + logger.error(f"{e}") return False diff --git a/automon/integrations/neo4jWrapper/clientAsync.py b/automon/integrations/neo4jWrapper/clientAsync.py index 4abcf8ff..31c70863 100644 --- a/automon/integrations/neo4jWrapper/clientAsync.py +++ b/automon/integrations/neo4jWrapper/clientAsync.py @@ -4,15 +4,15 @@ from neo4j import GraphDatabase from queue import Queue -from automon.log import logger +from automon import log from automon.integrations.neo4jWrapper.cypher import Cypher from .config import Neo4jConfig from .results import Results -logger.logging.getLogger('neo4j').setLevel(logger.ERROR) -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +log.logging.getLogger('neo4j').setLevel(logger.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Neo4jAsyncClient: @@ -56,11 +56,11 @@ def _client(self): client = GraphDatabase.driver( uri=self.config.NEO4J_HOST, auth=(self.config.NEO4J_USER, self.config.NEO4J_PASSWORD)) - log.info(f'Connected to neo4j server: {self.config.NEO4J_HOST}') + logger.info(f'Connected to neo4j server: {self.config.NEO4J_HOST}') return client except Exception as e: - log.error(f'Cannot connect to neo4j server: {self.config.NEO4J_HOST}, {e}') + logger.error(f'Cannot connect to neo4j server: {self.config.NEO4J_HOST}, {e}') return False @@ -78,10 +78,10 @@ def run(self): try: while not self.cypher.empty(): cypher = self.cypher.get_nowait() - log.debug(f'cypher: {cypher}') + logger.debug(f'cypher: {cypher}') self.session.run(cypher) return True except Exception as e: - log.error(f'{e}') + logger.error(f'{e}') return False diff --git a/automon/integrations/neo4jWrapper/config.py b/automon/integrations/neo4jWrapper/config.py index 8d454888..c1258d08 100644 --- a/automon/integrations/neo4jWrapper/config.py +++ b/automon/integrations/neo4jWrapper/config.py @@ -1,11 +1,11 @@ import os -from automon.log import logger +from automon import log from automon.helpers.sanitation import Sanitation from automon.helpers.osWrapper.environ import environ -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Neo4jConfig: @@ -25,14 +25,16 @@ def __init__( self.encrypted = encrypted self.trust = trust - if not self.NEO4J_USER: log.error(f'missing NEO4J_USER') - if not self.NEO4J_PASSWORD: log.error(f'missing NEO4J_PASSWORD') - if not self.NEO4J_HOST: log.error(f'missing NEO4J_HOST') - @property def is_ready(self) -> bool: if self.NEO4J_USER and self.NEO4J_PASSWORD and self.NEO4J_HOST: return True + if not self.NEO4J_USER: + logger.error(f'missing NEO4J_USER') + if not self.NEO4J_PASSWORD: + logger.error(f'missing NEO4J_PASSWORD') + if not self.NEO4J_HOST: + logger.error(f'missing NEO4J_HOST') return False def __repr__(self): diff --git a/automon/integrations/neo4jWrapper/cypher.py b/automon/integrations/neo4jWrapper/cypher.py index 70350872..12e71044 100644 --- a/automon/integrations/neo4jWrapper/cypher.py +++ b/automon/integrations/neo4jWrapper/cypher.py @@ -3,10 +3,10 @@ from urllib.parse import urlencode from datetime import datetime, timezone -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Node: @@ -62,14 +62,14 @@ def assert_label(label: str) -> str: return '' if re.search('[:]', label): - log.warning(f"Invalid label '{label}': Colon is not needed here") + logger.warning(f"Invalid label '{label}': Colon is not needed here") label = label.replace(':', '') if re.search('[`]', label): label = label.replace('`', '') if not re.search('[a-zA-Z]', label[0]): # First letter of a label must be a letter - log.error(f"Invalid label '{label}': First character of Neo4j :LABEL must be a letter") + logger.error(f"Invalid label '{label}': First character of Neo4j :LABEL must be a letter") else: return f':`{label}`' # :`LABEL` diff --git a/automon/integrations/neo4jWrapper/results.py b/automon/integrations/neo4jWrapper/results.py index bb7d78b2..1b7d6f6b 100644 --- a/automon/integrations/neo4jWrapper/results.py +++ b/automon/integrations/neo4jWrapper/results.py @@ -2,10 +2,10 @@ from neo4j.work.summary import ResultSummary -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Results(ResultSummary): diff --git a/automon/integrations/nmap/client.py b/automon/integrations/nmap/client.py index 4e14866f..0c78595e 100644 --- a/automon/integrations/nmap/client.py +++ b/automon/integrations/nmap/client.py @@ -1,14 +1,14 @@ import os -from automon.log import logger +from automon import log from automon.helpers import Run from automon.helpers.dates import Dates from .config import NmapConfig from .output import NmapResult -log = logger.logging.getLogger(__name__) -log.setLevel(logger.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class Nmap(object): @@ -55,7 +55,7 @@ def pretty(self): def run(self, command: str, output: bool = True, cleanup: bool = True, **kwargs) -> bool: if not self.isReady(): - log.error(msg=f'nmap not found') + logger.error(msg=f'nmap not found') return False nmap_command = f'{self.config.nmap} ' @@ -66,9 +66,9 @@ def run(self, command: str, output: bool = True, cleanup: bool = True, **kwargs) nmap_command += f'{command}' - log.info(f'running {nmap_command}') + logger.info(f'running {nmap_command}') self._runner.run(nmap_command, **kwargs) - log.debug(f'finished') + logger.debug(f'finished') self.command = nmap_command self._stdout = self._runner.stdout @@ -82,10 +82,10 @@ def run(self, command: str, output: bool = True, cleanup: bool = True, **kwargs) if cleanup: os.remove(self.output_file) - log.info(f'deleted {self.output_file}') + logger.info(f'deleted {self.output_file}') if self._stderr: - log.error(msg=f'{self._stderr.decode()}') + logger.error(msg=f'{self._stderr.decode()}') return False return True diff --git a/automon/integrations/nmap/config.py b/automon/integrations/nmap/config.py index 28a2d225..4359907f 100644 --- a/automon/integrations/nmap/config.py +++ b/automon/integrations/nmap/config.py @@ -1,8 +1,8 @@ -from automon.log import logger +from automon import log from automon.helpers import Run -log = logger.logging.getLogger(__name__) -log.setLevel(logger.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class NmapConfig(object): @@ -19,9 +19,9 @@ def isReady(self, **kwargs): if check.stdout: self.nmap = check.stdout.decode().strip() - log.debug(f'nmap located, {self.nmap}') + logger.debug(f'nmap located, {self.nmap}') return True else: - log.error(f'nmap not found') + logger.error(f'nmap not found') return False diff --git a/automon/integrations/nmap/output.py b/automon/integrations/nmap/output.py index e5080f13..1c5d751f 100644 --- a/automon/integrations/nmap/output.py +++ b/automon/integrations/nmap/output.py @@ -2,13 +2,13 @@ import xmltodict import pandas as pd -from automon.log import logger +from automon import log from pandas import DataFrame from automon.helpers import Run from automon.integrations.datascience import Pandas -log = logger.logging.getLogger(__name__) -log.setLevel(logger.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class NmapResult(object): @@ -57,11 +57,11 @@ def __init__(self, file: str = None, run: Run = None, **kwargs): self.summary = self._runstats.loc[:, 'finished.@summary'][0] self.time_finished = self._runstats.loc[:, 'finished.@time'][0] - log.info(f'hosts up: {self.hosts_up}') - log.info(f'hosts down: {self.hosts_down}') - # log.info(f'hosts total: {self.hosts_total}') - log.info(f'{self.summary}') - log.info(f'finished {self.file} ({round(df.memory_usage().sum() / 1024, 2)} Kb)') + logger.info(f'hosts up: {self.hosts_up}') + logger.info(f'hosts down: {self.hosts_down}') + # logger.info(f'hosts total: {self.hosts_total}') + logger.info(f'{self.summary}') + logger.info(f'finished {self.file} ({round(df.memory_usage().sum() / 1024, 2)} Kb)') def __repr__(self): msg = f'{self.summary} ' @@ -117,7 +117,7 @@ def _normalize_ports(self, df): else: df_host[port].update(status) - log.debug(f"{df_host.loc[:, ['address.@addr'] + [x for x in scanned_ports if x in df_host]]}") + logger.debug(f"{df_host.loc[:, ['address.@addr'] + [x for x in scanned_ports if x in df_host]]}") i += 1 diff --git a/automon/integrations/openvpn/openvpn.py b/automon/integrations/openvpn/openvpn.py index dd634526..75c33f4d 100644 --- a/automon/integrations/openvpn/openvpn.py +++ b/automon/integrations/openvpn/openvpn.py @@ -2,10 +2,11 @@ import io from automon.integrations.minioWrapper import MinioClient, MinioConfig -from automon.log import Logging +from automon import log from automon.helpers.sleeper import Sleeper -log = Logging(name='openvpn', level='info') +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class ClientConfig: @@ -46,7 +47,7 @@ def __init__(self, name, hosts, options=None): """ - log.logging.debug('[ClientConfig] Creating new OpenVPN client config') + logger.debug('[ClientConfig] Creating new OpenVPN client config') self.name = name @@ -114,7 +115,7 @@ def build_config(self, prefix): def collector(minio_client, bucket, folder): """ Collect required files to build an OpenVPN client """ - log.logging.debug('[collector] Collecting all Minio bucket files') + logger.debug('[collector] Collecting all Minio bucket files') ca = None cert = [] @@ -150,7 +151,7 @@ def collector(minio_client, bucket, folder): def put_object(minio_client: MinioClient, bucket, client_configs, config_name, config_data, config_len): """ Minio object uploader """ - log.logging.debug(f'[put_object] Uploading: {config_name}') + logger.debug(f'[put_object] Uploading: {config_name}') return minio_client.put_object(bucket, f'{client_configs}/{config_name}', config_data, config_len) @@ -176,11 +177,11 @@ def create_configs(minio_client: MinioClient, bucket: str, client_configs: str, put_object(minio_client, bucket, client_configs, config_name, config_data, config_len) - log.logging.debug(f'[create_configs] OpenVPN client config uploaded: {config_name}') + logger.debug(f'[create_configs] OpenVPN client config uploaded: {config_name}') def run(minio_config: MinioConfig, openvpn_config): - log.logging.info('Running...') + logger.info('Running...') while True: minio_config.secure = False @@ -206,8 +207,8 @@ def run(minio_config: MinioConfig, openvpn_config): ca, cert, key, ta = collector(minio_client, minio_bucket, keys) create_configs(minio_client, minio_bucket, client_configs, ca, cert, key, ta, hosts, prefix, options) - log.logging.info('[build client configs] Finshed building all OpenVPN clients') - log.logging.debug('[ClientConfig] sleeping') + logger.info('[build client configs] Finshed building all OpenVPN clients') + logger.debug('[ClientConfig] sleeping') Sleeper.day() diff --git a/automon/integrations/requestsWrapper/client.py b/automon/integrations/requestsWrapper/client.py index a9b7ae23..bf6c82c4 100644 --- a/automon/integrations/requestsWrapper/client.py +++ b/automon/integrations/requestsWrapper/client.py @@ -1,11 +1,11 @@ import json import requests -from automon.log import logger +from automon import log from .config import RequestsConfig -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class RequestsClient(object): @@ -42,7 +42,7 @@ def _log_result(self): self.results.status_code, ] msg = ' '.join(msg) - return log.debug(msg) + return logger.debug(msg) msg = [ self.results.request.method, @@ -53,7 +53,7 @@ def _log_result(self): ] msg = ' '.join(msg) - return log.error(msg) + return logger.error(msg) def _params(self, url, data, headers): if url is None: @@ -94,7 +94,7 @@ def delete(self, return True except Exception as e: self.errors = e - log.error(f'delete failed. {e}') + logger.error(f'delete failed. {e}') return False def get(self, @@ -108,7 +108,7 @@ def get(self, try: self.results = requests.get(url=url, data=data, headers=headers, **kwargs) - log.debug( + logger.debug( f'{self.results.url} ' f'{round(len(self.results.content) / 1024, 2)} KB ' f'{self.results.status_code}' @@ -117,7 +117,7 @@ def get(self, return True except Exception as e: self.errors = e - log.error(f'{e}') + logger.error(f'{e}') return False def patch(self, @@ -131,7 +131,7 @@ def patch(self, try: self.results = requests.patch(url=url, data=data, headers=headers, **kwargs) - log.debug( + logger.debug( f'{self.results.url} ' f'{round(len(self.results.content) / 1024, 2)} KB ' f'{self.results.status_code}' @@ -140,7 +140,7 @@ def patch(self, return True except Exception as e: self.errors = e - log.error(f'patch failed. {e}') + logger.error(f'patch failed. {e}') return False def post(self, @@ -154,7 +154,7 @@ def post(self, try: self.results = requests.post(url=url, data=data, headers=headers, **kwargs) - log.debug( + logger.debug( f'{self.results.url} ' f'{round(len(self.results.content) / 1024, 2)} KB ' f'{self.results.status_code}' @@ -163,7 +163,7 @@ def post(self, return True except Exception as e: self.errors = e - log.error(f'post failed. {e}') + logger.error(f'post failed. {e}') return False def put(self, @@ -177,7 +177,7 @@ def put(self, try: self.results = requests.put(url=url, data=data, headers=headers, **kwargs) - log.debug( + logger.debug( f'{self.results.url} ' f'{round(len(self.results.content) / 1024, 2)} KB ' f'{self.results.status_code}' @@ -186,7 +186,7 @@ def put(self, return True except Exception as e: self.errors = e - log.error(f'put failed. {e}') + logger.error(f'put failed. {e}') return False def to_dict(self): diff --git a/automon/integrations/requestsWrapper/config.py b/automon/integrations/requestsWrapper/config.py index bec75a1a..2b7060b0 100644 --- a/automon/integrations/requestsWrapper/config.py +++ b/automon/integrations/requestsWrapper/config.py @@ -1,9 +1,9 @@ import requests -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class RequestsConfig(object): diff --git a/automon/integrations/requestsWrapper/rest.py b/automon/integrations/requestsWrapper/rest.py index b7ec653f..3221e794 100644 --- a/automon/integrations/requestsWrapper/rest.py +++ b/automon/integrations/requestsWrapper/rest.py @@ -1,10 +1,10 @@ from .client import RequestsClient from .config import RequestsConfig -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class BaseRestClient: requests: RequestsClient diff --git a/automon/integrations/scrapyWrapper/client.py b/automon/integrations/scrapyWrapper/client.py index 634ae6d6..53a97b39 100644 --- a/automon/integrations/scrapyWrapper/client.py +++ b/automon/integrations/scrapyWrapper/client.py @@ -1,8 +1,9 @@ import scrapy -from automon.log import Logging +from automon import log -log = Logging(name='ScrapyClient', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class ScrapyClient(object): diff --git a/automon/integrations/seleniumWrapper/actions.py b/automon/integrations/seleniumWrapper/actions.py index ab917a67..d960c67c 100644 --- a/automon/integrations/seleniumWrapper/actions.py +++ b/automon/integrations/seleniumWrapper/actions.py @@ -1,9 +1,9 @@ from selenium.webdriver.common.action_chains import ActionChains -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class SeleniumActions: diff --git a/automon/integrations/seleniumWrapper/browser.py b/automon/integrations/seleniumWrapper/browser.py index 6875e0d5..01f160ae 100644 --- a/automon/integrations/seleniumWrapper/browser.py +++ b/automon/integrations/seleniumWrapper/browser.py @@ -11,7 +11,7 @@ from urllib.parse import urlparse from bs4 import BeautifulSoup -from automon.log import logger +from automon import log from automon.helpers.dates import Dates from automon.helpers.sleeper import Sleeper from automon.helpers.sanitation import Sanitation @@ -19,8 +19,8 @@ from .config import SeleniumConfig from .user_agents import SeleniumUserAgentBuilder -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SeleniumBrowser(object): @@ -54,7 +54,7 @@ def config(self): return self._config def cookie_file_to_dict(self, file: str = 'cookies.txt'): - log.debug(f'{file}') + logger.debug(f'{file}') with open(file, 'r') as file: return json.loads(file.read()) @@ -90,7 +90,7 @@ def keys(self): def refresh(self): self.webdriver.refresh() - log.info(f'{True}') + logger.info(f'{True}') @property def url(self): @@ -106,12 +106,12 @@ def user_agent(self): @property def current_url(self): if self.webdriver: - log.debug(self._current_url) + logger.debug(self._current_url) if self._current_url == 'data:,': return '' return self._current_url - log.info(None) + logger.info(None) return '' @property @@ -141,19 +141,19 @@ def action_click(self, xpath: str, note: str = None) -> str or False: click = self.find_element(value=xpath, by=self.by.XPATH) click.click() if note: - log.debug(str(dict( + logger.debug(str(dict( note=note, xpath=xpath, ))) else: - log.debug(str(dict( + logger.debug(str(dict( xpath=xpath, ))) return click except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, xpath=xpath, message=message, @@ -173,14 +173,14 @@ def action_type(self, key: str or Keys, secret: bool = True): if secret: key = f'*' * len(key) - log.debug(str(dict( + logger.debug(str(dict( send_keys=key, ))) return True except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( url=self.url, send_keys=key, message=message, @@ -193,10 +193,10 @@ def add_cookie(self, cookie_dict: dict) -> bool: result = self.webdriver.add_cookie(cookie_dict=cookie_dict) if result is None: - log.debug(f'{cookie_dict}') + logger.debug(f'{cookie_dict}') return True - log.error(f'{cookie_dict}') + logger.error(f'{cookie_dict}') return False def add_cookie_from_file(self, file: str) -> bool: @@ -207,18 +207,18 @@ def add_cookie_from_file(self, file: str) -> bool: ) return True - log.error(f'{file}') + logger.error(f'{file}') return False def add_cookies_from_list(self, cookies_list: list): for cookie in cookies_list: self.add_cookie(cookie_dict=cookie) - log.debug(f'{True}') + logger.debug(f'{True}') return True def add_cookie_from_current_url(self): - log.info(f'{self.url}') + logger.info(f'{self.url}') return self.add_cookie_from_url(self.url) def add_cookie_from_url(self, url: str): @@ -226,54 +226,54 @@ def add_cookie_from_url(self, url: str): cookie_file = self._url_filename(url=url) if os.path.exists(cookie_file): - log.info(f'{cookie_file}') + logger.info(f'{cookie_file}') return self.add_cookie_from_file(file=cookie_file) - log.error(f'{cookie_file}') + logger.error(f'{cookie_file}') def add_cookie_from_base64(self, base64_str: str): if base64_str: self.add_cookies_from_list( json.loads(base64.b64decode(base64_str)) ) - log.debug(f'{True}') + logger.debug(f'{True}') return True - log.error(f'{base64_str}') + logger.error(f'{base64_str}') return False def delete_all_cookies(self) -> None: result = self.webdriver.delete_all_cookies() - log.info(f'{True}') + logger.info(f'{True}') return result def _url_filename(self, url: str): parsed = self.urlparse(url) hostname = parsed.hostname cookie_file = f'cookies-{hostname}.txt' - log.info(f'{cookie_file}') + logger.info(f'{cookie_file}') return cookie_file def get_cookie(self, name: str) -> dict: result = self.webdriver.get_cookie(name=name) - log.info(f'{result}') + logger.info(f'{result}') return result def get_cookies(self) -> [dict]: result = self.webdriver.get_cookies() - log.debug(f'{True}') + logger.debug(f'{True}') return result def get_cookies_base64(self) -> base64: result = self.get_cookies() - log.debug(f'{True}') + logger.debug(f'{True}') return base64.b64encode( json.dumps(result).encode() ).decode() def get_cookies_json(self) -> json.dumps: cookies = self.get_cookies() - log.debug(f'{True}') + logger.debug(f'{True}') return json.dumps(cookies) def get_cookies_summary(self): @@ -291,12 +291,12 @@ def get_cookies_summary(self): else: summary[domain] = [cookie] - log.debug(f'{summary}') + logger.debug(f'{summary}') return summary def close(self): """close browser""" - log.info(f'closed') + logger.info(f'closed') self.webdriver.close() @staticmethod @@ -317,7 +317,7 @@ def find_element( **kwargs): """find element""" element = self.webdriver.find_element(value=value, by=by, **kwargs) - log.info(str(dict( + logger.info(str(dict( url=self.url, text=element.text, value=value, @@ -327,7 +327,7 @@ def find_element( def find_xpath(self, value: str, by: By = By.XPATH, **kwargs): """find xpath""" xpath = self.find_element(value=value, by=by, **kwargs) - log.info(str(dict( + logger.info(str(dict( url=self.url, text=xpath.text, value=value, @@ -338,14 +338,14 @@ def get(self, url: str, **kwargs) -> bool: """get url""" try: if self.webdriver.get(url, **kwargs) is None: - log.info(str(dict( + logger.info(str(dict( url=url, current_url=self._current_url, kwargs=kwargs ))) return True except Exception as error: - log.error(str(dict( + logger.error(str(dict( error=error, ))) @@ -371,7 +371,7 @@ def get_random_user_agent(self, filter: list or str = None, case_sensitive: bool def get_screenshot_as_base64(self, **kwargs): """screenshot as base64""" screenshot = self.webdriver.get_screenshot_as_base64(**kwargs) - log.debug(f'{round(len(screenshot) / 1024)} KB') + logger.debug(f'{round(len(screenshot) / 1024)} KB') return screenshot @@ -393,21 +393,21 @@ def get_screenshot_as_file( def get_screenshot_as_png(self, **kwargs): """screenshot as png""" screenshot = self.webdriver.get_screenshot_as_png(**kwargs) - log.debug(f'{round(len(screenshot) / 1024)} KB') + logger.debug(f'{round(len(screenshot) / 1024)} KB') return screenshot def is_running(self) -> bool: """browser is running""" if self.webdriver: - log.info(f'{True}') + logger.info(f'{True}') return True - log.error(f'{False}') + logger.error(f'{False}') return False def urlparse(self, url: str): parsed = urlparse(url=url) - log.debug(f'{parsed}') + logger.debug(f'{parsed}') return parsed def quit(self) -> bool: @@ -418,7 +418,7 @@ def quit(self) -> bool: self.webdriver.stop_client() except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( message=message, session=session, stacktrace=stacktrace, @@ -432,7 +432,7 @@ def run(self): def save_cookies_for_current_url(self): filename = self._url_filename(url=self.url) - log.info(f'{filename}') + logger.info(f'{filename}') return self.save_cookies_to_file(file=filename) def save_cookies_to_file(self, file: str): @@ -442,10 +442,10 @@ def save_cookies_to_file(self, file: str): ) if os.path.exists(file): - log.info(f'{os.path.abspath(file)} ({os.stat(file).st_size} B)') + logger.info(f'{os.path.abspath(file)} ({os.stat(file).st_size} B)') return True - log.error(f'{file}') + logger.error(f'{file}') return False def save_screenshot( @@ -470,7 +470,7 @@ def save_screenshot( save = os.path.join(path, filename) if self.webdriver.save_screenshot(save, **kwargs): - log.info(f'Saving screenshot to: {save} ({round(os.stat(save).st_size / 1024)} KB)') + logger.info(f'Saving screenshot to: {save} ({round(os.stat(save).st_size / 1024)} KB)') return True return False @@ -483,7 +483,7 @@ def set_window_size(self, width=1920, height=1080, device_type=None) -> bool: device_type=device_type) except Exception as error: message, session, stacktrace = self.error_parsing(error) - log.error(str(dict( + logger.error(str(dict( message=message, session=session, stacktrace=stacktrace, @@ -494,7 +494,7 @@ def set_window_size(self, width=1920, height=1080, device_type=None) -> bool: def set_window_position(self, x: int = 0, y: int = 0): """set browser position""" result = self.webdriver.set_window_position(x, y) - log.info(f'{result}') + logger.info(f'{result}') return result def start(self): @@ -523,14 +523,14 @@ def wait_for( by=by, value=value, **kwargs) - log.debug(str(dict( + logger.debug(str(dict( by=by, url=self.url, value=value, ))) return value except: - log.error(str(dict( + logger.error(str(dict( by=by, url=self.url, value=value, @@ -540,14 +540,14 @@ def wait_for( by=by, value=value, **kwargs) - log.debug(str(dict( + logger.debug(str(dict( by=by, url=self.url, value=value, ))) return value except Exception as error: - log.error(str(dict( + logger.error(str(dict( by=by, url=self.url, value=value, @@ -557,7 +557,7 @@ def wait_for( retry += 1 - log.error(str(dict( + logger.error(str(dict( url=self.url, retry=f'{retry}/{retries}', ))) diff --git a/automon/integrations/seleniumWrapper/browser_types.py b/automon/integrations/seleniumWrapper/browser_types.py index 35288e63..3987578c 100644 --- a/automon/integrations/seleniumWrapper/browser_types.py +++ b/automon/integrations/seleniumWrapper/browser_types.py @@ -10,7 +10,7 @@ from selenium.webdriver import Safari from selenium.webdriver import WebKitGTK -from automon.log import logger +from automon import log from .config import SeleniumConfig @@ -23,8 +23,8 @@ from selenium.webdriver import Chrome as ChromiumEdge from selenium.webdriver import Chrome as WPEWebKit -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SeleniumBrowserType(object): config: SeleniumConfig @@ -39,7 +39,7 @@ def __repr__(self): def chrome(self, options: list = None) -> Chrome: """Chrome""" - log.info(f'Browser set as Chrome') + logger.info(f'Browser set as Chrome') chrome_options = selenium.webdriver.chrome.options.Options() @@ -52,7 +52,7 @@ def chrome(self, options: list = None) -> Chrome: return self.webdriver.ChromeWrapper(executable_path=self.chromedriver, options=chrome_options) return self.webdriver.ChromeWrapper(options=chrome_options) except Exception as e: - log.error(f'Browser not set. {e}') + logger.error(f'Browser not set. {e}') def chrome_headless(self, options: list = None, **kwargs) -> Chrome: """Chrome headless @@ -66,7 +66,7 @@ def chrome_headless(self, options: list = None, **kwargs) -> Chrome: chrome_options.headless = True # also works """ - log.info(f'Browser set as Chrome Headless') + logger.info(f'Browser set as Chrome Headless') chrome_options = selenium.webdriver.chrome.options.Options() chrome_options.headless = True @@ -80,12 +80,12 @@ def chrome_headless(self, options: list = None, **kwargs) -> Chrome: return self.webdriver.ChromeWrapper(self.chromedriver, options=chrome_options, **kwargs) return self.webdriver.ChromeWrapper(options=chrome_options, **kwargs) except Exception as e: - log.error(f'Browser not set. {e}') + logger.error(f'Browser not set. {e}') @property def chromium_edge(self, options: list = None, **kwargs) -> ChromiumEdge: """Chromium""" - log.info(f'Browser set as Chromium Edge') + logger.info(f'Browser set as Chromium Edge') chromium_options = selenium.webdriver.chromium.options.ChromiumOptions() @@ -98,52 +98,52 @@ def chromium_edge(self, options: list = None, **kwargs) -> ChromiumEdge: return self.webdriver.ChromiumEdge(self.chromedriver, options=chromium_options, **kwargs) return self.webdriver.ChromiumEdge(options=chromium_options, **kwargs) except Exception as e: - log.error(f'Browser not set. {e}') + logger.error(f'Browser not set. {e}') def edge(self, **kwargs) -> Edge: """Edge""" - log.info(f'Browser set as Edge') + logger.info(f'Browser set as Edge') return self.webdriver.Edge(**kwargs) def firefox(self, **kwargs) -> Firefox: """Firefox""" - log.info(f'Browser set as Firefox') + logger.info(f'Browser set as Firefox') return self.webdriver.Firefox(**kwargs) def ie(self, **kwargs) -> Ie: """Internet Explorer""" - log.info(f'Browser set as Internet Explorer') + logger.info(f'Browser set as Internet Explorer') return self.webdriver.Ie(**kwargs) def opera(self): """Depreciated: Opera""" - log.warning(f'Opera is depreciated') + logger.warning(f'Opera is depreciated') def proxy(self, **kwargs) -> Proxy: """Proxy""" - log.info(f'Browser using proxy') + logger.info(f'Browser using proxy') return self.webdriver.Proxy(**kwargs) def phantomjs(self): """PhantomJS""" - log.warning(f'PhantomJS not supported') + logger.warning(f'PhantomJS not supported') def remote(self, **kwargs) -> Remote: """Remote""" - log.info(f'Browser using remote browser') + logger.info(f'Browser using remote browser') return self.webdriver.Remote(**kwargs) def safari(self, **kwargs) -> Safari: """Safari""" - log.info(f'Browser set as Safari') + logger.info(f'Browser set as Safari') return self.webdriver.Safari(**kwargs) def webkit_gtk(self, **kwargs) -> WebKitGTK: """WebKit GTK""" - log.info(f'Browser set as WebKitGTK') + logger.info(f'Browser set as WebKitGTK') return self.webdriver.WebKitGTK(**kwargs) def wpewebkit(self, **kwargs) -> WPEWebKit: """WPE WebKit""" - log.info(f'Browser set as WPEWebKit') + logger.info(f'Browser set as WPEWebKit') return self.webdriver.WPEWebKit(**kwargs) diff --git a/automon/integrations/seleniumWrapper/config.py b/automon/integrations/seleniumWrapper/config.py index a336261b..7485dd60 100644 --- a/automon/integrations/seleniumWrapper/config.py +++ b/automon/integrations/seleniumWrapper/config.py @@ -1,10 +1,10 @@ import selenium.webdriver -from automon.log import logger +from automon import log from automon.helpers.osWrapper import environ -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SeleniumConfig(object): @@ -32,19 +32,19 @@ def window_size(self): @property def cookies_base64(self): - log.debug(f'{len(self._cookies_base64) if self._cookies_base64 else None}') + logger.debug(f'{len(self._cookies_base64) if self._cookies_base64 else None}') return self._cookies_base64 @property def cookies_file(self): - log.info(f'{self._cookies_file}') + logger.info(f'{self._cookies_file}') return self._cookies_file def run(self): """run webdriver""" run = self.webdriver_wrapper.run() self._webdriver = self.webdriver_wrapper.webdriver - log.info(str(dict( + logger.info(str(dict( webdriver=self.webdriver ))) return run diff --git a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py index e4ab4410..bf8ee4e7 100644 --- a/automon/integrations/seleniumWrapper/config_webdriver_chrome.py +++ b/automon/integrations/seleniumWrapper/config_webdriver_chrome.py @@ -3,13 +3,13 @@ import selenium import selenium.webdriver -from automon.log import logger +from automon import log from automon.helpers.osWrapper.environ import environ from .config_window_size import set_window_size -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class ChromeWrapper(object): @@ -72,23 +72,23 @@ def window_size(self): return self._window_size def disable_certificate_verification(self): - log.warning('Certificates are not verified') + logger.warning('Certificates are not verified') self.chrome_options.add_argument('--ignore-certificate-errors') - log.debug(str(dict( + logger.debug(str(dict( add_argument='--ignore-certificate-errors' ))) return self def disable_extensions(self): self.chrome_options.add_argument("--disable-extensions") - log.debug(str(dict( + logger.debug(str(dict( add_argument=f'--disable-extensions' ))) return self def disable_infobars(self): self.chrome_options.add_argument("--disable-infobars") - log.debug(str(dict( + logger.debug(str(dict( add_argument=f'--disable-infobars' ))) return self @@ -101,28 +101,28 @@ def disable_notifications(self): "prefs", {"profile.default_content_setting_values.notifications": 2} ) - log.debug(str(dict( + logger.debug(str(dict( add_experimental_option=("prefs", {"profile.default_content_setting_values.notifications": 2}) ))) return self def disable_sandbox(self): self.chrome_options.add_argument('--no-sandbox') - log.debug(str(dict( + logger.debug(str(dict( add_argument=f'--no-sandbox' ))) return self def disable_shm(self): - log.warning('Disabled shm will use disk I/O, and will be slow') + logger.warning('Disabled shm will use disk I/O, and will be slow') self.chrome_options.add_argument('--disable-dev-shm-usage') - log.debug(str(dict( + logger.debug(str(dict( add_argument=f'--disable-dev-shm-usage' ))) return self def enable_bigshm(self): - log.warning('Big shm not yet implemented') + logger.warning('Big shm not yet implemented') return self def enable_defaults(self): @@ -131,14 +131,14 @@ def enable_defaults(self): def enable_fullscreen(self): self.chrome_options.add_argument("--start-fullscreen") - log.debug(str(dict( + logger.debug(str(dict( add_argument=f'--start-fullscreen' ))) return self def enable_headless(self): self.chrome_options.add_argument('headless') - log.debug(str(dict( + logger.debug(str(dict( add_argument='headless' ))) return self @@ -150,14 +150,14 @@ def enable_notifications(self): self.chrome_options.add_experimental_option( "prefs", {"profile.default_content_setting_values.notifications": 1} ) - log.debug(str(dict( + logger.debug(str(dict( add_experimental_option=("prefs", {"profile.default_content_setting_values.notifications": 1}) ))) return self def enable_maximized(self): self.chrome_options.add_argument('--start-maximized') - log.debug(str(dict( + logger.debug(str(dict( add_argument='--start-maximized' ))) return self @@ -172,7 +172,7 @@ def enable_translate(self, native_language: str = 'en'): value=prefs, ) - log.debug(str(dict( + logger.debug(str(dict( add_experimental_option=dict( name="prefs", value=prefs, @@ -185,7 +185,7 @@ def close(self): """ result = self.webdriver.close() - log.info(f'{result}') + logger.info(f'{result}') return result def in_docker(self): @@ -210,7 +210,7 @@ def in_headless_sandboxed(self): """Headless Chrome with sandbox enabled """ - log.warning( + logger.warning( 'Docker does not support sandbox option. ' 'Default shm size is 64m, which will cause chrome driver to crash.' ) @@ -223,7 +223,7 @@ def in_headless_sandbox_disabled(self): """Headless Chrome with sandbox disabled """ - log.warning('Default shm size is 64m, which will cause chrome driver to crash.') + logger.warning('Default shm size is 64m, which will cause chrome driver to crash.') self.enable_defaults() self.enable_headless() @@ -234,7 +234,7 @@ def in_headless_sandbox_disabled_certificate_unverified(self): """Headless Chrome with sandbox disabled with no certificate verification """ - log.warning('Default shm size is 64m, which will cause chrome driver to crash.') + logger.warning('Default shm size is 64m, which will cause chrome driver to crash.') self.enable_defaults() self.enable_headless() @@ -256,7 +256,7 @@ def in_headless_sandbox_disabled_bigshm(self): """Headless Chrome with sandbox disabled """ - log.warning('Larger shm option is not implemented') + logger.warning('Larger shm option is not implemented') self.enable_defaults() self.enable_headless() @@ -268,7 +268,7 @@ def in_remote_driver(self, host: str = '127.0.0.1', port: str = '4444', executor """Remote Selenium """ - log.info( + logger.info( f'Remote WebDriver Hub URL: http://{host}:{port}{executor_path}/static/resource/hub.html') selenium.webdriver.Remote( @@ -281,7 +281,7 @@ def in_sandbox(self): """Chrome with sandbox enabled """ - log.warning( + logger.warning( 'Docker does not support sandbox option. ' 'Default shm size is 64m, which will cause chrome driver to crash.' ) @@ -293,7 +293,7 @@ def in_sandbox_disabled(self): """Chrome with sandbox disabled """ - log.warning('Default shm size is 64m, which will cause chrome driver to crash.') + logger.warning('Default shm size is 64m, which will cause chrome driver to crash.') self.enable_defaults() self.disable_sandbox() @@ -305,7 +305,7 @@ def run(self) -> selenium.webdriver.Chrome: self._ChromeService = selenium.webdriver.ChromeService( executable_path=self.chromedriver_path ) - log.debug(str(dict( + logger.debug(str(dict( ChromeService=self.ChromeService ))) @@ -313,26 +313,27 @@ def run(self) -> selenium.webdriver.Chrome: service=self.ChromeService, options=self.chrome_options ) - log.info(f'{self}') + logger.info(f'{self}') return self.webdriver self._webdriver = selenium.webdriver.Chrome(options=self.chrome_options) - log.info(f'{self}') + logger.info(f'{self}') return self.webdriver except Exception as error: - log.error(f'{error}') + logger.error(f'{error}') + raise Exception(error) def set_chromedriver(self, chromedriver_path: str): - log.debug(f'{chromedriver_path}') + logger.debug(f'{chromedriver_path}') self._chromedriver_path = chromedriver_path self.update_paths() return self def set_locale(self, locale: str = 'en'): self.chrome_options.add_argument(f"--lang={locale}") - log.debug(str(dict( + logger.debug(str(dict( add_argument=f"--lang={locale}" ))) return self @@ -343,7 +344,7 @@ def set_locale_experimental(self, locale: str = 'en-US'): value={'intl.accept_languages': locale} ) - log.debug(str(dict( + logger.debug(str(dict( add_experimental_option=dict( name='prefs', value={'intl.accept_languages': locale} @@ -353,7 +354,7 @@ def set_locale_experimental(self, locale: str = 'en-US'): def set_user_agent(self, user_agent: str): self.chrome_options.add_argument(f"user-agent={user_agent}") - log.debug(str(dict( + logger.debug(str(dict( add_argument=f"user-agent={user_agent}" ))) return self @@ -362,7 +363,7 @@ def set_window_size(self, *args, **kwargs): self._window_size = set_window_size(*args, **kwargs) width, height = self.window_size self.webdriver.set_window_size(width=width, height=height) - log.debug(f'{width}, {height}') + logger.debug(f'{width}, {height}') return self def start(self): @@ -376,14 +377,14 @@ def stop_client(self): """ result = self.webdriver.stop_client() - log.info(f'{result}') + logger.info(f'{result}') return result def update_paths(self): if self.chromedriver_path: if self.chromedriver_path not in os.getenv('PATH'): os.environ['PATH'] = f"{os.getenv('PATH')}:{self._chromedriver_path}" - log.debug(str(dict( + logger.debug(str(dict( PATH=os.environ['PATH'] ))) @@ -392,7 +393,7 @@ def quit(self): """ result = self.webdriver.quit() - log.info(f'{result}') + logger.info(f'{result}') return result def quit_gracefully(self): @@ -404,6 +405,6 @@ def quit_gracefully(self): self.quit() self.stop_client() except Exception as error: - log.error(f'failed to gracefully quit. {error}') + logger.error(f'failed to gracefully quit. {error}') return False return True diff --git a/automon/integrations/seleniumWrapper/config_window_size.py b/automon/integrations/seleniumWrapper/config_window_size.py index 6bb576f8..62c36579 100644 --- a/automon/integrations/seleniumWrapper/config_window_size.py +++ b/automon/integrations/seleniumWrapper/config_window_size.py @@ -1,7 +1,7 @@ -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) def set_window_size(width: int = 1920, height: int = 1080, device_type: str = None) -> (int, int): @@ -51,6 +51,6 @@ def set_window_size(width: int = 1920, height: int = 1080, device_type: str = No width = 1920 height = 1080 - log.debug(f'{int(width)}, {int(height)}') + logger.debug(f'{int(width)}, {int(height)}') return int(width), int(height) diff --git a/automon/integrations/seleniumWrapper/user_agents.py b/automon/integrations/seleniumWrapper/user_agents.py index 0de723dc..b9939011 100644 --- a/automon/integrations/seleniumWrapper/user_agents.py +++ b/automon/integrations/seleniumWrapper/user_agents.py @@ -1,9 +1,9 @@ import random -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SeleniumUserAgentBuilder: diff --git a/automon/integrations/sentryio/tests/test_sentryio.py b/automon/integrations/sentryio/tests/test_sentryio.py index 2a03a2e4..a53c0c3e 100644 --- a/automon/integrations/sentryio/tests/test_sentryio.py +++ b/automon/integrations/sentryio/tests/test_sentryio.py @@ -2,26 +2,19 @@ from datetime import datetime +from automon import log from automon.integrations.sentryio import SentryClient from automon.integrations.geoip import Geoip -from automon.log import Logging - class SentryClientTest(unittest.TestCase): def test_sentry(self): s = SentryClient() - l = Logging() if s.isConnected(): self.assertTrue(s.capture_exception(Exception(f'test capture_exception'))) self.assertTrue(s.capture_message(f'test capture_message')) # self.assertTrue(s.capture_event('test capture_event', 'warning')) - self.assertTrue(l.info(f'test log info')) - self.assertTrue(l.debug(f'test log debug')) - self.assertTrue(l.warning(f'test log warning')) - self.assertTrue(l.error(f'test log error')) - self.assertTrue(l.critical(f'test log critical')) if __name__ == '__main__': diff --git a/automon/integrations/sentryio/tests/test_sentryio_callback.py b/automon/integrations/sentryio/tests/test_sentryio_callback.py index 3c1038c7..56a81c88 100644 --- a/automon/integrations/sentryio/tests/test_sentryio_callback.py +++ b/automon/integrations/sentryio/tests/test_sentryio_callback.py @@ -1,21 +1,21 @@ import unittest -from automon.log import Logging +from automon import log from automon.integrations.sentryio.client import SentryClient +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) + class CallbackTest(unittest.TestCase): sentry = SentryClient() - log = Logging(name=__name__, level=Logging.DEBUG) - log.callbacks.append(sentry) def test_sentry(self): - self.assertTrue(self.log) - self.assertTrue(self.log.info('test')) - self.assertTrue(self.log.debug('test')) - self.assertTrue(self.log.error('test')) - self.assertTrue(self.log.warning('test')) - self.assertTrue(self.log.critical('test')) + self.assertIsNone(logger.info('test')) + self.assertIsNone(logger.debug('test')) + self.assertIsNone(logger.error('test')) + self.assertIsNone(logger.warning('test')) + self.assertIsNone(logger.critical('test')) if __name__ == '__main__': diff --git a/automon/integrations/shodan/__init__.py b/automon/integrations/shodan/__init__.py index 7b96ac2a..a139f82e 100644 --- a/automon/integrations/shodan/__init__.py +++ b/automon/integrations/shodan/__init__.py @@ -1,19 +1,19 @@ import os -from automon.log import Logging +from automon import log -log = Logging(__name__, Logging.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class ShodanConfig: token = os.getenv('SHODAN_API') def __init__(self): - self._log = Logging(ShodanConfig.__name__, Logging.ERROR) self.token = os.getenv('SHODAN_API') if not self.token: - self._log.error(f'Missing SHODAN_API') + logger.error(f'Missing SHODAN_API') class Shodan: diff --git a/automon/integrations/slackWrapper/client.py b/automon/integrations/slackWrapper/client.py index f08e0716..e742f477 100644 --- a/automon/integrations/slackWrapper/client.py +++ b/automon/integrations/slackWrapper/client.py @@ -1,14 +1,14 @@ import os import slack -from automon.log import logger +from automon import log from .config import SlackConfig from .bots import BotInfo from .error import SlackError -log = logger.logging.getLogger(__name__) -log.setLevel(logger.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class SlackClient(SlackConfig): @@ -44,11 +44,11 @@ def _get_bot_info(self): try: name = BotInfo(self.client.bots_info()).name - log.debug(f'Bot name: {name}') + logger.debug(f'Bot name: {name}') return name except Exception as e: error = SlackError(e) - log.error( + logger.error( f'''[{self._get_bot_info.__name__}]\tCouldn't get bot name, missing permission: {error.needed}''') return '' @@ -63,7 +63,7 @@ def chat_postMessage(self, channel: str, text: str) -> slack.WebClient.chat_post return SyntaxError msg = f'{channel} @{self.username}: {text}' - log.debug(msg) + logger.debug(msg) try: response = self.client.chat_postMessage( @@ -72,7 +72,7 @@ def chat_postMessage(self, channel: str, text: str) -> slack.WebClient.chat_post assert response["ok"] return response except Exception as e: - log.error(e) + logger.error(e) return False @@ -96,8 +96,8 @@ def files_upload(self, file, filename=None): # check if file exists if not os.path.isfile(file): - log.error(f'File not found: {file}') - log.error(f'Working dir: {os.getcwd()}') + logger.error(f'File not found: {file}') + logger.error(f'Working dir: {os.getcwd()}') return False # get filename @@ -116,6 +116,6 @@ def files_upload(self, file, filename=None): file=file, filename=filename, title=title, username=self.username, channels=self.channel) assert response["ok"] - log.debug(f'File uploaded: {file} ({file_size}B) ({self.username}') + logger.debug(f'File uploaded: {file} ({file_size}B) ({self.username}') return response diff --git a/automon/integrations/slackWrapper/clientAsync.py b/automon/integrations/slackWrapper/clientAsync.py index b0063ac3..cdc89668 100644 --- a/automon/integrations/slackWrapper/clientAsync.py +++ b/automon/integrations/slackWrapper/clientAsync.py @@ -3,15 +3,15 @@ import random import asyncio -from automon.log import logger +from automon import log from automon.helpers.nest_asyncioWrapper import AsyncStarter from .config import ConfigSlack from .bots import BotInfo from .error import SlackError -log = logger.logging.getLogger(__name__) -log.setLevel(logger.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class SlackAsyncClient(ConfigSlack): @@ -50,11 +50,11 @@ def _get_bot_info(self): try: name = BotInfo(self.client.bots_info()).name - log.debug(f'Bot name: {name}') + logger.debug(f'Bot name: {name}') return name except Exception as e: error = SlackError(e) - log.error( + logger.error( f'''[{self._get_bot_info.__name__}]\tCouldn't get bot name, missing permission: {error.needed}''') return '' @@ -98,8 +98,8 @@ def files_upload(self, file, filename=None): # check if file exists if not os.path.isfile(file): - log.error(f'File not found: {file}') - log.error(f'Working dir: {os.getcwd()}') + logger.error(f'File not found: {file}') + logger.error(f'Working dir: {os.getcwd()}') return False # get filename @@ -119,7 +119,7 @@ def files_upload(self, file, filename=None): assert response["ok"] - log.debug(f'File uploaded: {file} ({file_size}B) ({self.username}') + logger.debug(f'File uploaded: {file} ({file_size}B) ({self.username}') return response @@ -138,7 +138,7 @@ async def _consumer(self): while self.connected: try: - log.debug(msg) + logger.debug(msg) response = self.client.chat_postMessage( text=text, channel=channel, username=self.username, icon_emoji=self.icon_emoji, icon_url=self.icon_url) @@ -149,10 +149,10 @@ async def _consumer(self): await asyncio.sleep(random.choice(range(2))) else: sleep = random.choice(range(4)) - log.debug(f'sleeping {sleep}, queue size is {self.queue.qsize()}') + logger.debug(f'sleeping {sleep}, queue size is {self.queue.qsize()}') await asyncio.sleep(sleep) - log.debug(f'Burst: {burst}, Retry: {retry}, Queue {self.queue.qsize()}') + logger.debug(f'Burst: {burst}, Retry: {retry}, Queue {self.queue.qsize()}') burst += 1 retry = 0 @@ -165,6 +165,6 @@ async def _consumer(self): retry += 1 burst_max = burst error = SlackError(e) - log.error( + logger.error( f'{self._consumer.__name__}\t{error.error}\t{msg}\tRetry: {retry}, Burst max: {burst_max}') burst = 0 diff --git a/automon/integrations/slackWrapper/config.py b/automon/integrations/slackWrapper/config.py index 9f4400f8..6664ec13 100644 --- a/automon/integrations/slackWrapper/config.py +++ b/automon/integrations/slackWrapper/config.py @@ -1,9 +1,9 @@ import os -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class SlackConfig(object): @@ -32,7 +32,7 @@ def __init__(self, username=None, self.SLACK_TEST_CHANNEL = os.getenv('SLACK_TEST_CHANNEL') or '' if not self.token: - log.warning(f'missing SLACK_TOKEN') + logger.warning(f'missing SLACK_TOKEN') class ConfigSlack: @@ -53,7 +53,7 @@ class ConfigSlack: SLACK_TEST_CHANNEL = os.getenv('SLACK_TEST_CHANNEL') or '' if not slack_token: - log.warning(f'missing SLACK_TOKEN') + logger.warning(f'missing SLACK_TOKEN') def __init__(self, slack_name: str = ''): self.slack_name = os.getenv('SLACK_USER') or slack_name or '' diff --git a/automon/integrations/slackWrapper/error.py b/automon/integrations/slackWrapper/error.py index fe12ccf1..4ec41488 100644 --- a/automon/integrations/slackWrapper/error.py +++ b/automon/integrations/slackWrapper/error.py @@ -1,7 +1,7 @@ -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.ERROR) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class SlackError: @@ -50,7 +50,7 @@ def error(self): if self._reason: return self.strerror - log.warning(f'{NotImplemented}') + logger.warning(f'{NotImplemented}') return f'{self._error}' def needed(self): @@ -60,7 +60,7 @@ def needed(self): if self._reason: return self.strerror - log.warning(f'{NotImplemented}') + logger.warning(f'{NotImplemented}') return f'{self._error}' def __repr__(self): @@ -70,7 +70,7 @@ def __repr__(self): if self._reason: return f'{self.strerror}' - log.warning(f'{NotImplemented}') + logger.warning(f'{NotImplemented}') return f'{self._error}' def __str__(self): diff --git a/automon/integrations/slackWrapper/slack_logger.py b/automon/integrations/slackWrapper/slack_logger.py index ed689d0d..91b5c5ff 100644 --- a/automon/integrations/slackWrapper/slack_logger.py +++ b/automon/integrations/slackWrapper/slack_logger.py @@ -8,7 +8,11 @@ from automon.integrations.slackWrapper.client import SlackClient from automon.integrations.slackWrapper.slack_formatting import Emoji, Chat, Format -from automon.log import Logging, INFO, ERROR, WARN, CRITICAL, DEBUG +from automon import log +from automon.log import INFO, ERROR, WARN, CRITICAL, DEBUG, TEST + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class AsyncSlackLogging(SlackClient): @@ -156,7 +160,7 @@ async def _producer(self): await self._error(channel, text) elif level is CRITICAL: await self._critical(channel, text) - elif level == Logging.TEST: + elif level == TEST: await self._test(channel, text) if self._stop: @@ -199,7 +203,7 @@ def critical(self, msg: str = None) -> asyncio.tasks: asyncio.run(self._put_queue(CRITICAL, self.critical_channel, msg)) def test(self, msg: str or list or dict or tuple) -> asyncio.tasks: - asyncio.run(self._put_queue(Logging.TEST, self.test_channel, msg)) + asyncio.run(self._put_queue(TEST, self.test_channel, msg)) async def _warn(self, channel: str, msg: str or list or dict or tuple): self.set_slack_config(WARN) @@ -249,7 +253,7 @@ async def _critical(self, channel: str, msg: str or list or dict or tuple = None # self.set_slack_config() async def _test(self, channel: str, msg: str): - self.set_slack_config(Logging.TEST) + self.set_slack_config(TEST) await self.slack.chat_postMessage(channel, self._msg(msg)) # self.set_slack_config() @@ -287,7 +291,7 @@ def set_slack_config(self, level=None): self.slack.icon_url = self._critical_url self.slack.channel = self.critical_channel - elif level == Logging.TEST: + elif level == TEST: self.slack.username = f'{self.slack.username}{self._test_suffix}' self.slack.icon_emoji = self._test_icon self.slack.icon_url = self._test_url diff --git a/automon/integrations/snmp/generate_maps.py b/automon/integrations/snmp/generate_maps.py index 2826d43d..74e6d223 100644 --- a/automon/integrations/snmp/generate_maps.py +++ b/automon/integrations/snmp/generate_maps.py @@ -7,9 +7,10 @@ import subprocess from automon.integrations.slackWrapper.slack_formatting import Chat -from automon.log import Logging +from automon import log -log = Logging(__name__, level=Logging.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) class SmidumpFormat: @@ -92,10 +93,10 @@ def __init__(self, device: str, path: str): self.filename = os.path.split(path)[-1] if os.path.isfile(path): - log.debug(f'found MIB {path}') + logger.debug(f'found MIB {path}') else: msg = f'{MibFile} {path} not found' - log.error(msg) + logger.error(msg) raise def __str__(self): @@ -164,14 +165,14 @@ def __init__(self): def check(self) -> True or False: if os.system(f'which {self.requires}') == 0: - log.debug(f'{self.requires} OK') + logger.debug(f'{self.requires} OK') return True else: msg = ( f'missing {self.requires}, ' f'please install {self.requires}, ' f'`apt install smitools`') - log.error(msg) + logger.error(msg) return False @@ -317,7 +318,7 @@ def __init__(self, path: str, device: str = None): mib = MibFile(device, path_to_mib_file) self.mibs.append(mib) else: - log.error(f'not found MIB {path_to_mib_file}') + logger.error(f'not found MIB {path_to_mib_file}') def _mib_generator(self) -> MibFile: for mib in self.mibs: @@ -380,7 +381,7 @@ def _list_walk(self, object: list) -> list: if isinstance(item, dict): tuple_oids.extend(self._dict_walk(item)) else: - log.critical(f'{self._list_walk} {NotImplemented}') + logger.critical(f'{self._list_walk} {NotImplemented}') return tuple_oids @@ -399,7 +400,7 @@ def _dict_walk(self, object: dict) -> list: elif isinstance(value, list): tuple_oids.extend(self._list_walk(value)) elif not isinstance(value, str) and value is not None: - log.critical(f'{self._dict_walk} {NotImplemented}') + logger.critical(f'{self._dict_walk} {NotImplemented}') oid_and_description = self._create_oid_tuple(object) @@ -417,7 +418,7 @@ def _xml_walk(self, xml: dict) -> list: elif isinstance(xml, list): tuple_oids.extend(self._list_walk(xml)) else: - log.critical(f'{self._xml_walk} {NotImplemented}') + logger.critical(f'{self._xml_walk} {NotImplemented}') return tuple_oids @@ -430,7 +431,7 @@ def generate_prometheus_config(self) -> open: device = mibmap.device list_of_oid_tuples = self._xml_walk(xml) - # log.debug(list_of_oid_tuples) + # logger.debug(list_of_oid_tuples) if not list_of_oid_tuples: continue @@ -470,7 +471,7 @@ def smidump_run(cmd: str, create_file: bool = False) -> None: data_err = run.stderr.decode() if data_err: - log.error(f'{data_err}') + logger.error(f'{data_err}') map_ = MibMap(mib, data) self.maps.append(map_) @@ -478,7 +479,7 @@ def smidump_run(cmd: str, create_file: bool = False) -> None: if create_file: with open(map_.path, 'wb') as f: f.write(map_._data) - log.debug(f'Wrote {map_.path} ({map_.len} B)') + logger.debug(f'Wrote {map_.path} ({map_.len} B)') return True @@ -502,7 +503,7 @@ def smidump_run(cmd: str, create_file: bool = False) -> None: cmd = f'smidump {opts} {preload} {mib}' - log.debug(cmd) + logger.debug(cmd) # smidump_run(cmd) @@ -516,7 +517,7 @@ def smidump_run(cmd: str, create_file: bool = False) -> None: cmd = f'smidump {opts} {preload} {mib}' # cmd = f'{self.smidump} {self.opts} {mib}' - log.debug(cmd) + logger.debug(cmd) # smidump_run(cmd) @@ -524,4 +525,4 @@ def smidump_run(cmd: str, create_file: bool = False) -> None: # m.generate_map(SmidumpFormat().xml) # yml = m.generate_prometheus_config() # -# log.info('Done') +# logger.info('Done') diff --git a/automon/integrations/splunk/client.py b/automon/integrations/splunk/client.py index fc31ad65..ce4226ed 100644 --- a/automon/integrations/splunk/client.py +++ b/automon/integrations/splunk/client.py @@ -4,11 +4,12 @@ import splunklib.results import splunklib.client -from automon.log import Logging +from automon import log from automon.integrations.splunk.config import SplunkConfig from automon.integrations.splunk.helpers import Job, Application -log = Logging(name='SplunkClient', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SplunkRestClient: @@ -57,7 +58,7 @@ def wrapped(self, *args, **kwargs): self.client return func(self, *args, **kwargs) except Exception as e: - log.error(f'not connected. {e}', enable_traceback=False) + logger.error(f'not connected. {e}') return False return wrapped diff --git a/automon/integrations/splunk/config.py b/automon/integrations/splunk/config.py index 592ba48e..6c63543f 100644 --- a/automon/integrations/splunk/config.py +++ b/automon/integrations/splunk/config.py @@ -1,9 +1,10 @@ import splunklib.binding as binding -from automon.log import Logging +from automon import log from automon.helpers import environ -log = Logging(name='SplunkConfig', level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SplunkConfig: diff --git a/automon/integrations/splunk/helpers.py b/automon/integrations/splunk/helpers.py index 37d6f1ad..85671105 100644 --- a/automon/integrations/splunk/helpers.py +++ b/automon/integrations/splunk/helpers.py @@ -1,14 +1,14 @@ import splunklib.client as client -from automon.log import Logging +from automon import log -log = Logging(name=__name__, level=Logging.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class JobError: def __init__(self, job: client.Job): - self._log = Logging(JobError.__name__, level=Logging.DEBUG) self._job = job self.messages = job.content['messages'] @@ -16,16 +16,15 @@ def __init__(self, job: client.Job): self.fatal = self.messages['fatal'] self.error = self.messages['error'] - self._log.critical(self.fatal) - self._log.error(self.error) + logger.critical(self.fatal) + logger.error(self.error) except Exception as e: - self._log.error(e) + logger.error(e) class Job: def __init__(self, job: client.Job): - self._log = Logging(Job.__name__, level=Logging.DEBUG) self._job = job try: @@ -47,7 +46,7 @@ def __init__(self, job: client.Job): self.error = JobError(job) except Exception as e: - self._log.error(e) + logger.error(e) def is_ready(self): return self._job.is_ready() diff --git a/automon/integrations/splunk_soar/action_run.py b/automon/integrations/splunk_soar/action_run.py index 11637de8..bab47688 100644 --- a/automon/integrations/splunk_soar/action_run.py +++ b/automon/integrations/splunk_soar/action_run.py @@ -1,9 +1,9 @@ -from automon.log import logger +from automon import log from .datatypes import AbstractDataType -log = logger.logging.getLogger(__name__) -log.setLevel(logger.CRITICAL) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.CRITICAL) class ActionRun(AbstractDataType): diff --git a/automon/integrations/splunk_soar/artifact.py b/automon/integrations/splunk_soar/artifact.py index 7dae9897..a8ba7fcb 100644 --- a/automon/integrations/splunk_soar/artifact.py +++ b/automon/integrations/splunk_soar/artifact.py @@ -1,9 +1,9 @@ -from automon.log import logger +from automon import log from .datatypes import AbstractDataType -log = logger.logging.getLogger(__name__) -log.setLevel(logger.CRITICAL) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.CRITICAL) class Artifact(AbstractDataType): diff --git a/automon/integrations/splunk_soar/asset.py b/automon/integrations/splunk_soar/asset.py index c69ea4d9..8cb0ad1d 100644 --- a/automon/integrations/splunk_soar/asset.py +++ b/automon/integrations/splunk_soar/asset.py @@ -1,9 +1,9 @@ -from automon.log import logger +from automon import log from .datatypes import AbstractDataType -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Asset(AbstractDataType): diff --git a/automon/integrations/splunk_soar/client.py b/automon/integrations/splunk_soar/client.py index 228263c0..1e0195d0 100644 --- a/automon/integrations/splunk_soar/client.py +++ b/automon/integrations/splunk_soar/client.py @@ -4,7 +4,7 @@ from typing import Optional -from automon.log import logger +from automon import log from automon.integrations.requestsWrapper import Requests from .action_run import ActionRun @@ -28,10 +28,10 @@ VaultResponse ) -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) -logger.logging.getLogger('RequestsClient').setLevel(logger.DEBUG) +log.logging.getLogger('RequestsClient').setLevel(log.DEBUG) class SplunkSoarClient: @@ -97,10 +97,10 @@ def close_container(self, container_id: int, **kwargs) -> Optional[CloseContaine if self._post(Urls.container(identifier=container_id, **kwargs), data=json.dumps(data)): if self.client.results.status_code == 200: response = CloseContainerResponse(self._content_dict()) - log.info(f'container closed: {response}') + logger.info(f'container closed: {response}') return response - log.error(msg=f'close failed. {self.client.to_dict()}') + logger.error(msg=f'close failed. {self.client.to_dict()}') @_is_connected def cancel_playbook_run( @@ -114,10 +114,10 @@ def cancel_playbook_run( if self._post(Urls.playbook_run(identifier=playbook_run_id, **kwargs), data=data): if self.client.results.status_code == 200: response = CancelPlaybookResponse(self._content_dict()) - log.info(f'cancel playbook run: {response}') + logger.info(f'cancel playbook run: {response}') return response - log.error(f'cancel failed: {playbook_run_id} {self.client.to_dict()}') + logger.error(f'cancel failed: {playbook_run_id} {self.client.to_dict()}') @_is_connected def create_artifact( @@ -165,14 +165,14 @@ def create_artifact( if self._post(Urls.artifact(*args, **kwargs), data=artifact.to_json()): if self.client.results.status_code == 200: id = self.client.to_dict()['id'] - log.info(f'artifact created. {artifact} {self.client.to_dict()}') + logger.info(f'artifact created. {artifact} {self.client.to_dict()}') return self.get_artifact(artifact_id=id) else: existing_artifact_id = self.client.to_dict()['existing_artifact_id'] - log.info(f'artifact exists. {artifact} {self.client.to_dict()}') + logger.info(f'artifact exists. {artifact} {self.client.to_dict()}') return self.get_artifact(artifact_id=existing_artifact_id) - log.error(f'create artifact. {self.client.to_dict()}') + logger.error(f'create artifact. {self.client.to_dict()}') return False @_is_connected @@ -239,9 +239,9 @@ def create_container( if self._post(Urls.container(*args, **kwargs), data=container.to_json()): if self.client.results.status_code == 200: response = CreateContainerResponse(self.client.to_dict()) - log.info(f'container created. {container} {response}') + logger.info(f'container created. {container} {response}') return response - log.error(f'create container. {self.client.to_dict()}') + logger.error(f'create container. {self.client.to_dict()}') return False @staticmethod @@ -273,10 +273,10 @@ def create_container_attachment( if self._post(Urls.container_attachment(**kwargs), data=data): response = CreateContainerAttachmentResponse(self.client.to_dict()) - log.info(f'create attachment: {response}') + logger.info(f'create attachment: {response}') return response - log.error(f'create attachment failed.') + logger.error(f'create attachment failed.') @_is_connected def create_vault( @@ -298,10 +298,10 @@ def create_vault( if self._post(Urls.vault_add(identifire=data.id, **kwargs), data=data.to_json()): response = Vault(self._content_dict()) - log.info(msg=f'add vault: {response}') + logger.info(msg=f'add vault: {response}') return response - log.error(msg=f'add vault failed.') + logger.error(msg=f'add vault failed.') @_is_connected def delete_container(self, container_id, *args, **kwargs): @@ -310,22 +310,22 @@ def delete_container(self, container_id, *args, **kwargs): if self._delete(Urls.container(identifier=container_id, *args, **kwargs)): if self.client.results.status_code == 200: - log.info(f'container deleted: {container_id}') + logger.info(f'container deleted: {container_id}') return True - log.error(f'delete container: {container_id}. {self.client.to_dict()}') + logger.error(f'delete container: {container_id}. {self.client.to_dict()}') return False def is_connected(self) -> bool: """check if client can connect""" if self.config.is_ready: if self._get(Urls.container(page_size=1)): - log.info(f'client connected ' - f'{self.config.host} ' - f'[{self.client.results.status_code}] ') + logger.info(f'client connected ' + f'{self.config.host} ' + f'[{self.client.results.status_code}] ') return True else: - log.warning(f'client not connected') + logger.warning(f'client not connected') return False @_is_connected @@ -349,40 +349,40 @@ def generic_delete(self, api: str, **kwargs) -> Optional[GenericResponse]: """Make generic delete calls""" if self._delete(Urls.generic(api=api, **kwargs)): response = GenericResponse(self._content_dict()) - log.info(f'generic delete {api}: {response}') + logger.info(f'generic delete {api}: {response}') return response - log.error(f'failed generic delete {api}') + logger.error(f'failed generic delete {api}') @_is_connected def generic_get(self, api: str, **kwargs) -> Optional[GenericResponse]: """Make generic get calls""" if self._get(Urls.generic(api=api, **kwargs)): response = GenericResponse(self._content_dict()) - log.info(f'generic get {api}: {response}') + logger.info(f'generic get {api}: {response}') return response - log.error(f'failed generic get {api}') + logger.error(f'failed generic get {api}') @_is_connected def generic_post(self, api: str, data: dict, **kwargs) -> Optional[GenericResponse]: """Make generic post calls""" if self._post(Urls.generic(api=api, **kwargs), data=data): response = GenericResponse(self._content_dict()) - log.info(f'generic post {api}: {response}') + logger.info(f'generic post {api}: {response}') return response - log.error(f'failed generic post {api}') + logger.error(f'failed generic post {api}') @_is_connected def get_action_run(self, action_run_id: int = None, **kwargs) -> ActionRun: """Get action run""" if self._get(Urls.action_run(identifier=action_run_id, **kwargs)): action_run = ActionRun(self._content_dict()) - log.info(f'get action run: {action_run}') + logger.info(f'get action run: {action_run}') return action_run - log.error(f'action run not found: {action_run_id}') + logger.error(f'action run not found: {action_run_id}') return ActionRun() @_is_connected @@ -390,10 +390,10 @@ def get_artifact(self, artifact_id: int = None, **kwargs) -> Artifact: """Get artifact""" if self._get(Urls.artifact(identifier=artifact_id, **kwargs)): artifact = Artifact(self._content_dict()) - log.info(f'get artifact: {artifact}') + logger.info(f'get artifact: {artifact}') return artifact - log.error(f'artifact not found: {artifact_id}') + logger.error(f'artifact not found: {artifact_id}') return Artifact() @_is_connected @@ -401,10 +401,10 @@ def get_container(self, container_id: int = None, **kwargs) -> Container: """Get container""" if self._get(Urls.container(identifier=container_id, **kwargs)): container = Container(self._content_dict()) - log.info(f'get container: {container}') + logger.info(f'get container: {container}') return container - log.error(f'container not found: {container_id}') + logger.error(f'container not found: {container_id}') return Container() @_is_connected @@ -414,13 +414,13 @@ def get_playbook_run(self, playbook_run_id: str, **kwargs) -> Optional[PlaybookR response = PlaybookRun(self._content_dict()) if response.status != 'failed': - log.info(f'playbook run: {response}') + logger.info(f'playbook run: {response}') return response - log.error(f'playbook run failed: {response.message_to_dict}') + logger.error(f'playbook run failed: {response.message_to_dict}') return response - log.error(f'playbook failed: {self.client.errors}') + logger.error(f'playbook failed: {self.client.errors}') @_is_connected def get_vault(self, vault_id: int, **kwargs) -> Optional[Vault]: @@ -428,17 +428,17 @@ def get_vault(self, vault_id: int, **kwargs) -> Optional[Vault]: if self._get(Urls.vault(identifier=vault_id, **kwargs)): if self.client.results.status_code == 200: response = Vault(self._content_dict()) - log.info(msg=f'get vault: {response}') + logger.info(msg=f'get vault: {response}') return response - log.error(msg=f'get vault failed: {self.client.to_dict()}') + logger.error(msg=f'get vault failed: {self.client.to_dict()}') @_is_connected def list_artifact(self, **kwargs) -> Response: """list artifacts""" if self._get(Urls.artifact(**kwargs)): response = Response(self._content_dict()) - log.info(f'list artifacts: {response.count}') + logger.info(f'list artifacts: {response.count}') return response return Response() @@ -469,7 +469,7 @@ def list_app_run( self.app_run = AppRunResponse(self._content_dict()) response = AppRunResponse(self._content_dict()) response.data = [AppRunResults(x) for x in response.data] - log.info(f'list app runs, page: {page} page_size: {page_size}, {response.summary()}') + logger.info(f'list app runs, page: {page} page_size: {page_size}, {response.summary()}') self.app_run = response return response return False @@ -489,26 +489,26 @@ def list_app_run_generator( if response.data: app_runs = response.data num_pages = response.num_pages - log.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') if page >= num_pages or page >= max_pages: - log.info(f'list app runs finished') + logger.info(f'list app runs finished') return True yield app_runs page += 1 elif response.data == []: - log.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') - log.info(f'list app runs finished. {response}') + logger.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'list app runs finished. {response}') return True elif response.data is None: - log.error(f'list app runs failed') + logger.error(f'list app runs failed') return False else: - log.info(f'no app runs. {response}') + logger.info(f'no app runs. {response}') return True return False @@ -537,7 +537,7 @@ def list_artifacts( """list artifacts""" if self._get(Urls.artifact(page=page, page_size=page_size, **kwargs)): response = Response(self._content()) - log.info(f'list artifacts: {len(response.data)}') + logger.info(f'list artifacts: {len(response.data)}') return response return Response() @@ -556,26 +556,26 @@ def list_artifact_generator( if response.data: containers = [Container(x) for x in response.data] num_pages = response.num_pages - log.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') if page >= num_pages or page >= max_pages: - log.info(f'list container finished') + logger.info(f'list container finished') return True yield containers page += 1 elif response.data == []: - log.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') - log.info(f'list container finished. {response}') + logger.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'list container finished. {response}') return True elif response.data is None: - log.error(f'list container failed') + logger.error(f'list container failed') return False else: - log.info(f'no containers. {response}') + logger.info(f'no containers. {response}') return True return False @@ -585,7 +585,7 @@ def list_asset(self, **kwargs) -> Response: """list asset""" if self._get(Urls.asset(**kwargs)): response = Response(self._content_dict()) - log.info(f'list assets: {len(response.data)}') + logger.info(f'list assets: {len(response.data)}') return response return Response() @@ -600,9 +600,9 @@ def list_containers( url = Urls.container(page=page, page_size=page_size, *args, **kwargs) if self._get(url): response = Response(self._content_dict()) - log.info(f'list containers: {len(response.data)}') + logger.info(f'list containers: {len(response.data)}') return response - log.error(f'no containers') + logger.error(f'no containers') return Response() @_is_connected @@ -629,26 +629,26 @@ def list_containers_generator( if response.data: containers = [Container(x) for x in response.data] num_pages = response.num_pages - log.info(f'container page {page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'container page {page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') if page > num_pages: - log.info(f'list container finished') + logger.info(f'list container finished') break yield containers page += 1 elif response.data == []: - log.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') - log.info(f'list container finished. {response}') + logger.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'list container finished. {response}') break elif response.data is None: - log.error(f'list container failed') + logger.error(f'list container failed') break else: - log.info(f'no containers. {response}') + logger.info(f'no containers. {response}') break return [] @@ -665,10 +665,10 @@ def list_vault(self, **kwargs) -> Optional[VaultResponse]: """List vault""" if self._get(Urls.vault(**kwargs)): response = VaultResponse(self._content_dict()) - log.info(msg=f'list vault: {response}') + logger.info(msg=f'list vault: {response}') return response - log.error(msg=f'list vault failed.') + logger.error(msg=f'list vault failed.') @_is_connected def list_vault_generator( @@ -692,26 +692,26 @@ def list_vault_generator( if response.data: vaults = [Vault(x) for x in response.data] num_pages = response.num_pages - log.info(f'vault page {page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'vault page {page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') if page > num_pages: - log.info(f'list vault finished') + logger.info(f'list vault finished') break yield vaults page += 1 elif response.data == []: - log.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') - log.info(f'list vault finished. {response}') + logger.info(f'{page}/{num_pages} ({round(page / num_pages * 100, 2)}%)') + logger.info(f'list vault finished. {response}') break elif response.data is None: - log.error(f'list vault failed') + logger.error(f'list vault failed') break else: - log.info(f'no vaults. {response}') + logger.info(f'no vaults. {response}') break return [] @@ -732,10 +732,10 @@ def update_playbook( if self._post(Urls.playbook(identifier=playbook_id, **kwargs), data=data): if self.client.results.status_code == 200: response = UpdatePlaybookResponse(self._content_dict()) - log.info(f'update playbook: {data}') + logger.info(f'update playbook: {data}') return response - log.error(f'update failed: {self.client.to_dict()}') + logger.error(f'update failed: {self.client.to_dict()}') @_is_connected def run_playbook( @@ -756,7 +756,7 @@ def run_playbook( if self._post(Urls.playbook_run(**kwargs), data=data): if self.client.results.status_code == 200: response = RunPlaybookResponse(self._content_dict()) - log.info(f'run playbook: {data}') + logger.info(f'run playbook: {data}') return response - log.error(f'run failed: {self.client.to_dict()}') + logger.error(f'run failed: {self.client.to_dict()}') diff --git a/automon/integrations/splunk_soar/config.py b/automon/integrations/splunk_soar/config.py index 5c1399d1..67f7be1c 100644 --- a/automon/integrations/splunk_soar/config.py +++ b/automon/integrations/splunk_soar/config.py @@ -1,8 +1,8 @@ -from automon.log import logger +from automon import log from automon.helpers.osWrapper import environ -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SplunkSoarConfig: @@ -21,7 +21,7 @@ def __init__(self, host: str = None, self.headers = {'ph-auth-token': self.auth_token} if not self.host: - log.warning(f'missing SPLUNK_SOAR_HOST') + logger.warning(f'missing SPLUNK_SOAR_HOST') def __repr__(self): return f'{self.__dict__}' @@ -30,5 +30,5 @@ def __repr__(self): def is_ready(self) -> bool: if self.host: return True - log.warning(f'bad config') + logger.warning(f'bad config') return False diff --git a/automon/integrations/splunk_soar/container.py b/automon/integrations/splunk_soar/container.py index f915e83e..5ef43d93 100644 --- a/automon/integrations/splunk_soar/container.py +++ b/automon/integrations/splunk_soar/container.py @@ -1,11 +1,11 @@ import datetime -from automon.log import logger +from automon import log from .datatypes import AbstractDataType -log = logger.logging.getLogger(__name__) -log.setLevel(logger.CRITICAL) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.CRITICAL) class Container(AbstractDataType): diff --git a/automon/integrations/splunk_soar/datatypes.py b/automon/integrations/splunk_soar/datatypes.py index 95f8e253..09807f3e 100644 --- a/automon/integrations/splunk_soar/datatypes.py +++ b/automon/integrations/splunk_soar/datatypes.py @@ -3,10 +3,10 @@ from dateutil import parser -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class AbstractDataType: diff --git a/automon/integrations/splunk_soar/responses.py b/automon/integrations/splunk_soar/responses.py index 06a1a1bb..4f4fa0e1 100644 --- a/automon/integrations/splunk_soar/responses.py +++ b/automon/integrations/splunk_soar/responses.py @@ -3,13 +3,13 @@ from dateutil import parser from typing import Optional -from automon.log import logger +from automon import log from .container import Container from .vault import Vault -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class GeneralResponse: @@ -134,7 +134,7 @@ def message_to_dict(self) -> Optional[dict]: try: return json.loads(self.message) except Exception as e: - log.warning(f'message is not json. {e}') + logger.warning(f'message is not json. {e}') @property def playbook_name(self): diff --git a/automon/integrations/splunk_soar/rest/urls.py b/automon/integrations/splunk_soar/rest/urls.py index 73008685..94ac6e3c 100644 --- a/automon/integrations/splunk_soar/rest/urls.py +++ b/automon/integrations/splunk_soar/rest/urls.py @@ -1,10 +1,10 @@ -from automon.log import logger +from automon import log from ..config import SplunkSoarConfig config = SplunkSoarConfig() -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class Urls: diff --git a/automon/integrations/splunk_soar/rules.py b/automon/integrations/splunk_soar/rules.py index 2516b360..93a8cd9c 100644 --- a/automon/integrations/splunk_soar/rules.py +++ b/automon/integrations/splunk_soar/rules.py @@ -18,12 +18,12 @@ def playbook(playbook_to_import: str, container: dict = container(), """Mock function""" log = logging.getLogger(f'{__name__}.{playbook.__name__}') - # log.info(f'playbook_to_import: str, container: dict = {}, name: str, callback: object') + # logger.info(f'playbook_to_import: str, container: dict = {}, name: str, callback: object') - log.debug(f'playbook_to_import: {playbook_to_import}') - log.debug(f'container: {container}') - log.debug(f'name: {name}') - log.debug(f'callback: {callback}') + logger.debug(f'playbook_to_import: {playbook_to_import}') + logger.debug(f'container: {container}') + logger.debug(f'name: {name}') + logger.debug(f'callback: {callback}') return import_playbook(playbook_to_import) @@ -32,8 +32,8 @@ def get_run_data(key: str, **kwargs): """Mock function""" log = logging.getLogger(f'{__name__}.{get_run_data.__name__}') - # log.info(f'key: {key}') - log.debug(f'key: {key}') + # logger.info(f'key: {key}') + logger.debug(f'key: {key}') return key @@ -42,11 +42,11 @@ def condition(container: dict = container(), conditions: list = [], name: str = """Mock function""" log = logging.getLogger(f'{__name__}.{condition.__name__}') - log.debug(f'{help(condition)}') + logger.debug(f'{help(condition)}') - log.debug(f'container: {container}') - log.debug(f'conditions: {conditions}') - log.debug(f'name: {name}') + logger.debug(f'container: {container}') + logger.debug(f'conditions: {conditions}') + logger.debug(f'name: {name}') return container, conditions, name @@ -56,7 +56,7 @@ def format(container: dict = container(), template: str = '', parameters: list = """Mock function""" log = logging.getLogger(f'{__name__}.{format.__name__}') - log.debug('container: dict = {}, template: str = '', parameters: list = [], name: str = ''') + logger.debug('container: dict = {}, template: str = '', parameters: list = [], name: str = ''') parameters_orig = parameters @@ -64,10 +64,10 @@ def format(container: dict = container(), template: str = '', parameters: list = [x.split(':')] for x in parameters ] - log.debug(f'container: {container}') - log.debug(f'template: {template}') - log.debug(f'parameters: {parameters}') - log.debug(f'name: {name}') + logger.debug(f'container: {container}') + logger.debug(f'template: {template}') + logger.debug(f'parameters: {parameters}') + logger.debug(f'name: {name}') return container, template, parameters, name @@ -76,7 +76,7 @@ def get_format_data(name: str, **kwargs): """Mock function""" log = logging.getLogger(f'{__name__}.{get_format_data.__name__}') - log.debug(f'name: {name}') + logger.debug(f'name: {name}') return name @@ -85,11 +85,11 @@ def collect2(container: dict = container(), datapath: list = [], action_results: """Mock function""" log = logging.getLogger(f'{__name__}.{collect2.__name__}') - log.info('container: dict = {}, datapath: list = [], action_results: object = None') + logger.info('container: dict = {}, datapath: list = [], action_results: object = None') - log.debug(f'container: {container}') - log.debug(f'datapath: {datapath}') - log.debug(f'action_results: {action_results}') + logger.debug(f'container: {container}') + logger.debug(f'datapath: {datapath}') + logger.debug(f'action_results: {action_results}') return [[artifact(), artifact()]] @@ -98,14 +98,14 @@ def act(action: str, parameters: str, assets: list, callback: object, name: str, """Mock function""" log = logging.getLogger(f'{__name__}.{act.__name__}') - # log.info(f'action: str, parameters: str, assets: list, callback: object, name: str') + # logger.info(f'action: str, parameters: str, assets: list, callback: object, name: str') - log.debug(f'action: {action}') - log.debug(f'parameters: {parameters}') - log.debug(f'assets: {assets}') - log.debug(f'callback: {callback}') - log.debug(f'name: {name}') - log.debug(f'parent_action: {parent_action}') + logger.debug(f'action: {action}') + logger.debug(f'parameters: {parameters}') + logger.debug(f'assets: {assets}') + logger.debug(f'callback: {callback}') + logger.debug(f'name: {name}') + logger.debug(f'parent_action: {parent_action}') return action, parameters, assets, callback, name, parent_action @@ -114,16 +114,16 @@ def save_run_data(key: str, value: str, auto: bool): """Mock function""" log = logging.getLogger(f'{__name__}.{save_run_data.__name__}') - log.debug(f'key: {key}') - log.debug(f'value: {value}') - log.debug(f'auto: {auto}') + logger.debug(f'key: {key}') + logger.debug(f'value: {value}') + logger.debug(f'auto: {auto}') return key, value, auto def debug(object: str): - return log.debug(f'{object}') + return logger.debug(f'{object}') def error(object: str): - return log.error(f'{object}') + return logger.error(f'{object}') diff --git a/automon/integrations/splunk_soar/vault.py b/automon/integrations/splunk_soar/vault.py index cbc17f01..bb42fe4a 100644 --- a/automon/integrations/splunk_soar/vault.py +++ b/automon/integrations/splunk_soar/vault.py @@ -1,9 +1,9 @@ -from automon.log import logger +from automon import log from .datatypes import AbstractDataType -log = logger.logging.getLogger(__name__) -log.setLevel(logger.CRITICAL) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.CRITICAL) class Vault(AbstractDataType): diff --git a/automon/integrations/swift/client.py b/automon/integrations/swift/client.py index 7bbbdd8e..ecc08383 100644 --- a/automon/integrations/swift/client.py +++ b/automon/integrations/swift/client.py @@ -6,20 +6,21 @@ from swiftclient.service import SwiftService -from automon.log import Logging +from automon import log from automon.integrations.swift.error import SwiftError_ # from automon.integrations.swift.config import SwiftConfig from automon.integrations.swift.iterables import SwiftList, SwiftItem -log = Logging(__name__, Logging.INFO) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) -Logging('requests', Logging.CRITICAL) -Logging('swiftclient', Logging.CRITICAL) +log.logging.getLogger('requests').setLevel(log.CRITICAL) +log.logging.getLogger('swiftclient').setLevel(log.CRITICAL) class SwiftClient: def __init__(self): - self._log = Logging(SwiftClient.__name__, Logging.DEBUG) + pass @staticmethod def list_summary(files, folders): @@ -28,7 +29,7 @@ def list_summary(files, folders): def list_container(self, container, summary=False, filter='', separate=False) -> [SwiftItem]: - self._log.info(f'listing {container} (filter: {filter})') + logger.info(f'listing {container} (filter: {filter})') swift_objects = [] swift_objects_v2 = [] @@ -57,7 +58,7 @@ def list_container(self, container, summary=False, filter='', files.append(item_v1) files_v2.append(item_v2) - self._log.debug(f'Listing for {container}: {len(swift_objects)} objects') + logger.debug(f'Listing for {container}: {len(swift_objects)} objects') if summary: return len(swift_objects) @@ -71,7 +72,7 @@ def cleanup(self, container, days=7): msg = (f'Starting cleanup \n' f'>Retention policy: {days} days \n') - self._log.info(msg) + logger.info(msg) today = datetime.date.today() start_time = time.time() @@ -106,20 +107,20 @@ def cleanup(self, container, days=7): regex = f'^{date}' if re.search(regex, item_s.name): - self._log.info(f"{percent}% ({progress}/{total}) cleanup retain: \'{regex}\', {name}") + logger.info(f"{percent}% ({progress}/{total}) cleanup retain: \'{regex}\', {name}") found = True break if not found: pending_deletion.append(item) - self._log.info(f"{percent}% ({progress}/{total}) pending deletion: {name}") + logger.info(f"{percent}% ({progress}/{total}) pending deletion: {name}") elapsed_time = time.time() - start_time minutes = int(elapsed_time / 60) msg = (f'>Created deletion list for past {days} days (Deleting: {len(pending_deletion)} objects)\n' f">Cleanup has been running for: {minutes} minutes") - self._log.debug(msg) + logger.debug(msg) objects = 0 folders = 0 @@ -141,14 +142,14 @@ def cleanup(self, container, days=7): else: objects += 1 - self._log.info(f'{percent}% ({progress}/{total}) summarizing deletion: {name}') + logger.info(f'{percent}% ({progress}/{total}) summarizing deletion: {name}') msg = (f'Cleaning up older than {days} days: \n' f'>{total} *total objects* \n' f'>{objects} *objects* \n' f'>{folders} *folders* \n' f'see debug messages') - self._log.debug(msg) + logger.debug(msg) progress = 0 @@ -163,7 +164,7 @@ def cleanup(self, container, days=7): # this does the actual deletion self.delete_object(container, item) - self._log.info(f'{percent}% ({progress}/{total}) deleted: {name}') + logger.info(f'{percent}% ({progress}/{total}) deleted: {name}') if progress % 10000 == 0 or progress % total == 0: elapsed_time = time.time() - start_time @@ -171,12 +172,12 @@ def cleanup(self, container, days=7): msg = (f'>Deletion is currently at `{percent}%` ({progress}/{total})\n' f">Backup has been running for: {minutes} minutes") - self._log.debug(msg) + logger.debug(msg) if pending_deletion: msg = ('Cleanup Finished\n' f">It took: {minutes} minutes") - self._log.debug(msg) + logger.debug(msg) def backup(self, source, destination, test=None, skip_known=True): @@ -188,7 +189,7 @@ def backup(self, source, destination, test=None, skip_known=True): retries = 0 msg = f'Backup {source} started' - self._log.debug(msg) + logger.debug(msg) items = self.list_container(source) backups = self.list_container(destination, filter=today) @@ -206,7 +207,7 @@ def backup(self, source, destination, test=None, skip_known=True): if test: regex = str(test) if re.search(regex, item): - self._log.debug(f'Test match: {test} \n>{item}') + logger.debug(f'Test match: {test} \n>{item}') else: continue @@ -219,7 +220,7 @@ def backup(self, source, destination, test=None, skip_known=True): for b in backups: b = SwiftItem(b) if f'{today}/{s_item.name}' == b.name: - self._log.info(f'{percent}% ({progress}/{total}) exists, skipping {s_item.name}') + logger.info(f'{percent}% ({progress}/{total}) exists, skipping {s_item.name}') exists = True break if exists: @@ -230,7 +231,7 @@ def backup(self, source, destination, test=None, skip_known=True): minutes = int(elapsed_time / 60) msg = (f'>Backup {source} is currently at `{percent}%` ({progress}/{total})\n' f">Backup has been running for: {minutes} minutes") - self._log.debug(msg) + logger.debug(msg) self.stats(destination, filter=today) @@ -250,19 +251,19 @@ def backup(self, source, destination, test=None, skip_known=True): for i in swift.upload(destination, [folder], options): if i["success"]: - self._log.info( + logger.info( f'{percent}% ({progress}/{total}) ' f'created directory /{destination}/{today}/{name}') retry = False if "error" in i and isinstance(i["error"], Exception): - self._log.error(f'{SwiftError_(i)}') + logger.error(f'{SwiftError_(i)}') retries += 1 except Exception as _: - self._log.error(item) + logger.error(item) retries += 1 @@ -280,12 +281,12 @@ def backup(self, source, destination, test=None, skip_known=True): if i["success"]: if i["action"] == "copy_object": - self._log.info( + logger.info( f'{percent}% ({progress}/{total}) ' f'copied {i["destination"]} from /{i["container"]}/{i["object"]}') if i["action"] == "create_container": - self._log.info( + logger.info( f'{percent}% ({progress}/{total}) ' f'container {i["container"]} created') @@ -304,29 +305,29 @@ def backup(self, source, destination, test=None, skip_known=True): else: error = f'{SwiftError_(i)}' - log.error(error) + logger.error(error) retries += 1 except Exception as _: - log.error(item) + logger.error(item) retries += 1 if not retry: break - self._log.info('building backup summary') + logger.info('building backup summary') source_total_objects, \ - source_total_dirs, \ - source_objects, _, _ = source_stats + source_total_dirs, \ + source_objects, _, _ = source_stats filter_destination_total_objects, \ - filter_destination_total_dirs, \ - filter_objects, _, _ = self.stats(destination, post_log=True, filter=today) + filter_destination_total_dirs, \ + filter_objects, _, _ = self.stats(destination, post_log=True, filter=today) destination_total_objects, \ - destination_total_dirs, \ - destination_objects, _, _ = self.stats(destination, post_log=False) + destination_total_dirs, \ + destination_objects, _, _ = self.stats(destination, post_log=False) # elapsed_time = time.time() - start_time # minutes = int(elapsed_time / 60) @@ -345,7 +346,7 @@ def backup(self, source, destination, test=None, skip_known=True): f'>{destination_total_objects} *total objects* \n' f'>{destination_objects} *objects* \n' f'>{destination_total_dirs} *dirs* \n') - log.debug(msg) + logger.debug(msg) # TODO: new files may be added during the backup, so need verify only the initial files being backed up @@ -353,7 +354,7 @@ def backup(self, source, destination, test=None, skip_known=True): # missing_count = len(missing) # if missing: # try: - # self.log.info( + # logger.info( # f'Missing {missing_count} objects' # '>'.join(missing) # ) @@ -365,7 +366,7 @@ def backup(self, source, destination, test=None, skip_known=True): # # msg = ('*Backup Finished* \n' # f'>It took: {minutes} minutes ({retries} retries)\n') - # self.log.debug(msg) + # logger.debug(msg) def find_missing(self, source, destination, filter, post_log=True): @@ -397,13 +398,13 @@ def find_missing(self, source, destination, filter, post_log=True): found = True break if not found: - self._log.info(f'{percent}% ({progress}/{total}) backup missing: {a_name}') + logger.info(f'{percent}% ({progress}/{total}) backup missing: {a_name}') missing_objects + (tuple(f' * {a_item} \n')) missing_objects_list.append(a_item) else: - self._log.info(f'{percent}% ({progress}/{total}) verified, {a_name}') + logger.info(f'{percent}% ({progress}/{total}) verified, {a_name}') - log.debug(f'missing_objects: {len(missing_objects)}') + logger.debug(f'missing_objects: {len(missing_objects)}') if missing_objects: msg = (f'Missing {len(missing_objects)} objects: \n' @@ -414,7 +415,7 @@ def find_missing(self, source, destination, filter, post_log=True): def stats(self, container, filter='', post_log=True, show_types=False): - self._log.info(f'stat {container} (filter: {filter})') + logger.info(f'stat {container} (filter: {filter})') list_items = self.list_container(container, filter=filter) total_dirs = 0 @@ -462,10 +463,10 @@ def delete_object(self, container, item): try: for i in swift.delete(container=container, objects=[name]): if i['success']: - self._log.info(f'deleted: {name}') + logger.info(f'deleted: {name}') except Exception as e: - log.error(f'{e}') + logger.error(f'{e}') def delete(self, container, filter): @@ -485,12 +486,12 @@ def delete(self, container, filter): try: for i in swift.delete(container=container, objects=[name]): if i['success']: - self._log.info(f'{percent}% ({progress}/{deletion_count}) deleted: {name}') + logger.info(f'{percent}% ({progress}/{deletion_count}) deleted: {name}') except Exception as e: - log.error(f'{e}') + logger.error(f'{e}') - log.debug( + logger.debug( f'Deletion summary: \n>container: {container} \n>filter: {filter} \n>{deletion_count} objects deleted') def delete_container(self, container): @@ -499,10 +500,10 @@ def delete_container(self, container): try: for _ in swift.delete(container): - self._log.info(f'deleting container: {container}') + logger.info(f'deleting container: {container}') except Exception as e: - log.error(f'{e}') + logger.error(f'{e}') def restore(self): return warnings.warn(NotImplemented) diff --git a/automon/integrations/swift/config.py b/automon/integrations/swift/config.py index 87f60303..a5a61232 100644 --- a/automon/integrations/swift/config.py +++ b/automon/integrations/swift/config.py @@ -1,6 +1,9 @@ import os -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SwiftConfig: @@ -17,39 +20,38 @@ class SwiftConfig: SWIFTCLIENT_INSECURE = os.getenv('SWIFTCLIENT_INSECURE') or 'True' def __init__(self): - self.log = Logging(name=SwiftConfig.__name__, level=Logging.DEBUG) if not self.OPENSTACK_USERNAME: - self.log.warning(f'missing OPENSTACK_USERNAME') + logger.warning(f'missing OPENSTACK_USERNAME') if not self.OPENSTACK_PASSWORD: - self.log.warning(f'missing OPENSTACK_PASSWORD') + logger.warning(f'missing OPENSTACK_PASSWORD') if not self.OPENSTACK_AUTH_URL: - self.log.warning(f'missing OPENSTACK_AUTH_URL') + logger.warning(f'missing OPENSTACK_AUTH_URL') if not self.OPENSTACK_PROJECT_ID: - self.log.warning(f'missing OPENSTACK_PROJECT_ID') + logger.warning(f'missing OPENSTACK_PROJECT_ID') if not self.OPENSTACK_PROJECT_NAME: - self.log.warning(f'missing OPENSTACK_PROJECT_NAME') + logger.warning(f'missing OPENSTACK_PROJECT_NAME') if not self.OPENSTACK_USER_DOMAIN_NAME: - self.log.warning(f'missing OPENSTACK_USER_DOMAIN_NAME') + logger.warning(f'missing OPENSTACK_USER_DOMAIN_NAME') if not self.OPENSTACK_PROJECT_DOMAIN_ID: - self.log.warning(f'missing OPENSTACK_PROJECT_DOMAIN_ID') + logger.warning(f'missing OPENSTACK_PROJECT_DOMAIN_ID') if not self.OPENSTACK_REGION_NAME: - self.log.warning(f'missing OPENSTACK_REGION_NAME') + logger.warning(f'missing OPENSTACK_REGION_NAME') if not self.OPENSTACK_INTERFACE: - self.log.warning(f'missing OPENSTACK_INTERFACE') + logger.warning(f'missing OPENSTACK_INTERFACE') if not self.OPENSTACK_IDENTITY_API_VERSION: - self.log.warning(f'missing OPENSTACK_IDENTITY_API_VERSION') + logger.warning(f'missing OPENSTACK_IDENTITY_API_VERSION') if not self.SWIFTCLIENT_INSECURE: - self.log.warning(f'missing SWIFTCLIENT_INSECURE') + logger.warning(f'missing SWIFTCLIENT_INSECURE') def __eq__(self, other): if not isinstance(other, SwiftConfig): - self.log.warning(f'Not implemented') + logger.warning(f'Not implemented') return NotImplemented return self.OPENSTACK_USERNAME == other.OPENSTACK_USERNAME and \ - self.OPENSTACK_PASSWORD == other.OPENSTACK_PASSWORD and \ - self.OPENSTACK_AUTH_URL == other.OPENSTACK_AUTH_URL and \ - self.OPENSTACK_PROJECT_ID == other.OPENSTACK_PROJECT_ID and \ - self.OPENSTACK_PROJECT_NAME == other.OPENSTACK_PROJECT_NAME and \ - self.OPENSTACK_PROJECT_DOMAIN_ID == other.OPENSTACK_PROJECT_DOMAIN_ID + self.OPENSTACK_PASSWORD == other.OPENSTACK_PASSWORD and \ + self.OPENSTACK_AUTH_URL == other.OPENSTACK_AUTH_URL and \ + self.OPENSTACK_PROJECT_ID == other.OPENSTACK_PROJECT_ID and \ + self.OPENSTACK_PROJECT_NAME == other.OPENSTACK_PROJECT_NAME and \ + self.OPENSTACK_PROJECT_DOMAIN_ID == other.OPENSTACK_PROJECT_DOMAIN_ID diff --git a/automon/integrations/swift/iterables.py b/automon/integrations/swift/iterables.py index ddcad4cc..c943036f 100644 --- a/automon/integrations/swift/iterables.py +++ b/automon/integrations/swift/iterables.py @@ -2,12 +2,14 @@ from swiftclient.service import SwiftService, SwiftError, ClientException -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class SwiftItem(object): def __init__(self, item: dict): - self._log = Logging(SwiftItem.__name__, Logging.DEBUG) self._item = item self._dict = item @@ -48,7 +50,6 @@ def __repr__(self): class SwiftPage(SwiftService): def __init__(self, page: dict) -> SwiftService.list: - self._log = Logging(SwiftPage.__name__, Logging.ERROR) self._page = page @@ -62,11 +63,11 @@ def __init__(self, page: dict) -> SwiftService.list: self.error_timestamp = self._page.get('error_timestamp') if self.error == ClientException: - self._log.error(f'{SwiftPage.__name__} {self.success} {self.error}') + logger.error(f'{SwiftPage.__name__} {self.success} {self.error}') if self.success: self.listing = self._page.get('listing') - self._log.debug(f'{SwiftPage.__name__} {self.success} {self.listing}') + logger.debug(f'{SwiftPage.__name__} {self.success} {self.listing}') else: self.listing = [] @@ -85,7 +86,7 @@ def _dict(self): def _error_handler(self): if not self.success and isinstance(self.error, Exception): - self._log.error(f'{SwiftError(self._page)}') + logger.error(f'{SwiftError(self._page)}') def list_gen(self) -> object or SwiftItem: if self.success: @@ -105,7 +106,6 @@ def __init__(self, container: str) -> SwiftService.list: see documentation """ - self._log = Logging(SwiftList.__name__, Logging.DEBUG) self.container = container def list_gen(self) -> object or SwiftPage: @@ -119,7 +119,7 @@ def list_gen(self) -> object or SwiftPage: if page["success"]: yield SwiftPage(page) else: - self._log.error(f'{page["error"]}') + logger.error(f'{page["error"]}') except Exception as e: - self._log.error(f'page failed, {e}') + logger.error(f'page failed, {e}') diff --git a/automon/integrations/vds/client.py b/automon/integrations/vds/client.py index f7798871..918731b3 100644 --- a/automon/integrations/vds/client.py +++ b/automon/integrations/vds/client.py @@ -9,7 +9,10 @@ from queue import Queue -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.INFO) # disable insecure ssl warnings urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -30,8 +33,6 @@ def __init__(self, config: VdsConfig = None): # test connection self.connected = self.check_connection() - self._log = Logging(name=VdsRestClient.__name__, level=Logging.INFO) - @staticmethod def check_connection(): """check if vds server is reacheable""" @@ -84,7 +85,7 @@ def search_all(self, ldap_filter: str = 'filter=cn=*', page_size: int = 10, cafi [self.records.put_nowait(x) for x in records['resources']] - self._log.info(f'Records: {self.records.qsize()}') + logger.info(f'Records: {self.records.qsize()}') # need to fix cookie 'hostname' to reflect vds config server # otherwise get request fails to match hostname to ssl hostname @@ -141,20 +142,20 @@ def _get(self, ldap_filter: str = None, **kwargs): url = f'{self.config.uri}/{self.config.basedn}{ldap_filter}' - self._log.debug(f'query: {url}') + logger.debug(f'query: {url}') try: r = requests.get(url, headers=headers, **kwargs) except Exception as e: raise Exception(e) - [self._log.debug(f'results: {x}') for x in r.__dict__.items()] + [logger.debug(f'results: {x}') for x in r.__dict__.items()] if r.status_code != 200: - self._log.error(f'{url} {r.status_code} {r.reason}\n\n{r.content.decode()}') + logger.error(f'{url} {r.status_code} {r.reason}\n\n{r.content.decode()}') ldap_result = False else: - self._log.debug(f'{r.status_code} {r.reason} {url}') + logger.debug(f'{r.status_code} {r.reason} {url}') ldap_result = json.loads(r.content.decode()) return ldap_result diff --git a/automon/integrations/vds/config.py b/automon/integrations/vds/config.py index f32d989d..303e870a 100644 --- a/automon/integrations/vds/config.py +++ b/automon/integrations/vds/config.py @@ -1,6 +1,9 @@ import os -from automon.log import Logging +from automon import log + +logger = log.logging.getLogger(__name__) +logger.setLevel(log.ERROR) class VdsConfig(object): @@ -30,9 +33,7 @@ def __init__(self, protocol: str = None, self.uri = f'{self.prot}://{self.server}:{self.port}/{self.path}' - self._log = Logging(name=VdsConfig.__name__, level=Logging.ERROR) - - [self._log.debug(f'config: {x}') for x in self.__dict__.items()] + [logger.debug(f'config: {x}') for x in self.__dict__.items()] def __repr__(self): return f'{self.prot}://{self.server}:{self.port} ({self.bind_user}) ({self.basedn})' diff --git a/automon/log/logger.py b/automon/log/logger.py index 0cb9c25f..0330f0b7 100644 --- a/automon/log/logger.py +++ b/automon/log/logger.py @@ -14,14 +14,8 @@ CRITICAL = logging.CRITICAL NOTSET = logging.NOTSET -logging.getLogger(__name__).setLevel(CRITICAL) - -TIMESTAMP = True -DEFAULT_LEVEL = INFO - -log_format = LogRecordAttribute(timestamp=TIMESTAMP).levelname().name_and_lineno().funcName().message() -log_format = f'{log_format}' -logging.basicConfig(level=DEFAULT_LEVEL, format=log_format) +logger = logging.getLogger(__name__) +logger.setLevel(CRITICAL) class Callback(object): @@ -30,7 +24,6 @@ def __init__(self, callbacks: list): """Log to callbacks """ - self.log = Logging(name=Callback.__name__, level=Logging.DEBUG) self.callbacks = callbacks def call(self, type: str, msg: str, *args, **kwargs) -> True: diff --git a/automon/log/tests/test_logger.py b/automon/log/tests/test_logger.py deleted file mode 100644 index 3d6a444c..00000000 --- a/automon/log/tests/test_logger.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest - -from automon.log.logger import Logging, LogStream - - -class LoggingTest(unittest.TestCase): - log = Logging() - - def test_logger(self): - self.assertTrue(self.log) - - def test_error(self): - self.assertTrue(self.log.error('test')) - - def test_debug(self): - self.assertTrue(self.log.debug('test')) - - def test_info(self): - self.assertTrue(self.log.info('test')) - - def test_critical(self): - self.assertTrue(self.log.critical('test')) - - def test_warn(self): - self.assertTrue(self.log.warning('test')) - with self.assertRaises(Exception): - self.log.error(raise_exception=True) - - def test_now(self): - self.assertTrue(self.log.now()) - - def test_delta(self): - self.assertTrue(self.log.uptime()) - - -class LogStreamTest(unittest.TestCase): - stream = LogStream() - - def test_stream(self): - self.assertTrue(self.stream) - - def test_repr(self): - self.assertFalse(f'{self.stream}') - - def test_flush(self): - self.assertIsNone(self.stream.flush()) - - def test_write(self): - self.assertIsNone(self.stream.write('test')) - - -if __name__ == '__main__': - unittest.main() diff --git a/automon/log/tests/test_logger_builtin.py b/automon/log/tests/test_logger_builtin.py index 29e2d511..4749ab83 100644 --- a/automon/log/tests/test_logger_builtin.py +++ b/automon/log/tests/test_logger_builtin.py @@ -1,30 +1,27 @@ import unittest -from automon.log import logger +from automon import log -log = logger.logging.getLogger(__name__) -log.setLevel(logger.DEBUG) +logger = log.logging.getLogger(__name__) +logger.setLevel(log.DEBUG) class LoggingTest(unittest.TestCase): - def test_logger(self): - self.assertTrue(log) - def test_error(self): - self.assertIsNone(log.error('test')) + self.assertIsNone(logger.error('test')) def test_debug(self): - self.assertIsNone(log.debug('test')) + self.assertIsNone(logger.debug('test')) def test_info(self): - self.assertIsNone(log.info('test')) + self.assertIsNone(logger.info('test')) def test_critical(self): - self.assertIsNone(log.critical('test')) + self.assertIsNone(logger.critical('test')) def test_warn(self): - self.assertIsNone(log.warning('test')) + self.assertIsNone(logger.warning('test')) if __name__ == '__main__':