From 6d2a8f9897ddd665a68602ca12713dd3d0249f4b Mon Sep 17 00:00:00 2001 From: Mayara Moromisato <44944954+moromimay@users.noreply.github.com> Date: Tue, 6 Apr 2021 09:09:46 -0300 Subject: [PATCH] [BUG] Change logging config (#307) * Change logging config. * Some adjusts. * Remove a code smell. --- Makefile | 1 + butterfree/__init__.py | 7 --- butterfree/_cli/migrate.py | 14 ++--- butterfree/configs/logger.py | 24 +++++++++ butterfree/logging.conf | 52 ------------------- .../database_migration/database_migration.py | 3 +- logs/logging.json | 0 setup.py | 2 +- tests/unit/butterfree/_cli/test_migrate.py | 14 +++++ 9 files changed, 50 insertions(+), 67 deletions(-) create mode 100644 butterfree/configs/logger.py delete mode 100644 butterfree/logging.conf delete mode 100644 logs/logging.json diff --git a/Makefile b/Makefile index 397d04bf4..95cc6e3a6 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,7 @@ clean: @find ./ -type f -name 'coverage.xml' -exec rm -f {} \; @find ./ -type f -name '.coverage*' -exec rm -f {} \; @find ./ -type f -name '*derby.log' -exec rm -f {} \; + @find ./ -type f -name 'logging.json' -exec rm -f {} \; @find ./ -name '*.pyc' -exec rm -f {} \; @find ./ -name '*.pyo' -exec rm -f {} \; @find ./ -name '*~' -exec rm -f {} \; diff --git a/butterfree/__init__.py b/butterfree/__init__.py index 25b955c6e..18759b031 100644 --- a/butterfree/__init__.py +++ b/butterfree/__init__.py @@ -1,8 +1 @@ """Module docstring example, following Google's docstring style.""" -import logging.config -import os -import sys - -sys.path.insert(0, os.path.abspath(".")) - -logging.config.fileConfig(fname="butterfree/logging.conf") diff --git a/butterfree/_cli/migrate.py b/butterfree/_cli/migrate.py index 6e3e9b592..f3c533d7c 100644 --- a/butterfree/_cli/migrate.py +++ b/butterfree/_cli/migrate.py @@ -1,6 +1,5 @@ import importlib import inspect -import logging import os import pkgutil import sys @@ -11,13 +10,14 @@ from butterfree.clients import SparkClient from butterfree.configs import environment +from butterfree.configs.logger import __logger from butterfree.extract.readers import FileReader from butterfree.migrations.database_migration import ALLOWED_DATABASE from butterfree.pipelines import FeatureSetPipeline app = typer.Typer(help="Apply the automatic migrations in a database.") -logger = logging.getLogger("migrate") +logger = __logger("migrate", True) def __find_modules(path: str) -> Set[str]: @@ -114,7 +114,9 @@ def __init__( def _send_logs_to_s3(self, file_local: bool) -> None: """Send all migration logs to S3.""" - file_reader = FileReader(id="name", path="logs/logging.json", format="json") + log_path = "../logging.json" + + file_reader = FileReader(id="name", path=log_path, format="json") df = file_reader.consume(self.spark_client) path = environment.get_variable("FEATURE_STORE_S3_BUCKET") @@ -126,10 +128,10 @@ def _send_logs_to_s3(self, file_local: bool) -> None: **{"path": f"s3a://{path}/logging"}, ) - if not file_local: - os.rmdir("logs/logging.json") + if not file_local and os.path.exists(log_path): + os.remove(log_path) - def run(self, generate_logs: bool) -> None: + def run(self, generate_logs: bool = False) -> None: """Construct and apply the migrations.""" for pipeline in self.pipelines: for writer in pipeline.sink.writers: diff --git a/butterfree/configs/logger.py b/butterfree/configs/logger.py new file mode 100644 index 000000000..60dab67c7 --- /dev/null +++ b/butterfree/configs/logger.py @@ -0,0 +1,24 @@ +"""Logger funcion.""" + +import logging + + +def __config(json_file_logs: bool = False) -> None: + + if json_file_logs: + return logging.basicConfig( + format='{"name": "%(name)s", "timestamp": "%(asctime)-15s", ' + '"level": "%(levelname)s", "message": "%(message)s"}', + level=logging.INFO, + filename="../logging.json", + ) + return logging.basicConfig( + format="%(name)s:%(asctime)-15s:%(levelname)s:< %(message)s >", + level=logging.INFO, + ) + + +def __logger(name: str, file_logs: bool = False) -> logging.Logger: + + __config(file_logs) + return logging.getLogger(name) diff --git a/butterfree/logging.conf b/butterfree/logging.conf deleted file mode 100644 index 1ee6da868..000000000 --- a/butterfree/logging.conf +++ /dev/null @@ -1,52 +0,0 @@ -[loggers] -keys=root,cli,migrate,database_migrate - -[handlers] -keys=consoleHandler,file - -[formatters] -keys=simpleFormatter,jsonFormatter - -[logger_root] -level=DEBUG -handlers=consoleHandler - -[logger_cli] -level=DEBUG -handlers=file -qualname=cli -propagate=0 - -[logger_migrate] -level=DEBUG -handlers=file -qualname=migrate -propagate=0 - -[logger_database_migrate] -level=DEBUG -handlers=file -qualname=database_migrate -propagate=0 - -[handler_consoleHandler] -class=StreamHandler -level=DEBUG -formatter=simpleFormatter -args=(sys.stdout,) - -[handler_file] -class=FileHandler -level=DEBUG -formatter=jsonFormatter -args=('logs/logging.json', "a") - -[formatter_simpleFormatter] -format=%(name)s:%(asctime)-15s:%(levelname)s:%(message)s -datefmt= -class=logging.Formatter - -[formatter_jsonFormatter] -format={"name": "%(name)s", "timestamp": "%(asctime)-15s", "level": "%(levelname)s", "message": "%(message)s"} -datefmt= -class=logging.Formatter \ No newline at end of file diff --git a/butterfree/migrations/database_migration/database_migration.py b/butterfree/migrations/database_migration/database_migration.py index 28f4f06c7..2ceca0b82 100644 --- a/butterfree/migrations/database_migration/database_migration.py +++ b/butterfree/migrations/database_migration/database_migration.py @@ -6,10 +6,11 @@ from typing import Any, Dict, List, Set from butterfree.clients import AbstractClient +from butterfree.configs.logger import __logger from butterfree.load.writers.writer import Writer from butterfree.transform import FeatureSet -logger = logging.getLogger("database_migrate") +logger = __logger("database_migrate", True) @dataclass diff --git a/logs/logging.json b/logs/logging.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/setup.py b/setup.py index abd56ccbd..5122a8319 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages, setup __package_name__ = "butterfree" -__version__ = "1.2.0.dev6" +__version__ = "1.2.0.dev7" __repository_url__ = "https://github.com/quintoandar/butterfree" with open("requirements.txt") as f: diff --git a/tests/unit/butterfree/_cli/test_migrate.py b/tests/unit/butterfree/_cli/test_migrate.py index aa2c86db0..75487bed4 100644 --- a/tests/unit/butterfree/_cli/test_migrate.py +++ b/tests/unit/butterfree/_cli/test_migrate.py @@ -1,12 +1,17 @@ from unittest.mock import call +from typer.testing import CliRunner + from butterfree._cli import migrate +from butterfree._cli.main import app from butterfree.migrations.database_migration import ( CassandraMigration, MetastoreMigration, ) from butterfree.pipelines import FeatureSetPipeline +runner = CliRunner() + class TestMigrate: def test_migrate_success(self, mocker): @@ -38,3 +43,12 @@ def test_migrate_all_pairs(self, mocker): cassandra_pairs, any_order=True ) migrate.Migrate._send_logs_to_s3.assert_called_once() + + def test_app_cli(self): + result = runner.invoke(app, "migrate") + assert result.exit_code == 0 + + def test_app_migrate(self, mocker): + mocker.patch.object(migrate.Migrate, "run") + result = runner.invoke(app, ["migrate", "apply", "tests/mocks/entities/"]) + assert result.exit_code == 0