From d8eb65372f22687d1c0b55a452966c94f39a34af Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 18:44:40 +0100 Subject: [PATCH 01/15] Re-arrange docker run command on multiple line, no chnages --- run.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 6143ca0..8d595a5 100755 --- a/run.sh +++ b/run.sh @@ -5,7 +5,11 @@ NAME="root-pythia" run__prod () { # prod mode docker build --file Dockerfile -t ${NAME}:latest . - docker run --rm --interactive --tty --detach --env-file .env.prod --name ${NAME} ${NAME}:latest + docker run --rm --interactive --tty \ + --detach \ + --env-file .env.prod \ + --name ${NAME} \ + ${NAME}:latest } run__dev () { From 4f3948d55dda1b3761dd61d148a7311ec996f88a Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 18:47:37 +0100 Subject: [PATCH 02/15] Share a volume to save logs on host in prod mode --- run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run.sh b/run.sh index 8d595a5..9ea3b1f 100755 --- a/run.sh +++ b/run.sh @@ -7,6 +7,7 @@ run__prod () { docker build --file Dockerfile -t ${NAME}:latest . docker run --rm --interactive --tty \ --detach \ + --volume $(realpath -P ${LOG_FOLDER}):/opt/${NAME}/logs --env-file .env.prod \ --name ${NAME} \ ${NAME}:latest From d43ef043b4f3520a70bb580302bee6d98d82d4f8 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 18:47:56 +0100 Subject: [PATCH 03/15] Add the LOG_FOLDER environment variable --- .env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.example b/.env.example index fad2b7d..180d2d1 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,7 @@ CHANNEL_ID=XXX ### Logging # accept integer or valid strings as described in the doc: https://docs.python.org/3/library/logging.html#logging.Logger.setLevel LOG_LEVEL=INFO +LOG_FOLDER=path/to/log/folder ### Root-me API # get an API Key by logging in on Root-me then go to 'my settings' From 03b540166555815dd7add244b0fd13407b8dd93f Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 18:48:43 +0100 Subject: [PATCH 04/15] Source .env.prod in run.sh --- run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run.sh b/run.sh index 9ea3b1f..aea6075 100755 --- a/run.sh +++ b/run.sh @@ -5,6 +5,8 @@ NAME="root-pythia" run__prod () { # prod mode docker build --file Dockerfile -t ${NAME}:latest . + + . .env.prod docker run --rm --interactive --tty \ --detach \ --volume $(realpath -P ${LOG_FOLDER}):/opt/${NAME}/logs From c97542570a760d11268894c29a83a3b3ba52b3cf Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 19:20:17 +0100 Subject: [PATCH 05/15] Add a rotating file handler to root logger to log to ./logs/RootPythia.log see rotating file handler doc: https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler --- src/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main.py b/src/main.py index a9e1b8f..7026820 100755 --- a/src/main.py +++ b/src/main.py @@ -15,6 +15,11 @@ def main(): # Setup a beautiful root logger discord.utils.setup_logging(root=True, level=LOG_LEVEL) + # Add a file handler to the root logger + file_handler = logging.RotatingFileHandler("./logs/RootPythia.log", mode='a', maxBytes=10e6, backupCount=5) + logging.getLogger().addHandler(file_handler) + logging.info("FileHandler added to root logger it will write to '%s'", file_handler.stream.name) + # are these call secure?? logging.debug("discord token: %s", DISCORD_TOKEN) From e655663efcf3c968d8a711fe6f6cbc36b2191d50 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 19:21:02 +0100 Subject: [PATCH 06/15] Fix a bug when setting LOG_LEVEL to numerical values --- src/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.py b/src/main.py index 7026820..15f8e57 100755 --- a/src/main.py +++ b/src/main.py @@ -9,6 +9,8 @@ MODE = getenv("MODE") DISCORD_TOKEN = getenv("DISCORD_TOKEN") LOG_LEVEL = getenv("LOG_LEVEL") +if LOG_LEVEL.isnumeric(): + LOG_LEVEL = int(LOG_LEVEL) def main(): From 94e5b0975158462416de133fb961b8ad379837c7 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 19:38:49 +0100 Subject: [PATCH 07/15] Homogenize .env.example --- .env.example | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 180d2d1..58e3eba 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,6 @@ MODE=dev|prod ### Discord env var - # You can follow https://realpython.com/how-to-make-a-discord-bot-python/#creating-an-application to get a discord token DISCORD_TOKEN=XXX @@ -11,6 +10,7 @@ DISCORD_TOKEN=XXX # You can follow https://turbofuture.com/internet/Discord-Channel-ID to get your channel ID CHANNEL_ID=XXX + ### Logging # accept integer or valid strings as described in the doc: https://docs.python.org/3/library/logging.html#logging.Logger.setLevel LOG_LEVEL=INFO @@ -21,7 +21,9 @@ LOG_FOLDER=path/to/log/folder API_KEY_ROOTME= API_URL=https://api.www.root-me.org/ +# [OPTIONAL] the maximum number of time a single request is retried (if relevant) MAX_API_ATTEMPT=5 ### Bot Configuration -REFRESH_DELAY= # in seconds ! (default value if not set here is 10 seconds) +# [OPTIONAL] in seconds, the delay between new solve checking (default is 10) +REFRESH_DELAY= From 11cc14c8daed35feb31abfab0db05f9dea7a9226 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 19:39:43 +0100 Subject: [PATCH 08/15] Add the capability of configuring log file size and number through env var --- .env.example | 6 ++++++ src/main.py | 25 ++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 58e3eba..75f2e1e 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,12 @@ CHANNEL_ID=XXX LOG_LEVEL=INFO LOG_FOLDER=path/to/log/folder +# [OPTIONAL] in Bytes, the maximum size a log file can grow (default is 10e6) +LOG_FILE_SIZE_MAX= +# [OPTIONAL] the maximum number of log file generated, if exceeded the older are removed (default is 5) +LOG_FILES_NUMBER= + + ### Root-me API # get an API Key by logging in on Root-me then go to 'my settings' API_KEY_ROOTME= diff --git a/src/main.py b/src/main.py index 15f8e57..2d49ab2 100755 --- a/src/main.py +++ b/src/main.py @@ -2,6 +2,13 @@ from os import getenv import sys +# in bytes +DEFAULT_LOG_FILE_SIZE_MAX = 10e6 +# the number of files the rotating file handler will generate at max +# if exceeded it removes the older one +# see https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler +DEFAULT_LOG_FILES_NUMBER = 5 + import discord from bot import BOT as root_pythia @@ -12,13 +19,29 @@ if LOG_LEVEL.isnumeric(): LOG_LEVEL = int(LOG_LEVEL) +LOG_FILE_SIZE_MAX = getenv("LOG_FILE_SIZE_MAX") or DEFAULT_LOG_FILE_SIZE_MAX +try: + LOG_FILE_SIZE_MAX = int(LOG_FILE_SIZE_MAX) +except ValueError as exc: + logging.exception("LOG_FILE_SIZE_MAX is not an integer") + exit(1) + +LOG_FILES_NUMBER = getenv("LOG_FILES_NUMBER") or DEFAULT_LOG_FILES_NUMBER +try: + LOG_FILES_NUMBER = int(LOG_FILES_NUMBER) +except ValueError as exc: + logging.exception("LOG_FILES_NUMBER is not an integer") + exit(1) + def main(): # Setup a beautiful root logger discord.utils.setup_logging(root=True, level=LOG_LEVEL) # Add a file handler to the root logger - file_handler = logging.RotatingFileHandler("./logs/RootPythia.log", mode='a', maxBytes=10e6, backupCount=5) + file_handler = logging.RotatingFileHandler( + "./logs/RootPythia.log", mode="a", maxBytes=LOG_FILE_SIZE_MAX, backupCount=LOG_FILES_NUMBER + ) logging.getLogger().addHandler(file_handler) logging.info("FileHandler added to root logger it will write to '%s'", file_handler.stream.name) From 635cae93095256b76fc1cc71a95c7b3ecdf99e9e Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 19:46:02 +0100 Subject: [PATCH 09/15] Solve pylint issues --- src/main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main.py b/src/main.py index 2d49ab2..8a466ed 100755 --- a/src/main.py +++ b/src/main.py @@ -2,6 +2,11 @@ from os import getenv import sys +import discord + +from bot import BOT as root_pythia + +### Default global variables # in bytes DEFAULT_LOG_FILE_SIZE_MAX = 10e6 # the number of files the rotating file handler will generate at max @@ -9,10 +14,8 @@ # see https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler DEFAULT_LOG_FILES_NUMBER = 5 -import discord - -from bot import BOT as root_pythia +### Global variables MODE = getenv("MODE") DISCORD_TOKEN = getenv("DISCORD_TOKEN") LOG_LEVEL = getenv("LOG_LEVEL") @@ -24,14 +27,14 @@ LOG_FILE_SIZE_MAX = int(LOG_FILE_SIZE_MAX) except ValueError as exc: logging.exception("LOG_FILE_SIZE_MAX is not an integer") - exit(1) + sys.exit(1) LOG_FILES_NUMBER = getenv("LOG_FILES_NUMBER") or DEFAULT_LOG_FILES_NUMBER try: LOG_FILES_NUMBER = int(LOG_FILES_NUMBER) except ValueError as exc: logging.exception("LOG_FILES_NUMBER is not an integer") - exit(1) + sys.exit(1) def main(): @@ -39,7 +42,7 @@ def main(): discord.utils.setup_logging(root=True, level=LOG_LEVEL) # Add a file handler to the root logger - file_handler = logging.RotatingFileHandler( + file_handler = logging.handlers.RotatingFileHandler( "./logs/RootPythia.log", mode="a", maxBytes=LOG_FILE_SIZE_MAX, backupCount=LOG_FILES_NUMBER ) logging.getLogger().addHandler(file_handler) From 7c6c9668ee3b62ae13c7d30e9c0248149c1a6e34 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 19:57:00 +0100 Subject: [PATCH 10/15] fix bugs in run.sh --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index aea6075..7fea5bd 100755 --- a/run.sh +++ b/run.sh @@ -6,10 +6,10 @@ run__prod () { # prod mode docker build --file Dockerfile -t ${NAME}:latest . - . .env.prod + source ./.env.prod docker run --rm --interactive --tty \ --detach \ - --volume $(realpath -P ${LOG_FOLDER}):/opt/${NAME}/logs + --volume $(realpath -P ${LOG_FOLDER}):/opt/${NAME}/logs \ --env-file .env.prod \ --name ${NAME} \ ${NAME}:latest From f4887fc3c0fd9392605bfb6cec636417da1a946b Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 19:58:58 +0100 Subject: [PATCH 11/15] Import logging.handlers where rotating file handler is located in main.py --- src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.py b/src/main.py index 8a466ed..bc5e181 100755 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,5 @@ import logging +import logging.handlers from os import getenv import sys From 29eec633a1d6fac1ffeeda5fdbf278fe2e7a4c0f Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 20:16:07 +0100 Subject: [PATCH 12/15] Use the discord formatter for the file handler too --- src/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index bc5e181..e6d3290 100755 --- a/src/main.py +++ b/src/main.py @@ -42,11 +42,14 @@ def main(): # Setup a beautiful root logger discord.utils.setup_logging(root=True, level=LOG_LEVEL) + root_logger = logging.getLogger() + discord_log_formatter = root_logger.handlers[0] # Add a file handler to the root logger file_handler = logging.handlers.RotatingFileHandler( "./logs/RootPythia.log", mode="a", maxBytes=LOG_FILE_SIZE_MAX, backupCount=LOG_FILES_NUMBER ) - logging.getLogger().addHandler(file_handler) + file_handler.setFormatter(discord_log_formatter) + root_logger.addHandler(file_handler) logging.info("FileHandler added to root logger it will write to '%s'", file_handler.stream.name) # are these call secure?? From 56657d0b23318aa346f6a8a3d77cff30749c68a5 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 20:23:47 +0100 Subject: [PATCH 13/15] Add a note in .env.example --- .env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.example b/.env.example index 75f2e1e..c68e504 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,7 @@ LOG_FOLDER=path/to/log/folder LOG_FILE_SIZE_MAX= # [OPTIONAL] the maximum number of log file generated, if exceeded the older are removed (default is 5) LOG_FILES_NUMBER= +# Note that if either of the previous variables are set to 0 the log file will grow indefinitely ### Root-me API From 116abcf298b0fa15b11cded7e437fcefbb8d5599 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 21:13:55 +0100 Subject: [PATCH 14/15] Add a specific FileHandler to save logs of level warning and above --- src/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main.py b/src/main.py index e6d3290..13d7a89 100755 --- a/src/main.py +++ b/src/main.py @@ -44,6 +44,7 @@ def main(): root_logger = logging.getLogger() discord_log_formatter = root_logger.handlers[0] + # Add a file handler to the root logger file_handler = logging.handlers.RotatingFileHandler( "./logs/RootPythia.log", mode="a", maxBytes=LOG_FILE_SIZE_MAX, backupCount=LOG_FILES_NUMBER @@ -52,6 +53,14 @@ def main(): root_logger.addHandler(file_handler) logging.info("FileHandler added to root logger it will write to '%s'", file_handler.stream.name) + # Add a specific file handler to save warnings and errors + warning_file_handler = logging.FileHandler( + "./logs/RootPythiaErrors.log", mode="a" + ) + warning_file_handler.setLevel(logging.WARNING) + warning_file_handler.setFormatter(discord_log_formatter) + root_logger.addHandler(warning_file_handler) + # are these call secure?? logging.debug("discord token: %s", DISCORD_TOKEN) From 8f809a4489505fc66d5ffbd32d72c0c2aebae6f1 Mon Sep 17 00:00:00 2001 From: ctmbl Date: Fri, 24 Nov 2023 21:15:01 +0100 Subject: [PATCH 15/15] Add logs folder to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index cd94870..900eec0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ **/.env **/.env.prod **/.env.dev + +# Logs +logs/**