From 685f41e8414784d6360053b6795b2fc6d37cc0ec Mon Sep 17 00:00:00 2001 From: ClementMabileau Date: Sat, 31 Aug 2024 20:46:04 +0200 Subject: [PATCH] Add a proper sqlite3 database to RootPythia (#44) --- .env.example | 6 ++ .gitignore | 4 ++ Dockerfile.dev | 4 +- run.sh | 1 + src/bot/root_pythia_bot.py | 16 +++-- src/classes/user.py | 26 +++++-- src/database/__init__.py | 1 + .../db_manager.py} | 67 ++++++++++++++++--- src/database/db_structure.py | 19 ++++++ tests/conftest.py | 12 ++-- tests/test_cog.py | 6 +- ...db_manager.py => test_database_manager.py} | 26 ++++--- 12 files changed, 144 insertions(+), 44 deletions(-) create mode 100644 src/database/__init__.py rename src/{bot/dummy_db_manager.py => database/db_manager.py} (50%) create mode 100644 src/database/db_structure.py rename tests/{test_dummy_db_manager.py => test_database_manager.py} (72%) diff --git a/.env.example b/.env.example index c68e504..93f191e 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,12 @@ API_URL=https://api.www.root-me.org/ # [OPTIONAL] the maximum number of time a single request is retried (if relevant) MAX_API_ATTEMPT=5 + ### Bot Configuration # [OPTIONAL] in seconds, the delay between new solve checking (default is 10) REFRESH_DELAY= + + +### Database Configuration +# the path to the Sqlite database folder +DB_FOLDER=data diff --git a/.gitignore b/.gitignore index 900eec0..b66b74c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ # Logs logs/** + +# Database folder +data/** + diff --git a/Dockerfile.dev b/Dockerfile.dev index 279886b..91eb7fa 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -5,7 +5,9 @@ COPY requirements.txt /tmp/requirements.txt COPY requirements-dev.txt /tmp/requirements-dev.txt RUN pip install -r /tmp/requirements-dev.txt -RUN apt update && apt install -y vim +RUN apt update && apt install -y vim sqlite3 WORKDIR /opt/root-pythia +RUN mkdir logs + CMD ["bash"] diff --git a/run.sh b/run.sh index 7fea5bd..c1fb7f3 100755 --- a/run.sh +++ b/run.sh @@ -10,6 +10,7 @@ run__prod () { docker run --rm --interactive --tty \ --detach \ --volume $(realpath -P ${LOG_FOLDER}):/opt/${NAME}/logs \ + --volume $(realpath -P ${DB_FOLDER}):/opt/${NAME}/${DB_FOLDER} \ --env-file .env.prod \ --name ${NAME} \ ${NAME}:latest diff --git a/src/bot/root_pythia_bot.py b/src/bot/root_pythia_bot.py index db9ac20..3e1a230 100644 --- a/src/bot/root_pythia_bot.py +++ b/src/bot/root_pythia_bot.py @@ -11,7 +11,7 @@ from api.rate_limiter import RateLimiter from bot.custom_help_command import RootPythiaHelpCommand from bot.root_pythia_cogs import RootPythiaCommands -from bot.dummy_db_manager import DummyDBManager +from database import DatabaseManager CHANNEL_ID = getenv("CHANNEL_ID") @@ -74,12 +74,15 @@ def is_my_channel(ctx): @BOT.event async def on_ready(): # is this call secure?? - logging.debug("channel id: %s", CHANNEL_ID) + BOT.logger.debug("channel id: %s", CHANNEL_ID) - # Create Rate Limiter, API Manager, and DB Manager objects + # Create Rate Limiter, API Manager, and Database Manager objects rate_limiter = RateLimiter() + BOT.logger.debug("Successfully created RateLimiter") api_manager = RootMeAPIManager(rate_limiter) - db_manager = DummyDBManager(api_manager) + BOT.logger.debug("Successfully created RootMeAPIManager") + db_manager = DatabaseManager(api_manager) + BOT.logger.debug("Successfully created DatabaseManager") # Fetch main channel and send initialization message BOT.channel = await BOT.fetch_channel(CHANNEL_ID) @@ -92,8 +95,9 @@ async def on_ready(): @BOT.event async def on_error(event, *args, **kwargs): if event == "on_ready": - BOT.logger.error( - "Event '%s' failed (probably from invalid channel ID), close connection and exit...", + BOT.logger.exception("Unhandled exception in 'on_ready' event:") + BOT.logger.critical( + "Event '%s' failed, please check debug logs, close connection and exit...", event, ) await BOT.close() diff --git a/src/classes/user.py b/src/classes/user.py index 27d5867..ba36d3a 100644 --- a/src/classes/user.py +++ b/src/classes/user.py @@ -5,7 +5,7 @@ class InvalidUserData(Exception): class User: """Class for the User object""" - def __init__(self, data: dict): + def __init__(self, data: [dict, tuple]): """ idx: int, username: str, @@ -13,9 +13,15 @@ def __init__(self, data: dict): rank: int, solves: int, """ - parsed_data = User.parse_rootme_user_data(data) + if isinstance(data, dict): + parsed_data = User.parse_rootme_user_data(data) + elif isinstance(data, tuple): + parsed_data = dict(zip(User.keys(), list(data))) - self.idx = parsed_data["idx"] + self.as_dict = parsed_data + self.as_tuple = tuple(parsed_data.values()) + + self.idx = parsed_data["id"] self.username = parsed_data["username"] self.score = parsed_data["score"] self.rank = parsed_data["rank"] @@ -25,7 +31,7 @@ def __init__(self, data: dict): @staticmethod def keys(): - return ["idx", "username", "score", "rank", "nb_solves"] + return ["id", "username", "score", "rank", "nb_solves"] # TODO: move these static methods to rootme_api, it make more sense # or create a Parser object? we could then have a RootMeParser and a HTBParser @@ -57,6 +63,12 @@ def parse_rootme_user_solves_and_yield(data): solve_id = int(solve["id_challenge"]) yield solve_id + def to_dict(self): + return self.as_dict + + def to_tuple(self): + return self.as_tuple + def __repr__(self): return ( f"User(id={self.idx}, username={self.username}, " @@ -66,6 +78,12 @@ def __repr__(self): def __str__(self): return f"{self.username} #{self.idx}" + def __eq__(self, other) -> bool: + if not isinstance(other, User): + # don't attempt to compare against unrelated types + return NotImplemented + return self.as_tuple == other.as_tuple + def update_new_solves(self, raw_user_data): parsed_data = User.parse_rootme_user_data(raw_user_data) parsed_nb_solves = parsed_data["nb_solves"] diff --git a/src/database/__init__.py b/src/database/__init__.py new file mode 100644 index 0000000..f5a9dfc --- /dev/null +++ b/src/database/__init__.py @@ -0,0 +1 @@ +from database.db_manager import DatabaseManager diff --git a/src/bot/dummy_db_manager.py b/src/database/db_manager.py similarity index 50% rename from src/bot/dummy_db_manager.py rename to src/database/db_manager.py index 79b5c91..8d8c931 100644 --- a/src/bot/dummy_db_manager.py +++ b/src/database/db_manager.py @@ -1,9 +1,21 @@ import logging - +import sqlite3 +from os import getenv, path + +from database.db_structure import ( + sql_create_user_table, + sql_add_user, + sql_get_user, + sql_get_users, + sql_has_user, +) from classes import User from classes import Challenge +DB_FILE_NAME = "RootPythia.db" + + class InvalidUser(Exception): def __init__(self, idx=None, message=None): self.idx = idx @@ -18,41 +30,74 @@ def __init__(self, idx=None, message=None): super().__init__(self.message) -class DummyDBManager: +class DatabaseManager: def __init__(self, api_manager): - self.users = [] + self.logger = logging.getLogger(__name__) + + self.DB_FOLDER = getenv("DB_FOLDER") + if self.DB_FOLDER is None or not path.isdir(self.DB_FOLDER): + self.logger.critical("DB_FOLDER: '%s', is not a directory", self.DB_FOLDER) + raise OSError(f"DB_FOLDER: '{self.DB_FOLDER}', is not a directory") + + # Init Connection object allowing interaction with the database + db_file_path = path.join(self.DB_FOLDER, DB_FILE_NAME) + self.db = sqlite3.connect(db_file_path) + self.logger.info("Succesfully connected to database '%s'", db_file_path) + self._init_db() + self.api_manager = api_manager - self.logger = logging.getLogger(__name__) + def _init_db(self): + """Private function that initializes the database tables (see db_strucure.py)""" + cur = self.db.cursor() + cur.execute(sql_create_user_table) + cur.close() async def add_user(self, idx): """Call the API Manager to get a user by his id then create a User object and store it""" + cur = self.db.cursor() # Check wether the user is already added if self.has_user(idx): return None + # Retreive information from RootMe API raw_user_data = await self.api_manager.get_user_by_id(idx) - user = User(raw_user_data) - self.users.append(user) - self.logger.debug("add user '%s'", repr(user)) + + cur.execute(sql_add_user, user.to_tuple()) + self.db.commit() + self.logger.debug("Add user '%s'", repr(user)) + cur.close() return user def has_user(self, idx): - return self.get_user(idx) is not None + cur = self.db.cursor() + res = cur.execute(sql_has_user, (idx, )).fetchone() + cur.close() + return res is not None def get_user(self, idx): """Retrieve the user object whose id matches 'id', None if not found""" - return next(filter(lambda user: user.idx == idx, self.users), None) + cur = self.db.cursor() + res = cur.execute(sql_get_user, (idx, )).fetchone() + if res is None: + return None + user = User(res) + cur.close() + return user def get_users(self): - return self.users + cur = self.db.cursor() + res = cur.execute(sql_get_users).fetchall() + users = [User(elt) for elt in res] + cur.close() + return users async def fetch_user_new_solves(self, idx): user = self.get_user(idx) if user is None: - raise InvalidUser(idx, "DummyDBManager.fetch_user_new_solves: User %s not in database") + raise InvalidUser(idx, "DatabaseManager.fetch_user_new_solves: User %s not in database") raw_user_data = await self.api_manager.get_user_by_id(idx) user.update_new_solves(raw_user_data) diff --git a/src/database/db_structure.py b/src/database/db_structure.py new file mode 100644 index 0000000..0095536 --- /dev/null +++ b/src/database/db_structure.py @@ -0,0 +1,19 @@ +sql_create_user_table = """ CREATE TABLE IF NOT EXISTS users ( + id integer PRIMARY KEY, + username text NOT NULL, + score int, + rank int, + nb_solves int + );""" + + +sql_add_user = """INSERT INTO users(id,username,score,rank,nb_solves) VALUES(?,?,?,?,?);""" + + +sql_get_user = """SELECT * FROM users WHERE id=?;""" + + +sql_get_users = """SELECT * FROM users;""" + + +sql_has_user = """SELECT * FROM users WHERE id=(?)""" diff --git a/tests/conftest.py b/tests/conftest.py index 73f85a4..b27e923 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ from bot.root_pythia_cogs import RootPythiaCommands from bot.root_pythia_cogs import NAME as COG_NAME -from bot.dummy_db_manager import DummyDBManager +from database import DatabaseManager # these plugins will be automatically imported by pytest pytest_plugins = ["pytest_mock", "pytest_asyncio"] @@ -57,17 +57,19 @@ def mock_rootme_api_manager(mocker): @pytest.fixture -def mock_dummy_db_manager(mock_rootme_api_manager): +def mock_database_manager(mock_rootme_api_manager, monkeypatch, tmp_path): rootme_api_manager = mock_rootme_api_manager + monkeypatch.setenv("DB_FOLDER", str(tmp_path)) - db = DummyDBManager(rootme_api_manager) + db = DatabaseManager(rootme_api_manager) yield db + db.db.close() # this pytest_asyncio decorator allows to automatically await async fixture before passing them # to tests @pytest_asyncio.fixture -async def config_bot(mock_dummy_db_manager, null_logger): +async def config_bot(mock_database_manager, null_logger): intents = discord.Intents.default() intents.members = True intents.message_content = True @@ -76,7 +78,7 @@ async def config_bot(mock_dummy_db_manager, null_logger): _bot.logger = null_logger await _bot._async_setup_hook() - await _bot.add_cog(RootPythiaCommands(_bot, mock_dummy_db_manager)) + await _bot.add_cog(RootPythiaCommands(_bot, mock_database_manager)) dpytest.configure(_bot) diff --git a/tests/test_cog.py b/tests/test_cog.py index f35113a..8f54128 100644 --- a/tests/test_cog.py +++ b/tests/test_cog.py @@ -88,7 +88,7 @@ async def test_command_exception_handling(config_bot, mocker): # patching this method in particular is arbitrary: the only goal is to raise an exception # during a command execution # I chose the "getuser" command purely arbitrary and this could be changed in the future - mocker.patch("bot.dummy_db_manager.DummyDBManager.get_user", side_effect=Exception) + mocker.patch("database.DatabaseManager.get_user", side_effect=Exception) # Trigger test with pytest.raises(Exception): @@ -100,8 +100,8 @@ async def test_command_exception_handling(config_bot, mocker): @pytest.mark.asyncio async def test_loop_exception_handling(config_bot, mocker): bot = config_bot - # patching "DummyDBManager.get_userS" here! - mocker.patch("bot.dummy_db_manager.DummyDBManager.get_users", side_effect=Exception) + # patching "DatabaseManager.get_userS" here! + mocker.patch("database.DatabaseManager.get_users", side_effect=Exception) # changing the check_new_solves loop delay interval to speed up the test cog = bot.get_cog(pytest.COG_NAME) cog.check_new_solves.change_interval(seconds=0.1) diff --git a/tests/test_dummy_db_manager.py b/tests/test_database_manager.py similarity index 72% rename from tests/test_dummy_db_manager.py rename to tests/test_database_manager.py index b7c7a94..3695666 100644 --- a/tests/test_dummy_db_manager.py +++ b/tests/test_database_manager.py @@ -2,13 +2,12 @@ from data.rootme_api_example_data import auteurs_with_score_zero_example_data -from bot.dummy_db_manager import DummyDBManager from classes import User @pytest.mark.asyncio -async def test_add_user(mock_dummy_db_manager): - db = mock_dummy_db_manager +async def test_add_user(mock_database_manager): + db = mock_database_manager # Trigger test # the user 0 doesn't exists on Root Me, and doesn't correspond to the id of @@ -22,8 +21,8 @@ async def test_add_user(mock_dummy_db_manager): @pytest.mark.asyncio -async def test_double_add_user(mock_dummy_db_manager, mocker): - db = mock_dummy_db_manager +async def test_double_add_user(mock_database_manager, mocker): + db = mock_database_manager # Trigger test # the 1 is on purpose the id of "g0uZ" auteur in tests/data/rootme_api_example_data to trigger @@ -42,8 +41,8 @@ async def test_double_add_user(mock_dummy_db_manager, mocker): (1, True), ], ) -async def test_has_user(idx, expected, mock_dummy_db_manager): - db = mock_dummy_db_manager +async def test_has_user(idx, expected, mock_database_manager): + db = mock_database_manager # Trigger test await db.add_user(idx) @@ -53,27 +52,26 @@ async def test_has_user(idx, expected, mock_dummy_db_manager): @pytest.mark.asyncio -async def test_get_user(mock_dummy_db_manager): - db = mock_dummy_db_manager +async def test_get_user(mock_database_manager): + db = mock_database_manager # Trigger test added_user = await db.add_user(1) got_user = db.get_user(1) - assert added_user is got_user + assert added_user == got_user @pytest.mark.asyncio -async def test_add_user_with_empty_position(mock_rootme_api_manager): +async def test_add_user_with_empty_position(mock_database_manager): # Regression test introduced with https://github.com/iScsc/RootPythia/pull/38 # the position was always parsed as an int however if the user's score is 0 # the root me api returns an empty position, causing the bug # this test ensures this bug doesn't reappear # Create an api manager returning the right data, and the associated db object - api_manager = mock_rootme_api_manager - api_manager.get_user_by_id.return_value = auteurs_with_score_zero_example_data - db = DummyDBManager(api_manager) + db = mock_database_manager + db.api_manager.get_user_by_id.return_value = auteurs_with_score_zero_example_data # Trigger test added_user = await db.add_user(819227)