From 84ffd335c39c49736b9c0952b0e981d9f48ae13b Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 12 Nov 2024 14:21:52 -0800 Subject: [PATCH 1/7] upgrade pymongo && add upgrade latest mongo into workflows --- .github/workflows/test.yml | 2 ++ requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f28a7fd..8886e7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,8 @@ jobs: include: - python: '3.7.13' mongo: 'mongodb-linux-x86_64-3.6.2' + - python: '3.7.13' + mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' steps: - name: Check out GitHub repo diff --git a/requirements.txt b/requirements.txt index 81ae942..db8c92f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ requests==2.20.0 gunicorn==19.9.0 gevent==1.3.7 cachetools==2.1.0 -pymongo==3.7.2 +pymongo==4.7.2 redis==2.10.6 flask-cors==3.0.6 From 8c15b2e884e1bfc46ba1effaa42f9a129473f515 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 12 Nov 2024 14:35:38 -0800 Subject: [PATCH 2/7] remove deprecated .count() --- feeds/storage/mongodb/timeline_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feeds/storage/mongodb/timeline_storage.py b/feeds/storage/mongodb/timeline_storage.py index 9c2e6c7..740fa80 100644 --- a/feeds/storage/mongodb/timeline_storage.py +++ b/feeds/storage/mongodb/timeline_storage.py @@ -69,7 +69,7 @@ def get_unseen_count(self) -> int: "unseen": self._user_doc(), "expires": {"$gt": now} } - return coll.find(query).count() + return coll.count_documents(query) def _user_doc(self) -> Dict[str, str]: return { From d82d45d9f8d8ccb009435cf460e5bc2c500c8d70 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 12 Nov 2024 14:58:44 -0800 Subject: [PATCH 3/7] update MongoController --- test/mongo_controller.py | 46 +++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/test/mongo_controller.py b/test/mongo_controller.py index f44f4ab..7e93b8c 100644 --- a/test/mongo_controller.py +++ b/test/mongo_controller.py @@ -12,6 +12,9 @@ import semver import test.util as test_util +# semver parser +sver = semver.VersionInfo.parse + class MongoController: """ @@ -50,9 +53,13 @@ def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool=Fa os.makedirs(data_dir) self.port = test_util.find_free_port() + mongodb_ver = self.get_mongodb_version(mongoexe) + + command = [str(mongoexe), '--port', str(self.port), '--dbpath', str(data_dir)] + + if sver(mongodb_ver) < sver('6.1.0'): + command.extend(['--nojournal']) - command = [str(mongoexe), '--port', str(self.port), '--dbpath', str(data_dir), - '--nojournal'] if use_wired_tiger: command.extend(['--storageEngine', 'wiredTiger']) @@ -60,17 +67,36 @@ def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool=Fa self._proc = subprocess.Popen(command, stdout=self._outfile, stderr=subprocess.STDOUT) time.sleep(1) # wait for server to start up - self.client = MongoClient('localhost', self.port) - # check that the server is up. See - # https://api.mongodb.com/python/3.7.0/api/pymongo/mongo_client.html - # #pymongo.mongo_client.MongoClient - self.client.admin.command('ismaster') + + try: + self.client: MongoClient = MongoClient('localhost', self.port) + # This line will raise an exception if the server is down + server_info = self.client.server_info() + except Exception as e: + raise ValueError("MongoDB server is down") from e # get some info about the db - self.db_version = self.client.server_info()['version'] + self.db_version = server_info['version'] self.index_version = 2 if (semver.compare(self.db_version, '3.4.0') >= 0) else 1 - self.includes_system_indexes = (semver.compare(self.db_version, '3.2.0') < 0 - and not use_wired_tiger) + self.includes_system_indexes = ( + semver.compare(self.db_version, '3.2.0') < 0 and not use_wired_tiger + ) + + def get_mongodb_version(self, mongoexe: Path) -> str: + try: + process = subprocess.Popen( + [str(mongoexe), '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = process.communicate() + + if process.returncode == 0: + version_line = stdout.decode().split('\n')[0] + mongodb_version = version_line.split()[2][1:] + return mongodb_version.strip() + else: + raise ValueError(f"Error: {stderr.decode()}") + except Exception as e: + raise ValueError("Failed to get MongoDB version") from e def destroy(self, delete_temp_files: bool) -> None: """ From 012822a958c876dca7ed52e7c3938d7d35fdaf6d Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 13 Nov 2024 13:31:05 -0800 Subject: [PATCH 4/7] update pymango to 4.10.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index db8c92f..301c176 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ requests==2.20.0 gunicorn==19.9.0 gevent==1.3.7 cachetools==2.1.0 -pymongo==4.7.2 +pymongo==4.10.1 redis==2.10.6 flask-cors==3.0.6 From a1d202c33045b05c0c50dffff4db9ac321f52857 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 13 Nov 2024 13:37:11 -0800 Subject: [PATCH 5/7] revert back to 4.7.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 301c176..db8c92f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ requests==2.20.0 gunicorn==19.9.0 gevent==1.3.7 cachetools==2.1.0 -pymongo==4.10.1 +pymongo==4.7.2 redis==2.10.6 flask-cors==3.0.6 From 69f046d4794d95e3652a6a014c7b08199194e056 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 13 Nov 2024 13:39:36 -0800 Subject: [PATCH 6/7] update python version & pymango --- .github/workflows/test.yml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8886e7e..522abcf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,9 +22,9 @@ jobs: fail-fast: false matrix: include: - - python: '3.7.13' + - python: '3.9.19' mongo: 'mongodb-linux-x86_64-3.6.2' - - python: '3.7.13' + - python: '3.9.19' mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' steps: diff --git a/requirements.txt b/requirements.txt index db8c92f..301c176 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ requests==2.20.0 gunicorn==19.9.0 gevent==1.3.7 cachetools==2.1.0 -pymongo==4.7.2 +pymongo==4.10.1 redis==2.10.6 flask-cors==3.0.6 From 389ca25e9515c77d5a4147493b4bd80186479ce6 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 13 Nov 2024 14:18:11 -0800 Subject: [PATCH 7/7] revert back python versio & pymango --- .github/workflows/test.yml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 522abcf..8886e7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,9 +22,9 @@ jobs: fail-fast: false matrix: include: - - python: '3.9.19' + - python: '3.7.13' mongo: 'mongodb-linux-x86_64-3.6.2' - - python: '3.9.19' + - python: '3.7.13' mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' steps: diff --git a/requirements.txt b/requirements.txt index 301c176..db8c92f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ requests==2.20.0 gunicorn==19.9.0 gevent==1.3.7 cachetools==2.1.0 -pymongo==4.10.1 +pymongo==4.7.2 redis==2.10.6 flask-cors==3.0.6