From 4460adf6e7a20268af90f9ceb856231928cda6d1 Mon Sep 17 00:00:00 2001 From: Amogh M Aradhya Date: Fri, 21 Jul 2023 15:18:56 +0530 Subject: [PATCH] API support for the Mobile Number Revocation List (MNRL) (#1810) This commit adds support for retrieving expired phone numbers from the monthly Mobile Number Revocation List, and comparing against existing phone numbers in the `phone_number` table. It does not (yet) implement workflow for processing a match, or for retrying MNRL API access when their servers are down (which has happened a lot when working on this feature). Co-authored-by: Kiran Jonnalagadda --- funnel/cli/periodic/__init__.py | 3 +- funnel/cli/periodic/mnrl.py | 280 +++++++++++++++++++++++++ funnel/cli/periodic/stats.py | 9 +- funnel/models/phone_number.py | 13 ++ requirements/base.in | 2 +- requirements/base.py37.txt | 15 +- requirements/base.txt | 17 +- requirements/dev.in | 1 + requirements/dev.py37.txt | 16 +- requirements/dev.txt | 16 +- requirements/test.in | 1 + requirements/test.py37.txt | 6 +- requirements/test.txt | 4 +- sample.env | 2 + tests/unit/cli/periodic_mnrl_test.py | 133 ++++++++++++ tests/unit/models/phone_number_test.py | 26 +++ 16 files changed, 503 insertions(+), 41 deletions(-) create mode 100644 funnel/cli/periodic/mnrl.py create mode 100644 tests/unit/cli/periodic_mnrl_test.py diff --git a/funnel/cli/periodic/__init__.py b/funnel/cli/periodic/__init__.py index d412dbd3c..7d0e34674 100644 --- a/funnel/cli/periodic/__init__.py +++ b/funnel/cli/periodic/__init__.py @@ -8,7 +8,6 @@ 'periodic', help="Periodic tasks from cron (with recommended intervals)" ) -from . import stats # isort:skip # noqa: F401 -from . import notification # isort:skip # noqa: F401 +from . import mnrl, notification, stats # noqa: F401 app.cli.add_command(periodic) diff --git a/funnel/cli/periodic/mnrl.py b/funnel/cli/periodic/mnrl.py new file mode 100644 index 000000000..6b1827d4e --- /dev/null +++ b/funnel/cli/periodic/mnrl.py @@ -0,0 +1,280 @@ +""" +Validate Indian phone numbers against the Mobile Number Revocation List. + +About MNRL: https://mnrl.trai.gov.in/homepage +API details (requires login): https://mnrl.trai.gov.in/api_details, contents reproduced +here: + +.. list-table:: API Description + :header-rows: 1 + + * - № + - API Name + - API URL + - Method + - Remark + * - 1 + - Get MNRL Status + - https://mnrl.trai.gov.in/api/mnrl/status/{key} + - GET + - Returns the current status of MNRL. + * - 2 + - Get MNRL Files + - https://mnrl.trai.gov.in/api/mnrl/files/{key} + - GET + - Returns the summary of MNRL files, to be used for further API calls to get the + list of mobile numbers or download the file. + * - 3 + - Get MNRL + - https://mnrl.trai.gov.in/api/mnrl/json/{file_name}/{key} + - GET + - Returns the list of mobile numbers of the requested (.json) file. + * - 4 + - Download MNRL + - https://mnrl.trai.gov.in/api/mnrl/download/{file_name}/{key} + - GET + - Can be used to download the file. (xlsx, pdf, json, rar) +""" + +import asyncio +from typing import List, Set, Tuple + +import click +import httpx +import ijson +from rich import get_console, print as rprint +from rich.progress import Progress + +from ... import app +from ...models import PhoneNumber, UserPhone, db +from . import periodic + + +class KeyInvalidError(ValueError): + """MNRL API key is invalid.""" + + message = "MNRL API key is invalid" + + +class KeyExpiredError(ValueError): + """MNRL API key has expired.""" + + message = "MNRL API key has expired" + + +class AsyncStreamAsFile: + """Provide a :meth:`read` interface to a HTTPX async stream response for ijson.""" + + def __init__(self, response: httpx.Response) -> None: + self.data = response.aiter_bytes() + + async def read(self, size: int) -> bytes: + """Async read method for ijson (which expects this to be 'read' not 'aread').""" + if size == 0: + # ijson calls with size 0 and expect b'', using it only to + # print a warning if the return value is '' (str instead of bytes) + return b'' + # Python >= 3.10 supports `return await anext(self.data, b'')` but for older + # versions we need this try/except block + try: + # Ignore size parameter since anext doesn't take it + # pylint: disable=unnecessary-dunder-call + return await self.data.__anext__() + except StopAsyncIteration: + return b'' + + +async def get_existing_phone_numbers(prefix: str) -> Set[str]: + """Async wrapper for PhoneNumber.get_numbers.""" + # TODO: This is actually an async-blocking call. We need full stack async here. + return PhoneNumber.get_numbers(prefix=prefix, remove=True) + + +async def get_mnrl_json_file_list(apikey: str) -> List[str]: + """ + Return filenames for the currently published MNRL JSON files. + + TRAI publishes the MNRL as a monthly series of files in Excel, PDF and JSON + formats, of which we'll use JSON (plaintext format isn't offered). + """ + response = await httpx.AsyncClient(http2=True).get( + f'https://mnrl.trai.gov.in/api/mnrl/files/{apikey}', timeout=300 + ) + if response.status_code == 401: + raise KeyInvalidError() + if response.status_code == 407: + raise KeyExpiredError() + response.raise_for_status() + + result = response.json() + # Fallback tests for non-200 status codes in a 200 response (current API behaviour) + if result['status'] == 401: + raise KeyInvalidError() + if result['status'] == 407: + raise KeyExpiredError() + return [row['file_name'] for row in result['mnrl_files']['json']] + + +async def get_mnrl_json_file_numbers( + client: httpx.AsyncClient, apikey: str, filename: str +) -> Tuple[str, Set[str]]: + """Return phone numbers from an MNRL JSON file URL.""" + async with client.stream( + 'GET', + f'https://mnrl.trai.gov.in/api/mnrl/json/{filename}/{apikey}', + timeout=300, + ) as response: + response.raise_for_status() + # The JSON structure is {"payload": [{"n": "number"}, ...]} + # The 'item' in 'payload.item' is ijson's code for array elements + return filename, { + value + async for key, value in ijson.kvitems( + AsyncStreamAsFile(response), 'payload.item' + ) + if key == 'n' and value is not None + } + + +async def forget_phone_numbers(phone_numbers: Set[str], prefix: str) -> None: + """Mark phone numbers as forgotten.""" + for unprefixed in phone_numbers: + number = prefix + unprefixed + userphone = UserPhone.get(number) + if userphone is not None: + # TODO: Dispatch a notification to userphone.user, but since the + # notification will not know the phone number (it'll already be forgotten), + # we need a new db model to contain custom messages + # TODO: Also delay dispatch until the full MNRL scan is complete -- their + # backup contact phone number may also have expired. That means this + # function will create notifications and return them, leaving dispatch to + # the outermost function + rprint(f"{userphone} - owned by {userphone.user.pickername}") + # TODO: MNRL isn't foolproof. Don't delete! Instead, notify the user and + # only delete if they don't respond (How? Maybe delete and send them a + # re-add token?) + # db.session.delete(userphone) + phone_number = PhoneNumber.get(number) + if phone_number is not None: + rprint( + f"{phone_number} - since {phone_number.created_at:%Y-%m-%d}, updated" + f" {phone_number.updated_at:%Y-%m-%d}" + ) + # phone_number.mark_forgotten() + db.session.commit() + + +async def process_mnrl_files( + apikey: str, + existing_phone_numbers: Set[str], + phone_prefix: str, + mnrl_filenames: List[str], +) -> Tuple[Set[str], int, int]: + """ + Scan all MNRL files and return a tuple of results. + + :return: Tuple of number to be revoked (set), total expired numbers in the MNRL, + and count of failures when accessing the MNRL lists + """ + revoked_phone_numbers: Set[str] = set() + mnrl_total_count = 0 + failures = 0 + async_tasks: Set[asyncio.Task] = set() + with Progress(transient=True) as progress: + ptask = progress.add_task( + f"Processing {len(mnrl_filenames)} MNRL files", total=len(mnrl_filenames) + ) + async with httpx.AsyncClient( + http2=True, limits=httpx.Limits(max_connections=3) + ) as client: + for future in asyncio.as_completed( + [ + get_mnrl_json_file_numbers(client, apikey, filename) + for filename in mnrl_filenames + ] + ): + try: + filename, mnrl_set = await future + except httpx.HTTPError as exc: + progress.advance(ptask) + failures += 1 + # Extract filename from the URL (ends with /filename/apikey) as we + # can't get any context from asyncio.as_completed's future + filename = exc.request.url.path.split('/')[-2] + progress.update(ptask, description=f"Error in {filename}...") + if isinstance(exc, httpx.HTTPStatusError): + rprint( + f"[red]{filename}: Server returned HTTP status code" + f" {exc.response.status_code}" + ) + else: + rprint(f"[red]{filename}: Failed with {exc!r}") + else: + progress.advance(ptask) + mnrl_total_count += len(mnrl_set) + progress.update(ptask, description=f"Processing {filename}...") + found_expired = existing_phone_numbers.intersection(mnrl_set) + if found_expired: + revoked_phone_numbers.update(found_expired) + rprint( + f"[blue]{filename}: {len(found_expired):,} matches in" + f" {len(mnrl_set):,} total" + ) + async_tasks.add( + asyncio.create_task( + forget_phone_numbers(found_expired, phone_prefix) + ) + ) + else: + rprint( + f"[cyan]{filename}: No matches in {len(mnrl_set):,} total" + ) + + # Await all the background tasks + for task in async_tasks: + try: + # TODO: Change this to `notifications = await task` then return them too + await task + except Exception as exc: # noqa: B902 # pylint: disable=broad-except + app.logger.exception("%s in forget_phone_numbers", repr(exc)) + return revoked_phone_numbers, mnrl_total_count, failures + + +async def process_mnrl(apikey: str) -> None: + """Process MNRL data using the API key.""" + console = get_console() + phone_prefix = '+91' + task_numbers = asyncio.create_task(get_existing_phone_numbers(phone_prefix)) + task_files = asyncio.create_task(get_mnrl_json_file_list(apikey)) + with console.status("Loading phone numbers..."): + existing_phone_numbers = await task_numbers + rprint(f"Evaluating {len(existing_phone_numbers):,} phone numbers for expiry") + try: + with console.status("Getting MNRL download list..."): + mnrl_filenames = await task_files + except httpx.HTTPError as exc: + err = f"{exc!r} in MNRL API getting download list" + rprint(f"[red]{err}") + raise click.ClickException(err) + + revoked_phone_numbers, mnrl_total_count, failures = await process_mnrl_files( + apikey, existing_phone_numbers, phone_prefix, mnrl_filenames + ) + rprint( + f"Processed {mnrl_total_count:,} expired phone numbers in MNRL with" + f" {failures:,} failure(s) and revoked {len(revoked_phone_numbers):,} phone" + f" numbers" + ) + + +@periodic.command('mnrl') +def periodic_mnrl() -> None: + """Remove expired phone numbers using TRAI's MNRL (1 week).""" + apikey = app.config.get('MNRL_API_KEY') + if not apikey: + raise click.UsageError("App config is missing `MNRL_API_KEY`") + try: + asyncio.run(process_mnrl(apikey)) + except (KeyInvalidError, KeyExpiredError) as exc: + app.logger.error(exc.message) + raise click.ClickException(exc.message) from exc diff --git a/funnel/cli/periodic/stats.py b/funnel/cli/periodic/stats.py index 529677a3b..6ba0ccb1d 100644 --- a/funnel/cli/periodic/stats.py +++ b/funnel/cli/periodic/stats.py @@ -13,7 +13,6 @@ import httpx import pytz import telegram -from asgiref.sync import async_to_sync from dataclasses_json import DataClassJsonMixin from dateutil.relativedelta import relativedelta from furl import furl @@ -371,8 +370,6 @@ async def user_stats() -> Dict[str, ResourceStats]: # --- Commands ------------------------------------------------------------------------- -@periodic.command('dailystats') -@async_to_sync async def dailystats() -> None: """Publish daily stats to Telegram.""" if ( @@ -461,3 +458,9 @@ async def dailystats() -> None: disable_web_page_preview=True, message_thread_id=app.config.get('TELEGRAM_STATS_THREADID'), ) + + +@periodic.command('dailystats') +def periodic_dailystats() -> None: + """Publish daily stats to Telegram (midnight).""" + asyncio.run(dailystats()) diff --git a/funnel/models/phone_number.py b/funnel/models/phone_number.py index d16af0c74..7201b5fbf 100644 --- a/funnel/models/phone_number.py +++ b/funnel/models/phone_number.py @@ -705,6 +705,19 @@ def validate_for( return 'not_new' return None + @classmethod + def get_numbers(cls, prefix: str, remove: bool = True) -> Set[str]: + """Get all numbers with the given prefix as a Python set.""" + query = ( + cls.query.filter(cls.number.startswith(prefix)) + .options(sa.orm.load_only(cls.number)) + .yield_per(1000) + ) + if remove: + skip = len(prefix) + return {r.number[skip:] for r in query} + return {r.number for r in query} + @declarative_mixin class PhoneNumberMixin: diff --git a/requirements/base.in b/requirements/base.in index 2f8c08ed6..7366ebcdb 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,7 +2,6 @@ -e file:./build/dependencies/coaster alembic argon2-cffi -asgiref Babel base58 bcrypt @@ -33,6 +32,7 @@ html2text httpx[http2] icalendar idna +ijson itsdangerous linkify-it-py markdown-it-py<3.0 # Breaks our plugins, needs refactoring diff --git a/requirements/base.py37.txt b/requirements/base.py37.txt index d422b5064..80f23ae97 100644 --- a/requirements/base.py37.txt +++ b/requirements/base.py37.txt @@ -1,4 +1,4 @@ -# SHA1:4278e04aa69612d326a315d85b7b37f7742d3dc7 +# SHA1:2d92258ba7951d85a229036d73ae5835e50f0b5c # # This file is autogenerated by pip-compile-multi # To update, run: @@ -35,8 +35,6 @@ argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 # via rq-dashboard -asgiref==3.7.2 - # via -r requirements/base.in async-timeout==4.0.2 # via # aiohttp @@ -70,9 +68,9 @@ blinker==1.6.2 # -r requirements/base.in # baseframe # coaster -boto3==1.28.7 +boto3==1.28.8 # via -r requirements/base.in -botocore==1.31.7 +botocore==1.31.8 # via # boto3 # s3transfer @@ -114,7 +112,7 @@ cssselect==1.2.0 # via premailer cssutils==2.7.1 # via premailer -dataclasses-json==0.5.12 +dataclasses-json==0.5.13 # via -r requirements/base.in dnspython==2.3.0 # via @@ -235,6 +233,8 @@ idna==3.4 # requests # tldextract # yarl +ijson==3.2.2 + # via -r requirements/base.in importlib-metadata==6.7.0 # via # alembic @@ -328,7 +328,7 @@ packaging==23.1 # marshmallow passlib==1.7.4 # via -r requirements/base.in -phonenumbers==8.13.16 +phonenumbers==8.13.17 # via -r requirements/base.in premailer==3.10.0 # via -r requirements/base.in @@ -508,7 +508,6 @@ typing-extensions==4.7.1 # anyio # argon2-cffi # arrow - # asgiref # async-timeout # baseframe # coaster diff --git a/requirements/base.txt b/requirements/base.txt index 5cbd79bac..c49687996 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:4278e04aa69612d326a315d85b7b37f7742d3dc7 +# SHA1:2d92258ba7951d85a229036d73ae5835e50f0b5c # # This file is autogenerated by pip-compile-multi # To update, run: @@ -35,8 +35,6 @@ argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 # via rq-dashboard -asgiref==3.7.2 - # via -r requirements/base.in async-timeout==4.0.2 # via # aiohttp @@ -65,9 +63,9 @@ blinker==1.6.2 # baseframe # coaster # flask -boto3==1.28.7 +boto3==1.28.8 # via -r requirements/base.in -botocore==1.31.7 +botocore==1.31.8 # via # boto3 # s3transfer @@ -109,7 +107,7 @@ cssselect==1.2.0 # via premailer cssutils==2.7.1 # via premailer -dataclasses-json==0.5.12 +dataclasses-json==0.5.13 # via -r requirements/base.in dnspython==2.4.0 # via @@ -230,6 +228,8 @@ idna==3.4 # requests # tldextract # yarl +ijson==3.2.2 + # via -r requirements/base.in importlib-metadata==6.8.0 # via # flask @@ -276,7 +276,7 @@ markupsafe==2.1.3 # mako # werkzeug # wtforms -marshmallow==3.19.0 +marshmallow==3.20.1 # via dataclasses-json maxminddb==2.4.0 # via geoip2 @@ -312,7 +312,7 @@ packaging==23.1 # marshmallow passlib==1.7.4 # via -r requirements/base.in -phonenumbers==8.13.16 +phonenumbers==8.13.17 # via -r requirements/base.in premailer==3.10.0 # via -r requirements/base.in @@ -488,7 +488,6 @@ typing-extensions==4.7.1 # via # -r requirements/base.in # alembic - # asgiref # baseframe # coaster # psycopg diff --git a/requirements/dev.in b/requirements/dev.in index d49d8d2a9..90870123e 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -32,6 +32,7 @@ toml tomli types-chevron types-geoip2 +types-mock types-python-dateutil types-pytz types-redis diff --git a/requirements/dev.py37.txt b/requirements/dev.py37.txt index 2acd8ba78..2197793e2 100644 --- a/requirements/dev.py37.txt +++ b/requirements/dev.py37.txt @@ -1,4 +1,4 @@ -# SHA1:8f1b556d0dcd9ac078682465a2238fa29b4784ef +# SHA1:8d67d5bb6492c975276e27d9da9f9b0a0f5b7d77 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -149,7 +149,7 @@ typed-ast==1.5.5 # black # flake8-annotations # mypy -types-chevron==0.14.2.4 +types-chevron==0.14.2.5 # via -r requirements/dev.in types-geoip2==3.0.0 # via -r requirements/dev.in @@ -157,17 +157,19 @@ types-ipaddress==1.0.8 # via types-maxminddb types-maxminddb==1.5.0 # via types-geoip2 -types-pyopenssl==23.2.0.1 +types-mock==5.1.0.1 + # via -r requirements/dev.in +types-pyopenssl==23.2.0.2 # via types-redis -types-python-dateutil==2.8.19.13 +types-python-dateutil==2.8.19.14 # via -r requirements/dev.in types-pytz==2023.3.0.0 # via -r requirements/dev.in -types-redis==4.6.0.2 +types-redis==4.6.0.3 # via -r requirements/dev.in -types-requests==2.31.0.1 +types-requests==2.31.0.2 # via -r requirements/dev.in -types-urllib3==1.26.25.13 +types-urllib3==1.26.25.14 # via types-requests virtualenv==20.24.1 # via pre-commit diff --git a/requirements/dev.txt b/requirements/dev.txt index 87e5631f7..ea0c3704e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ -# SHA1:8f1b556d0dcd9ac078682465a2238fa29b4784ef +# SHA1:8d67d5bb6492c975276e27d9da9f9b0a0f5b7d77 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -143,7 +143,7 @@ tokenize-rt==5.1.0 # via pyupgrade toposort==1.10 # via pip-compile-multi -types-chevron==0.14.2.4 +types-chevron==0.14.2.5 # via -r requirements/dev.in types-geoip2==3.0.0 # via -r requirements/dev.in @@ -151,17 +151,19 @@ types-ipaddress==1.0.8 # via types-maxminddb types-maxminddb==1.5.0 # via types-geoip2 -types-pyopenssl==23.2.0.1 +types-mock==5.1.0.1 + # via -r requirements/dev.in +types-pyopenssl==23.2.0.2 # via types-redis -types-python-dateutil==2.8.19.13 +types-python-dateutil==2.8.19.14 # via -r requirements/dev.in types-pytz==2023.3.0.0 # via -r requirements/dev.in -types-redis==4.6.0.2 +types-redis==4.6.0.3 # via -r requirements/dev.in -types-requests==2.31.0.1 +types-requests==2.31.0.2 # via -r requirements/dev.in -types-urllib3==1.26.25.13 +types-urllib3==1.26.25.14 # via types-requests virtualenv==20.24.1 # via pre-commit diff --git a/requirements/test.in b/requirements/test.in index 28f7bb9b3..481bcca33 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -4,6 +4,7 @@ colorama coverage coveralls lxml +mock;python_version=="3.7" Pygments pytest pytest-asyncio diff --git a/requirements/test.py37.txt b/requirements/test.py37.txt index 2aa8b0334..9f1899fb2 100644 --- a/requirements/test.py37.txt +++ b/requirements/test.py37.txt @@ -1,4 +1,4 @@ -# SHA1:f1dd8d59a43608d547da08b1d958365a3ca6e564 +# SHA1:210785110b0bd0694e5a75e2f1c15924908a21c4 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -27,6 +27,8 @@ docopt==0.6.2 # via coveralls iniconfig==2.0.0 # via pytest +mock==5.1.0 ; python_version == "3.7" + # via -r requirements/test.in outcome==1.2.0 # via trio parse==1.19.1 @@ -82,7 +84,7 @@ pytest-variables==3.0.0 # via pytest-selenium requests-mock==1.11.0 # via -r requirements/test.in -respx==0.20.1 +respx==0.20.2 # via -r requirements/test.in selenium==4.9.1 # via diff --git a/requirements/test.txt b/requirements/test.txt index 2acc990c2..367974ae4 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,4 @@ -# SHA1:f1dd8d59a43608d547da08b1d958365a3ca6e564 +# SHA1:210785110b0bd0694e5a75e2f1c15924908a21c4 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -82,7 +82,7 @@ pytest-variables==3.0.0 # via pytest-selenium requests-mock==1.11.0 # via -r requirements/test.in -respx==0.20.1 +respx==0.20.2 # via -r requirements/test.in selenium==4.9.1 # via diff --git a/sample.env b/sample.env index f9c49b8d5..7c1fdc651 100644 --- a/sample.env +++ b/sample.env @@ -171,6 +171,8 @@ FLASK_GOOGLE_MAPS_API_KEY= FLASK_RECAPTCHA_USE_SSL=true FLASK_RECAPTCHA_PUBLIC_KEY=null FLASK_RECAPTCHA_PRIVATE_KEY=null +# TRAI Mobile Number Revocation List (MNRL) for expired phone numbers +FLASK_MNRL_API_KEY=null # --- SMS integrations # Exotel (for SMS to Indian numbers; primary) diff --git a/tests/unit/cli/periodic_mnrl_test.py b/tests/unit/cli/periodic_mnrl_test.py new file mode 100644 index 000000000..738845af6 --- /dev/null +++ b/tests/unit/cli/periodic_mnrl_test.py @@ -0,0 +1,133 @@ +"""Tests for the periodic CLI stats commands.""" +# pylint: disable=redefined-outer-name + +from __future__ import annotations + +from unittest.mock import patch + +import httpx +import pytest +from click.testing import CliRunner +from respx import MockRouter + +from funnel.cli.periodic import mnrl as cli_mnrl, periodic + +MNRL_FILES_URL = 'https://mnrl.trai.gov.in/api/mnrl/files/{apikey}' +MNRL_JSON_URL = 'https://mnrl.trai.gov.in/api/mnrl/json/{filename}/{apikey}' + + +@pytest.fixture(scope='module') +def mnrl_files_response() -> bytes: + """Sample response for MNRL files API.""" + return ( + b'{"status":200,"message":"Success","mnrl_files":{' + b'"zip":[{"file_name":"test.rar","size_in_kb":1}],' + b'"json":[{"file_name":"test.json","size_in_kb":1}]' + b'}}' + ) + + +@pytest.fixture(scope='module') +def mnrl_files_response_keyinvalid() -> bytes: + return b'{"status": 401, "message": "Invalid Key"}' + + +@pytest.fixture(scope='module') +def mnrl_files_response_keyexpired() -> bytes: + return b'{"status": 407,"message": "Key Expired"}' + + +@pytest.fixture(scope='module') +def mnrl_json_response() -> bytes: + """Sample response for MNRL JSON API.""" + return ( + b'{"status":200,"file_name":"test.json",' + b'"payload":[{"n":"1111111111"},{"n":"2222222222"},{"n":"3333333333"}]}' + ) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize('status_code', [200, 401]) +async def test_mnrl_file_list_apikey_invalid( + respx_mock: MockRouter, mnrl_files_response_keyinvalid: bytes, status_code: int +) -> None: + """MNRL file list getter raises KeyInvalidError if the API key is invalid.""" + respx_mock.get(MNRL_FILES_URL.format(apikey='invalid')).mock( + return_value=httpx.Response(status_code, content=mnrl_files_response_keyinvalid) + ) + with pytest.raises(cli_mnrl.KeyInvalidError): + await cli_mnrl.get_mnrl_json_file_list('invalid') + + +@pytest.mark.asyncio() +@pytest.mark.parametrize('status_code', [200, 407]) +async def test_mnrl_file_list_apikey_expired( + respx_mock: MockRouter, mnrl_files_response_keyexpired: bytes, status_code: int +) -> None: + """MNRL file list getter raises KeyExpiredError if the API key has expired.""" + respx_mock.get(MNRL_FILES_URL.format(apikey='expired')).mock( + return_value=httpx.Response(status_code, content=mnrl_files_response_keyexpired) + ) + with pytest.raises(cli_mnrl.KeyExpiredError): + await cli_mnrl.get_mnrl_json_file_list('expired') + + +@pytest.mark.asyncio() +async def test_mnrl_file_list( + respx_mock: MockRouter, mnrl_files_response: bytes +) -> None: + """MNRL file list getter returns a list.""" + respx_mock.get(MNRL_FILES_URL.format(apikey='12345')).mock( + return_value=httpx.Response(200, content=mnrl_files_response) + ) + assert await cli_mnrl.get_mnrl_json_file_list('12345') == ['test.json'] + + +@pytest.mark.asyncio() +@pytest.mark.mock_config('app', {'MNRL_API_KEY': '12345'}) +async def test_mnrl_file_numbers( + respx_mock: MockRouter, mnrl_json_response: bytes +) -> None: + async with httpx.AsyncClient(http2=True) as client: + respx_mock.get(MNRL_JSON_URL.format(apikey='12345', filename='test.json')).mock( + return_value=httpx.Response(200, content=mnrl_json_response) + ) + assert await cli_mnrl.get_mnrl_json_file_numbers( + client, apikey='12345', filename='test.json' + ) == ('test.json', {'1111111111', '2222222222', '3333333333'}) + + +# --- CLI interface + + +@pytest.mark.mock_config('app', {'MNRL_API_KEY': ...}) +def test_cli_mnrl_needs_api_key() -> None: + """CLI command requires API key in config.""" + runner = CliRunner() + result = runner.invoke(periodic, ['mnrl']) + assert "App config is missing `MNRL_API_KEY`" in result.output + assert result.exit_code == 2 # click exits with 2 for UsageError + + +@pytest.mark.mock_config('app', {'MNRL_API_KEY': '12345'}) +def test_cli_mnrl_accepts_api_key() -> None: + """CLI command runs if an API key is present.""" + with patch('funnel.cli.periodic.mnrl.process_mnrl', return_value=None) as mock: + runner = CliRunner() + runner.invoke(periodic, ['mnrl']) + assert mock.called + + +@pytest.mark.mock_config('app', {'MNRL_API_KEY': 'invalid'}) +@pytest.mark.usefixtures('db_session') +def test_cli_mnrl_invalid_api_key( + respx_mock: MockRouter, mnrl_files_response_keyinvalid: bytes +) -> None: + """CLI command prints an exception given an invalid API key.""" + respx_mock.get(MNRL_FILES_URL.format(apikey='invalid')).mock( + return_value=httpx.Response(200, content=mnrl_files_response_keyinvalid) + ) + runner = CliRunner() + result = runner.invoke(periodic, ['mnrl']) + assert "key is invalid" in result.output + assert result.exit_code == 1 # click exits with 1 for ClickException diff --git a/tests/unit/models/phone_number_test.py b/tests/unit/models/phone_number_test.py index 87c5ff9cf..3a268972c 100644 --- a/tests/unit/models/phone_number_test.py +++ b/tests/unit/models/phone_number_test.py @@ -532,6 +532,32 @@ def test_phone_number_blocked() -> None: assert pn1.formatted == EXAMPLE_NUMBER_IN_FORMATTED +def test_get_numbers(db_session) -> None: + """Get phone numbers in bulk.""" + for number in ( + EXAMPLE_NUMBER_IN, + EXAMPLE_NUMBER_GB, + EXAMPLE_NUMBER_CA, + EXAMPLE_NUMBER_DE, + EXAMPLE_NUMBER_US, + ): + models.PhoneNumber.add(number) + assert models.PhoneNumber.get_numbers(prefix='+91') == { + EXAMPLE_NUMBER_IN_UNPREFIXED + } + assert models.PhoneNumber.get_numbers(prefix='+1', remove=False) == { + EXAMPLE_NUMBER_CA, + EXAMPLE_NUMBER_US, + } + assert models.PhoneNumber.get_numbers('+', False) == { + EXAMPLE_NUMBER_IN, + EXAMPLE_NUMBER_GB, + EXAMPLE_NUMBER_CA, + EXAMPLE_NUMBER_DE, + EXAMPLE_NUMBER_US, + } + + def test_phone_number_mixin( # pylint: disable=too-many-locals,too-many-statements phone_models, db_session ) -> None: