From a750426ae0302a20903105a09549a37d1d8a6f1e Mon Sep 17 00:00:00 2001 From: Bruce Schubert Date: Tue, 3 Nov 2020 11:48:37 -0800 Subject: [PATCH 1/6] Refactored phone_no diplay formatting to use a single method. - Re: #112 --- callattendant/userinterface/webapp.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/callattendant/userinterface/webapp.py b/callattendant/userinterface/webapp.py index 1555834..e060ebc 100644 --- a/callattendant/userinterface/webapp.py +++ b/callattendant/userinterface/webapp.py @@ -320,7 +320,7 @@ def calls(): calls = [] for row in result_set: number = row[2] - phone_no = '{}-{}-{}'.format(number[0:3], number[3:6], number[6:]) + phone_no = format_phone_no(number) # Flask pages use the static folder to get resources. # In the static folder we have created a soft-link to the # data/messsages folder containing the actual messages. @@ -403,7 +403,7 @@ def calls_view(call_no): caller = {} if len(row) > 0: number = row[2] - phone_no = '{}-{}-{}'.format(number[0:3], number[3:6], number[6:]) + phone_no = format_phone_no(number) # Flask pages use the static folder to get resources. # In the static folder we have created a soft-link to the # data/messsages folder containing the actual messages. @@ -503,7 +503,7 @@ def callers_manage(call_no): number = record[2] caller.update(dict( call_no=record[0], - phone_no='{}-{}-{}'.format(number[0:3], number[3:6], number[6:]), + phone_no=format_phone_no(number), name=record[1], whitelisted=record[3], blacklisted=record[4], @@ -544,7 +544,7 @@ def callers_blocked(): records = [] for record in result_set: number = record[0] - phone_no = '{}-{}-{}'.format(number[0:3], number[3:6], number[6:]) + phone_no = format_phone_no(number) records.append(dict( Phone_Number=phone_no, Name=record[1], @@ -636,7 +636,7 @@ def callers_permitted(): records = [] for record in result_set: number = record[0] - phone_no = '{}-{}-{}'.format(number[0:3], number[3:6], number[6:]) + phone_no = format_phone_no(number) records.append(dict( Phone_Number=phone_no, Name=record[1], @@ -763,7 +763,7 @@ def messages(): msg_no=row[0], call_no=row[1], name=row[2], - phone_no='{}-{}-{}'.format(number[0:3], number[3:6], number[6:]), + phone_no=format_phone_no(number), wav_file=filepath, msg_played=row[5], date=date_time.strftime('%d-%b-%y'), From bec5276bf32dcabf9f162243706b7e28deaa0ad5 Mon Sep 17 00:00:00 2001 From: Bruce Schubert Date: Tue, 3 Nov 2020 16:36:25 -0800 Subject: [PATCH 2/6] Updated phone number display formatting to use configuration settings. - User defined phone number formats supportted. - Raw support (no formatting) supported. - Handles variable length phone numbers. - Closes #112 --- callattendant/app.cfg.example | 36 +++++++++++++++++++++++++++ callattendant/config.py | 3 +++ callattendant/userinterface/webapp.py | 35 +++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/callattendant/app.cfg.example b/callattendant/app.cfg.example index baad0b4..05ac58b 100644 --- a/callattendant/app.cfg.example +++ b/callattendant/app.cfg.example @@ -24,6 +24,42 @@ TESTING = False # This should not be changed/overrriden except during development/testing #DATABASE = "callattendant.db" +# PHONE_DISPLAY_SEPARATOR: Specify the character used to format phone numbers, e.g, a space, hyphen or period, +PHONE_DISPLAY_SEPARATOR = "-" + +# PHONE_DISPLAY_FORMAT: Define the formatting of phone numbers in the various lists. You must use the +# separator character defined by PHONE_DISPLAY_SEPARATOR above in the format string. +# +# The phone display format handles variable length phone numbers. Excess digits not included +# in the format string are prepended to the number with a separator. +# For example, the AUS number 006173XXXYYYY would be formatted as follows: +# General format: 006173-XXX-YYYY +# AU format: 00-61-73-XXX-YYYY +# US format: 006-173-XXX-YYYY +# UK format: 00-6173-XXX-YYYY +# FR format: 0061-73X-XX-YY-YY +# +# Example: General +# PHONE_DISPLAY_FORMAT = "###-####" +# +# Example: US +# PHONE_DISPLAY_FORMAT = "###-###-####" +# +# Example: UK +# PHONE_DISPLAY_FORMAT = "####-###-####" +# +# Example: FR +# PHONE_DISPLAY_FORMAT = "###-##-##-##" +# +# Example: AU +# PHONE_DISPLAY_FORMAT = "##-##-###-####" +# +# Example: Raw - no formatting +# PHONE_DISPLAY_FORMAT = "" +# +PHONE_DISPLAY_FORMAT = "###-###-####" + + # SCREENING_MODE: A tuple containing: "whitelist" and/or "blacklist", or empty SCREENING_MODE = ("whitelist", "blacklist") diff --git a/callattendant/config.py b/callattendant/config.py index 392efa5..902fa5f 100644 --- a/callattendant/config.py +++ b/callattendant/config.py @@ -28,6 +28,9 @@ "DATABASE": "callattendant.db", "SCREENING_MODE": ("whitelist", "blacklist"), + "PHONE_DISPLAY_SEPARATOR": "-", + "PHONE_DISPLAY_FORMAT": "###-###-####", + "BLOCK_ENABLED": True, "BLOCK_NAME_PATTERNS": {"V[0-9]{15}": "Telemarketer Caller ID", }, "BLOCK_NUMBER_PATTERNS": {}, diff --git a/callattendant/userinterface/webapp.py b/callattendant/userinterface/webapp.py index e060ebc..0cdc3cc 100644 --- a/callattendant/userinterface/webapp.py +++ b/callattendant/userinterface/webapp.py @@ -865,7 +865,40 @@ def settings(): def format_phone_no(number): - return'{}-{}-{}'.format(number[0:3], number[3:6], number[6:]) + ''' Format the phone number based on a template.''' + + config = current_app.config.get("MASTER_CONFIG") + template = config.get("PHONE_DISPLAY_FORMAT") + separator = config.get("PHONE_DISPLAY_SEPARATOR") + if separator == "" or template == "": + return number + + # Get the template and split into reverse ordered parts for processing + tmpl_parts = template.split(separator) + tmpl_parts.reverse() + + # Piece together the phone no from right to left to handle variable len numbers + number_len = len(number) + end = number_len + total_digits = 0 + phone_parts = [] + for tmpl in tmpl_parts: + # Assemble parts from right to left + start = max(0, end - len(tmpl)) + digits = number[start: end] + phone_parts.insert(0, digits) + # Prepare for next part + end = start + total_digits += len(digits) + # if number is shorter than template then exit loop + if start == 0: + break + # If number is longer then template, then capture remaining digits + if total_digits < number_len: + # Prepend remaining digits to parts + phone_parts.insert(0, number[0: number_len - total_digits]) + # Return the formatted number + return separator.join(phone_parts) def get_db(): From abbb9d131403a91a99d30f51716a339ef4a9d74e Mon Sep 17 00:00:00 2001 From: Bruce Schubert Date: Wed, 4 Nov 2020 09:27:27 -0800 Subject: [PATCH 3/6] Removed phone number input validation mask to accomodate various number formats. - Now using a common method to strip phone numbers of non-alphanumeric chars. - Removed input pattern mask from the Add Permitted Number and Add Blocked Number dialogs. - Displaying phone number format string in dialogs for guidance only. - closes #111 --- .../templates/callers_blocked.html | 5 +-- .../templates/callers_permitted.html | 5 +-- callattendant/userinterface/webapp.py | 42 ++++++++++++------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/callattendant/userinterface/templates/callers_blocked.html b/callattendant/userinterface/templates/callers_blocked.html index cbf4537..cafabb2 100644 --- a/callattendant/userinterface/templates/callers_blocked.html +++ b/callattendant/userinterface/templates/callers_blocked.html @@ -68,9 +68,8 @@
- - + +
diff --git a/callattendant/userinterface/templates/callers_permitted.html b/callattendant/userinterface/templates/callers_permitted.html index a93b19f..0867881 100644 --- a/callattendant/userinterface/templates/callers_permitted.html +++ b/callattendant/userinterface/templates/callers_permitted.html @@ -73,9 +73,8 @@
- - + +
diff --git a/callattendant/userinterface/webapp.py b/callattendant/userinterface/webapp.py index 0cdc3cc..96eae97 100644 --- a/callattendant/userinterface/webapp.py +++ b/callattendant/userinterface/webapp.py @@ -448,7 +448,7 @@ def callers_manage(call_no): # Post changes to the blacklist or whitelist table before rendering if request.method == 'POST': - number = request.form['phone_no'].replace('-', '') + number = strip_phone_no(request.form['phone_no']) if request.form['action'] == 'add-permit': caller = {} caller['NMBR'] = number @@ -564,6 +564,7 @@ def callers_blocked(): return render_template( 'callers_blocked.html', active_nav_item='blocked', + phone_no_format=current_app.config.get("MASTER_CONFIG").get("PHONE_DISPLAY_FORMAT"), blacklist=records, page=page, per_page=per_page, @@ -577,8 +578,7 @@ def callers_blocked_add(): Add a new blacklist entry """ caller = {} - # TODO: Strip all none digits from phone via regex - number = request.form["phone"].replace('-', '') + number = strip_phone_no(request.form["phone"]) caller['NMBR'] = number caller['NAME'] = request.form["name"] print("Adding " + number + " to blacklist") @@ -596,7 +596,7 @@ def callers_blocked_update(phone_no): """ Update the blacklist entry associated with the phone number. """ - number = phone_no.replace('-', '') + number = strip_phone_no(phone_no) print("Updating " + number + " in blacklist") blacklist = Blacklist(get_db(), current_app.config) blacklist.update_number(number, request.form['name'], request.form['reason']) @@ -609,7 +609,7 @@ def callers_blocked_delete(phone_no): """ Delete the blacklist entry associated with the phone number. """ - number = phone_no.replace('-', '') + number = strip_phone_no(phone_no) print("Removing " + number + " from blacklist") blacklist = Blacklist(get_db(), current_app.config) @@ -655,6 +655,7 @@ def callers_permitted(): return render_template( 'callers_permitted.html', active_nav_item='permitted', + phone_no_format=current_app.config.get("MASTER_CONFIG").get("PHONE_DISPLAY_FORMAT"), whitelist=records, total_calls=total, page=page, @@ -669,8 +670,7 @@ def callers_permitted_add(): Add a new whitelist entry """ caller = {} - # TODO: Strip all none digits from phone via regex - number = request.form['phone'].replace('-', '') + number = strip_phone_no(request.form['phone']) caller['NMBR'] = number caller['NAME'] = request.form['name'] print("Adding " + number + " to whitelist") @@ -688,7 +688,8 @@ def callers_permitted_update(phone_no): """ Update the whitelist entry associated with the phone number. """ - number = phone_no.replace('-', '') + number = strip_phone_no(phone_no) + print("Updating " + number + " in whitelist") whitelist = Whitelist(get_db(), current_app.config) whitelist.update_number(number, request.form['name'], request.form['reason']) @@ -701,7 +702,7 @@ def callers_permitted_delete(phone_no): """ Delete the whitelist entry associated with the phone number. """ - number = phone_no.replace('-', '') + number = strip_phone_no(phone_no) print("Removing " + number + " from whitelist") whitelist = Whitelist(get_db(), current_app.config) @@ -865,8 +866,9 @@ def settings(): def format_phone_no(number): - ''' Format the phone number based on a template.''' - + ''' + Returns a formatted the phone number based on the PHONE_DISPLAY_FORMAT configuration setting. + ''' config = current_app.config.get("MASTER_CONFIG") template = config.get("PHONE_DISPLAY_FORMAT") separator = config.get("PHONE_DISPLAY_SEPARATOR") @@ -900,9 +902,17 @@ def format_phone_no(number): # Return the formatted number return separator.join(phone_parts) +def strip_phone_no(phone_no): + ''' + Returns the phone no stripped of all non-alphanumeric characters. + ''' + return "".join(filter(str.isalnum, phone_no)) + def get_db(): - '''Get a connection to the database''' + ''' + Get a connection to the database + ''' # Flask template for database connections if 'db' not in g: master_config = current_app.config.get("MASTER_CONFIG") @@ -916,7 +926,9 @@ def get_db(): def close_db(e=None): - '''Clost the connection to the database''' + ''' + Clost the connection to the database + ''' # Flask template for database connections db = g.pop('db', None) @@ -925,7 +937,9 @@ def close_db(e=None): def get_row_count(table_name): - '''Returns the row count for the given table''' + ''' + Returns the row count for the given table + ''' # Using the current request's db connection sql = 'select count(*) from {}'.format(table_name) g.cur.execute(sql) From 27215f29a00b8c06b9a1e277436b30e484306356 Mon Sep 17 00:00:00 2001 From: Bruce Schubert Date: Wed, 4 Nov 2020 10:24:32 -0800 Subject: [PATCH 4/6] Fixed search to accomodate alpha chars. - Ensures phone numbers use uppercase alpha chars. - Alpha chars now allowed in search phone number. - Closes #114 --- callattendant/userinterface/webapp.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/callattendant/userinterface/webapp.py b/callattendant/userinterface/webapp.py index 96eae97..dc0453f 100644 --- a/callattendant/userinterface/webapp.py +++ b/callattendant/userinterface/webapp.py @@ -273,8 +273,7 @@ def calls(): search_criteria = "" if search_text: if search_type == "phone": - num_list = re.findall('[0-9]+', search_text) - number = "".join(num_list) # override GET arg if we're searching + number = transform_number(search_text) # override GET arg if we're searching search_criteria = "WHERE Number='{}'".format(number) else: search_criteria = "WHERE Caller LIKE '%{}%'".format(search_text) @@ -448,7 +447,7 @@ def callers_manage(call_no): # Post changes to the blacklist or whitelist table before rendering if request.method == 'POST': - number = strip_phone_no(request.form['phone_no']) + number = transform_number(request.form['phone_no']) if request.form['action'] == 'add-permit': caller = {} caller['NMBR'] = number @@ -578,7 +577,7 @@ def callers_blocked_add(): Add a new blacklist entry """ caller = {} - number = strip_phone_no(request.form["phone"]) + number = transform_number(request.form["phone"]) caller['NMBR'] = number caller['NAME'] = request.form["name"] print("Adding " + number + " to blacklist") @@ -596,7 +595,7 @@ def callers_blocked_update(phone_no): """ Update the blacklist entry associated with the phone number. """ - number = strip_phone_no(phone_no) + number = transform_number(phone_no) print("Updating " + number + " in blacklist") blacklist = Blacklist(get_db(), current_app.config) blacklist.update_number(number, request.form['name'], request.form['reason']) @@ -609,7 +608,7 @@ def callers_blocked_delete(phone_no): """ Delete the blacklist entry associated with the phone number. """ - number = strip_phone_no(phone_no) + number = transform_number(phone_no) print("Removing " + number + " from blacklist") blacklist = Blacklist(get_db(), current_app.config) @@ -670,7 +669,7 @@ def callers_permitted_add(): Add a new whitelist entry """ caller = {} - number = strip_phone_no(request.form['phone']) + number = transform_number(request.form['phone']) caller['NMBR'] = number caller['NAME'] = request.form['name'] print("Adding " + number + " to whitelist") @@ -688,7 +687,7 @@ def callers_permitted_update(phone_no): """ Update the whitelist entry associated with the phone number. """ - number = strip_phone_no(phone_no) + number = transform_number(phone_no) print("Updating " + number + " in whitelist") whitelist = Whitelist(get_db(), current_app.config) @@ -702,7 +701,7 @@ def callers_permitted_delete(phone_no): """ Delete the whitelist entry associated with the phone number. """ - number = strip_phone_no(phone_no) + number = transform_number(phone_no) print("Removing " + number + " from whitelist") whitelist = Whitelist(get_db(), current_app.config) @@ -902,11 +901,11 @@ def format_phone_no(number): # Return the formatted number return separator.join(phone_parts) -def strip_phone_no(phone_no): +def transform_number(phone_no): ''' - Returns the phone no stripped of all non-alphanumeric characters. + Returns the phone no stripped of all non-alphanumeric characters and makes uppercase. ''' - return "".join(filter(str.isalnum, phone_no)) + return "".join(filter(str.isalnum, phone_no)).upper() def get_db(): From e3cea4fa0fcf6d9126102af2fa09c4113d4d551a Mon Sep 17 00:00:00 2001 From: Bruce Schubert Date: Wed, 4 Nov 2020 11:55:37 -0800 Subject: [PATCH 5/6] Made the NomoRobo online lookup service optional. - Added the BLOCK_SERVICE to the configuration. Permitted values are "NOMOROBO" and "" (blank). - Cleaned up the logging during testing. - Closes #113 --- callattendant/app.cfg.example | 8 ++++++++ callattendant/app.py | 5 +++++ callattendant/config.py | 1 + callattendant/screening/callscreener.py | 17 +++++++++-------- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/callattendant/app.cfg.example b/callattendant/app.cfg.example index 05ac58b..64ab3f1 100644 --- a/callattendant/app.cfg.example +++ b/callattendant/app.cfg.example @@ -74,6 +74,14 @@ BLOCK_NAME_PATTERNS = {"V[0-9]{15}": "Telemarketer Caller ID", } # Example: {"P": "Private number", "O": "Out of area",} BLOCK_NUMBER_PATTERNS = {} +# BLOCK_SERVICE: The name of the online service used to lookup robocallers and spam numbers. +# Currently, only NOMOROBO is supported and it is for the USA. Areas outside the USA should set +# to blank. When the online service is blank (disabled), only the blacklist and blocked +# name/number patterns are used to block numbers. +# +# Example: "NOMOROBO" (USA) or "" (disabled). +BLOCK_SERVICE = "NOMOROBO" + # BLOCKED_ACTIONS: A tuple containing a combination of the following actions: # "greeting", "record_message", "voice_mail". diff --git a/callattendant/app.py b/callattendant/app.py index 034bece..bb3f6e2 100755 --- a/callattendant/app.py +++ b/callattendant/app.py @@ -26,6 +26,7 @@ import sys import queue import sqlite3 +import time from pprint import pprint from shutil import copyfile @@ -115,6 +116,10 @@ def process_calls(self): # Instruct the modem to start feeding calls into the caller queue self.modem.handle_calls(self.handle_caller) + # If testing, allow queue to be filled before processing for clean, readable logs + if self.config["TESTING"]: + time.sleep(1) + # Process incoming calls while 1: try: diff --git a/callattendant/config.py b/callattendant/config.py index 902fa5f..378dbf9 100644 --- a/callattendant/config.py +++ b/callattendant/config.py @@ -32,6 +32,7 @@ "PHONE_DISPLAY_FORMAT": "###-###-####", "BLOCK_ENABLED": True, + "BLOCK_SERVICE": "NOMOROBO", "BLOCK_NAME_PATTERNS": {"V[0-9]{15}": "Telemarketer Caller ID", }, "BLOCK_NUMBER_PATTERNS": {}, diff --git a/callattendant/screening/callscreener.py b/callattendant/screening/callscreener.py index f81d444..4898f02 100644 --- a/callattendant/screening/callscreener.py +++ b/callattendant/screening/callscreener.py @@ -62,14 +62,15 @@ def is_blacklisted(self, callerid): reason = block["number_patterns"][key] print(reason) return True, reason - print(">> Checking nomorobo...") - result = self._nomorobo.lookup_number(number) - if result["spam"]: - reason = "{} with score {}".format(result["reason"], result["score"]) - if self.config["DEBUG"]: - print(">>> {}".format(reason)) - self.blacklist_caller(callerid, reason) - return True, reason + if block["service"].upper() == "NOMOROBO": + print(">> Checking nomorobo...") + result = self._nomorobo.lookup_number(number) + if result["spam"]: + reason = "{} with score {}".format(result["reason"], result["score"]) + if self.config["DEBUG"]: + print(">>> {}".format(reason)) + self.blacklist_caller(callerid, reason) + return True, reason print("Caller has been screened") return False, "Not found" finally: From d7898c25753dde6f3fee6647def2f003e8bd3a8f Mon Sep 17 00:00:00 2001 From: Bruce Schubert Date: Wed, 4 Nov 2020 12:36:56 -0800 Subject: [PATCH 6/6] Updated webapp unit test with new phone number configuration settings. --- tests/test_webapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_webapp.py b/tests/test_webapp.py index 33ddc92..67d997a 100644 --- a/tests/test_webapp.py +++ b/tests/test_webapp.py @@ -50,6 +50,8 @@ def myapp(): master_config = { "DB_FILE": db_path, + "PHONE_DISPLAY_FORMAT": "###-###-####", + "PHONE_DISPLAY_SEPARATOR": "-", } app.config['MASTER_CONFIG'] = master_config