diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 876eb4f1..560e2243 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -35,6 +35,7 @@ "editor.tabSize": 4, "editor.formatOnSave": true, "python.formatting.provider": "black", + "python.formatting.black": true, "python.envFile": "${workspaceFolder}/.env", "editor.codeActionsOnSave": { "source.organizeImports": true diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt index ffc429af..3389cb83 100644 --- a/.devcontainer/requirements.txt +++ b/.devcontainer/requirements.txt @@ -5,4 +5,8 @@ pandas==1.5.3 pandera==0.14.5 ipykernel openpyxl -pytest \ No newline at end of file +pytest +python-multipart +uvicorn +fastapi +httpx \ No newline at end of file diff --git a/config.py b/config.py index 00a125f2..c2a46585 100644 --- a/config.py +++ b/config.py @@ -1,16 +1,16 @@ # path to original/raw NAICS excel file -NAICS_EXCEL_PATH = "./data/naics/raw/2-6 digit_2022_Codes.xlsx" +NAICS_EXCEL_PATH = "data/naics/raw/2-6 digit_2022_Codes.xlsx" # path to parsed/filtered naics codes file -NAICS_CSV_PATH = "./data/naics/processed/2022_codes.csv" +NAICS_CSV_PATH = "data/naics/processed/2022_codes.csv" # column header text containing naics code NAICS_CODE_COL = "2022 NAICS US Code" # column header text containing naics title/description NAICS_TITLE_COL = "2022 NAICS US Title" # path to original/raw NAICS zip file -CENSUS_RAW_ZIP_PATH = "./data/census/raw/CensusFlatFile2022.zip" +CENSUS_RAW_ZIP_PATH = "data/census/raw/CensusFlatFile2022.zip" # path to parsed/filtered naics codes file -CENSUS_PROCESSED_CSV_PATH = "./data/census/processed/Census2022.processed.csv" +CENSUS_PROCESSED_CSV_PATH = "data/census/processed/Census2022.processed.csv" # census file col indexes CENSUS_STATE_COL_INDEX = 2 CENSUS_COUNTY_COL_INDEX = 3 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..f63e9260 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,5 @@ +import os +import sys + +CURR_DIR = os.path.dirname(os.path.abspath(__file__)) # noqa: E402 +sys.path.append(CURR_DIR) # noqa: E402 diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 00000000..854f4390 --- /dev/null +++ b/src/api/__init__.py @@ -0,0 +1,9 @@ +import os +import sys + +CURR_DIR = os.path.dirname(os.path.abspath(__file__)) # noqa: E402 +PARENT_DIR = os.path.dirname(CURR_DIR) # noqa: E402 +ROOT_DIR = os.path.dirname(PARENT_DIR) # noqa: E402 +sys.path.append(CURR_DIR) # noqa: E402 +sys.path.append(PARENT_DIR) # noqa: E402 +sys.path.append(ROOT_DIR) # noqa: E402 \ No newline at end of file diff --git a/src/api/main.py b/src/api/main.py new file mode 100644 index 00000000..4eee50bf --- /dev/null +++ b/src/api/main.py @@ -0,0 +1,44 @@ +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from routers import validate_router + +from util.global_data import read_geoids, read_naics_codes +from validator.create_schemas import get_schemas + +# HOWTO +# run: uvicorn main:app --host 0.0.0.0 --port 8080 --reload + +app = FastAPI() +# get our schems +app.schemas = get_schemas(read_naics_codes(), read_geoids()) + + +# custom exception handlers +@app.exception_handler(Exception) +async def api_exception_handler(request: Request, err: Exception): + """ + handle exception/error that is triggered by incorrect data format + + Args: + request (Request): HTTP request + err (Exception): the actual exception object + + Returns: + returns JSON Response with status code and message + """ + error_code = 400 + error_msg = str(err) + if type(err) is ValueError: + error_code = 422 + elif type(err) is AttributeError: + error_code = 500 + error_msg = f"Internal Server Error. Type: {str(type(err))}" + + return JSONResponse( + status_code=error_code, + content=[{"response": error_msg}], + ) + + +# add anonymous validation endpoints +app.include_router(validate_router, prefix="/v1") diff --git a/src/api/routers/__init__.py b/src/api/routers/__init__.py new file mode 100644 index 00000000..40a1da29 --- /dev/null +++ b/src/api/routers/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["validate_router"] + +from .validate import router as validate_router diff --git a/src/api/routers/validate.py b/src/api/routers/validate.py new file mode 100644 index 00000000..3a57d935 --- /dev/null +++ b/src/api/routers/validate.py @@ -0,0 +1,44 @@ +from fastapi import APIRouter, Request, UploadFile +from fastapi.responses import JSONResponse + +from validator.create_schemas import validate_data_list, validate_raw_csv + +# validate router +router = APIRouter() + + +@router.post("/validate/") +async def validate(request: Request, payload: dict): + """ + handle validate endpoint + + Args: + request (Request): request object + payload (dict): data received from user + + Returns: + JSON response + """ + phase1_schema = request.app.schemas[0] + phase2_schema = request.app.schemas[1] + response = validate_data_list(phase1_schema, phase2_schema, payload) + return JSONResponse(status_code=200, content=response) + + +@router.post("/upload/") +async def upload(request: Request, file: UploadFile): + """ + handle upload request + + Args: + request (Request): request object + file (UploadFile): uploaded file + + Returns: + JSON response + """ + contents = file.file.read() + phase1_schema = request.app.schemas[0] + phase2_schema = request.app.schemas[1] + response = validate_raw_csv(phase1_schema, phase2_schema, contents) + return JSONResponse(status_code=200, content=response) diff --git a/src/tests/__init__.py b/src/tests/__init__.py index 6e9e654f..ce64fc3b 100644 --- a/src/tests/__init__.py +++ b/src/tests/__init__.py @@ -1,6 +1,10 @@ import os import sys -ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -sys.path.append(os.path.join(ROOT_DIR, "validator")) \ No newline at end of file +CURR_DIR = os.path.dirname(os.path.abspath(__file__)) # noqa: E402 +PARENT_DIR = os.path.dirname(CURR_DIR) # noqa: E402 +ROOT_DIR = os.path.dirname(PARENT_DIR) # noqa: E402 +sys.path.append(CURR_DIR) # noqa: E402 +sys.path.append(PARENT_DIR) # noqa: E402 +sys.path.append(ROOT_DIR) # noqa: E402 +sys.path.append(os.path.join(ROOT_DIR, "validator")) diff --git a/src/tests/sample_csv_files/invalid_phase1_sample_data.csv b/src/tests/sample_csv_files/invalid_phase1_sample_data.csv new file mode 100644 index 00000000..5107047f --- /dev/null +++ b/src/tests/sample_csv_files/invalid_phase1_sample_data.csv @@ -0,0 +1,128 @@ +uid,app_date,app_method,app_recipient,ct_credit_product,ct_credit_product_ff,ct_guarantee,ct_guarantee_ff,ct_loan_term_flag,ct_loan_term,credit_purpose,credit_purpose_ff,amount_applied_for_flag,amount_applied_for,amount_approved,action_taken,action_taken_date,denial_reasons,denial_reasons_ff,pricing_interest_rate_type,pricing_init_rate_period,pricing_fixed_rate,pricing_adj_margin,pricing_adj_index_name,pricing_adj_index_name_ff,pricing_adj_index_value,pricing_origination_charges,pricing_broker_fees,pricing_initial_charges,pricing_mca_addcost_flag,pricing_mca_addcost,pricing_prepenalty_allowed,pricing_prepenalty_exists,census_tract_adr_type,census_tract_number,gross_annual_revenue_flag,gross_annual_revenue,naics_code_flag,naics_code,number_of_workers,time_in_business_type,time_in_business,business_ownership_status,num_principal_owners_flag,num_principal_owners,po_1_ethnicity,po_1_ethnicity_ff,po_1_race,po_1_race_anai_ff,po_1_race_asian_ff,po_1_race_baa_ff,po_1_race_pi_ff,po_1_gender_flag,po_1_gender_ff,po_2_ethnicity,po_2_ethnicity_ff,po_2_race,po_2_race_anai_ff,po_2_race_asian_ff,po_2_race_baa_ff,po_2_race_pi_ff,po_2_gender_flag,po_2_gender_ff,po_3_ethnicity,po_3_ethnicity_ff,po_3_race,po_3_race_anai_ff,po_3_race_asian_ff,po_3_race_baa_ff,po_3_race_pi_ff,po_3_gender_flag,po_3_gender_ff,po_4_ethnicity,po_4_ethnicity_ff,po_4_race,po_4_race_anai_ff,po_4_race_asian_ff,po_4_race_baa_ff,po_4_race_pi_ff,po_4_gender_flag,po_4_gender_ff +000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID13XTC1,20241201,1,1,988,,999,,999,,999,,999,,,2,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID14XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID21XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC2,20241201,2,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC3,20241201,3,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC4,20241201,4,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID41XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID41XTC2,20241201,1,2,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC1,20241201,1,1,1,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC2,20241201,1,1,2,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC3,20241201,1,1,3,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC4,20241201,1,1,4,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC5,20241201,1,1,5,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC6,20241201,1,1,6,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC7,20241201,1,1,7,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC8,20241201,1,1,8,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC9,20241201,1,1,977,is not blank,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC10,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID61XTC1,20241201,1,1,977,less than 300 characters,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID62XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID62XTC2,20241201,1,1,977,is not blank,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC1,20241201,1,1,988,,1,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC2,20241201,1,1,988,,2,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC3,20241201,1,1,988,,3,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC4,20241201,1,1,988,,4,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC5,20241201,1,1,988,,5,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC6,20241201,1,1,988,,6,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC7,20241201,1,1,988,,7,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC8,20241201,1,1,988,,8,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC9,20241201,1,1,988,,9,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC10,20241201,1,1,988,,10,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC11,20241201,1,1,988,,11,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC12,20241201,1,1,988,,977,not blank,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC13,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID72XTC1,20241201,1,1,988,,1,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID73XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID74XTC1,20241201,1,1,988,,1;2;3;4;5,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID81XTC1,20241201,1,1,988,,977,less than 300 characters,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID82XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID82XTC2,20241201,1,1,988,,977,is not blank,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID83XTC2,20241201,1,1,988,,1;2;3;4;977,value1,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC2,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC1,20241201,1,1,1,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC2,20241201,1,1,2,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID101XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID102XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID103XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID103XTC2,20241201,1,1,1,,999,,900,1,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID104XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC1,20241201,1,1,988,,999,,999,,1,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC2,20241201,1,1,988,,999,,999,,2,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC3,20241201,1,1,988,,999,,999,,3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC4,20241201,1,1,988,,999,,999,,4,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC5,20241201,1,1,988,,999,,999,,5,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC6,20241201,1,1,988,,999,,999,,6,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC7,20241201,1,1,988,,999,,999,,7,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC8,20241201,1,1,988,,999,,999,,8,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC9,20241201,1,1,988,,999,,999,,9,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC10,20241201,1,1,988,,999,,999,,10,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC11,20241201,1,1,988,,999,,999,,11,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC12,20241201,1,1,988,,999,,999,,977,not blank,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC13,20241201,1,1,988,,999,,999,,988,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC14,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID112XTC1,20241201,1,1,988,,999,,999,,1,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID112XTC2,20241201,1,1,988,,999,,999,,1;2;3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID113XTC1,20241201,1,1,988,,999,,999,,988,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID113XTC2,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID114XTC1,20241201,1,1,988,,999,,999,,1;2;3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID121XTC1,20241201,1,1,988,,999,,999,,977,less than 300 characters,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID122XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID122XTC2,20241201,1,1,988,,999,,999,,977,is not blank,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID123XTC1,20241201,1,1,988,,999,,999,,1;2;977,value1,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC2,20241201,1,1,988,,999,,999,,999,,988,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID141XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID142XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID143XTC1,20241201,1,1,988,,999,,999,,999,,988,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID143XTC2,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID151XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID152XTC1,20241201,1,1,988,,999,,999,,999,,999,,1,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC5,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC5,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID171XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID172XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID173XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID174XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241202,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,2,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,3,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC4,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC5,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,5,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC6,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,6,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC7,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,7,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC8,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,8,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC9,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,9,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC10,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,not blank,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC11,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC4,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC3,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID184XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID185XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID191XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,less than 300 characters,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID192XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID192XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,"is, not, blank",999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID193XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;977,value1,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID201XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,1,,,1,1,,1,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/src/tests/sample_csv_files/invalid_phase2_sample_data.csv b/src/tests/sample_csv_files/invalid_phase2_sample_data.csv new file mode 100644 index 00000000..aa3620f9 --- /dev/null +++ b/src/tests/sample_csv_files/invalid_phase2_sample_data.csv @@ -0,0 +1,128 @@ +uid,app_date,app_method,app_recipient,ct_credit_product,ct_credit_product_ff,ct_guarantee,ct_guarantee_ff,ct_loan_term_flag,ct_loan_term,credit_purpose,credit_purpose_ff,amount_applied_for_flag,amount_applied_for,amount_approved,action_taken,action_taken_date,denial_reasons,denial_reasons_ff,pricing_interest_rate_type,pricing_init_rate_period,pricing_fixed_rate,pricing_adj_margin,pricing_adj_index_name,pricing_adj_index_name_ff,pricing_adj_index_value,pricing_origination_charges,pricing_broker_fees,pricing_initial_charges,pricing_mca_addcost_flag,pricing_mca_addcost,pricing_prepenalty_allowed,pricing_prepenalty_exists,census_tract_adr_type,census_tract_number,gross_annual_revenue_flag,gross_annual_revenue,naics_code_flag,naics_code,number_of_workers,time_in_business_type,time_in_business,business_ownership_status,num_principal_owners_flag,num_principal_owners,po_1_ethnicity,po_1_ethnicity_ff,po_1_race,po_1_race_anai_ff,po_1_race_asian_ff,po_1_race_baa_ff,po_1_race_pi_ff,po_1_gender_flag,po_1_gender_ff,po_2_ethnicity,po_2_ethnicity_ff,po_2_race,po_2_race_anai_ff,po_2_race_asian_ff,po_2_race_baa_ff,po_2_race_pi_ff,po_2_gender_flag,po_2_gender_ff,po_3_ethnicity,po_3_ethnicity_ff,po_3_race,po_3_race_anai_ff,po_3_race_asian_ff,po_3_race_baa_ff,po_3_race_pi_ff,po_3_gender_flag,po_3_gender_ff,po_4_ethnicity,po_4_ethnicity_ff,po_4_race,po_4_race_anai_ff,po_4_race_asian_ff,po_4_race_baa_ff,po_4_race_pi_ff,po_4_gender_flag,po_4_gender_ff +000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID12XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID13XTC1,20241201,1,1,988,,999,,999,,999,,999,,,2,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID14XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID21XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC2,20241201,2,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC3,20241201,3,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC4,20241201,4,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID41XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID41XTC2,20241201,1,2,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC1,20241201,1,1,1,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC2,20241201,1,1,2,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC3,20241201,1,1,3,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC4,20241201,1,1,4,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC5,20241201,1,1,5,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC6,20241201,1,1,6,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC7,20241201,1,1,7,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC8,20241201,1,1,8,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC9,20241201,1,1,977,is not blank,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC10,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID61XTC1,20241201,1,1,977,less than 300 characters,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID62XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID62XTC2,20241201,1,1,977,is not blank,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC1,20241201,1,1,988,,1,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC2,20241201,1,1,988,,2,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC3,20241201,1,1,988,,3,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC4,20241201,1,1,988,,4,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC5,20241201,1,1,988,,5,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC6,20241201,1,1,988,,6,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC7,20241201,1,1,988,,7,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC8,20241201,1,1,988,,8,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC9,20241201,1,1,988,,9,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC10,20241201,1,1,988,,10,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC11,20241201,1,1,988,,11,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC12,20241201,1,1,988,,977,not blank,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC13,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID72XTC1,20241201,1,1,988,,1,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID73XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID74XTC1,20241201,1,1,988,,1;2;3;4;5,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID81XTC1,20241201,1,1,988,,977,less than 300 characters,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID82XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID82XTC2,20241201,1,1,988,,977,is not blank,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID83XTC2,20241201,1,1,988,,1;2;3;4;977,value1,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC2,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC1,20241201,1,1,1,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC2,20241201,1,1,2,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID101XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID102XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID103XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID103XTC2,20241201,1,1,1,,999,,900,1,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID104XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC1,20241201,1,1,988,,999,,999,,1,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC2,20241201,1,1,988,,999,,999,,2,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC3,20241201,1,1,988,,999,,999,,3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC4,20241201,1,1,988,,999,,999,,4,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC5,20241201,1,1,988,,999,,999,,5,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC6,20241201,1,1,988,,999,,999,,6,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC7,20241201,1,1,988,,999,,999,,7,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC8,20241201,1,1,988,,999,,999,,8,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC9,20241201,1,1,988,,999,,999,,9,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC10,20241201,1,1,988,,999,,999,,10,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC11,20241201,1,1,988,,999,,999,,11,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC12,20241201,1,1,988,,999,,999,,977,not blank,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC13,20241201,1,1,988,,999,,999,,988,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC14,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID112XTC1,20241201,1,1,988,,999,,999,,1,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID112XTC2,20241201,1,1,988,,999,,999,,1;2;3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID113XTC1,20241201,1,1,988,,999,,999,,988,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID113XTC2,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID114XTC1,20241201,1,1,988,,999,,999,,1;2;3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID121XTC1,20241201,1,1,988,,999,,999,,977,less than 300 characters,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID122XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID122XTC2,20241201,1,1,988,,999,,999,,977,is not blank,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID123XTC1,20241201,1,1,988,,999,,999,,1;2;977,value1,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC2,20241201,1,1,988,,999,,999,,999,,988,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID141XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID142XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID143XTC1,20241201,1,1,988,,999,,999,,999,,988,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID143XTC2,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID151XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID152XTC1,20241201,1,1,988,,999,,999,,999,,999,,1,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC5,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC5,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID171XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID172XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID173XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID174XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241202,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,2,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,3,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC4,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC5,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,5,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC6,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,6,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC7,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,7,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC8,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,8,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC9,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,9,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC10,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,not blank,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC11,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC4,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC3,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID184XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID185XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID191XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,less than 300 characters,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID192XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID192XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,"is, not, blank",999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID193XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;977,value1,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID201XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,1,,,1,1,,1,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/src/tests/sample_csv_files/valid_sample_data.csv b/src/tests/sample_csv_files/valid_sample_data.csv new file mode 100644 index 00000000..fc92f943 --- /dev/null +++ b/src/tests/sample_csv_files/valid_sample_data.csv @@ -0,0 +1,128 @@ +uid,app_date,app_method,app_recipient,ct_credit_product,ct_credit_product_ff,ct_guarantee,ct_guarantee_ff,ct_loan_term_flag,ct_loan_term,credit_purpose,credit_purpose_ff,amount_applied_for_flag,amount_applied_for,amount_approved,action_taken,action_taken_date,denial_reasons,denial_reasons_ff,pricing_interest_rate_type,pricing_init_rate_period,pricing_fixed_rate,pricing_adj_margin,pricing_adj_index_name,pricing_adj_index_name_ff,pricing_adj_index_value,pricing_origination_charges,pricing_broker_fees,pricing_initial_charges,pricing_mca_addcost_flag,pricing_mca_addcost,pricing_prepenalty_allowed,pricing_prepenalty_exists,census_tract_adr_type,census_tract_number,gross_annual_revenue_flag,gross_annual_revenue,naics_code_flag,naics_code,number_of_workers,time_in_business_type,time_in_business,business_ownership_status,num_principal_owners_flag,num_principal_owners,po_1_ethnicity,po_1_ethnicity_ff,po_1_race,po_1_race_anai_ff,po_1_race_asian_ff,po_1_race_baa_ff,po_1_race_pi_ff,po_1_gender_flag,po_1_gender_ff,po_2_ethnicity,po_2_ethnicity_ff,po_2_race,po_2_race_anai_ff,po_2_race_asian_ff,po_2_race_baa_ff,po_2_race_pi_ff,po_2_gender_flag,po_2_gender_ff,po_3_ethnicity,po_3_ethnicity_ff,po_3_race,po_3_race_anai_ff,po_3_race_asian_ff,po_3_race_baa_ff,po_3_race_pi_ff,po_3_gender_flag,po_3_gender_ff,po_4_ethnicity,po_4_ethnicity_ff,po_4_race,po_4_race_anai_ff,po_4_race_asian_ff,po_4_race_baa_ff,po_4_race_pi_ff,po_4_gender_flag,po_4_gender_ff +000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID12XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID13XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID14XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID21XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC2,20241201,2,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC3,20241201,3,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID31XTC4,20241201,4,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID41XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID41XTC2,20241201,1,2,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC1,20241201,1,1,1,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC2,20241201,1,1,2,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC3,20241201,1,1,3,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC4,20241201,1,1,4,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC5,20241201,1,1,5,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC6,20241201,1,1,6,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC7,20241201,1,1,7,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC8,20241201,1,1,8,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC9,20241201,1,1,977,is not blank,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID51XTC10,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID61XTC1,20241201,1,1,977,less than 300 characters,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID62XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID62XTC2,20241201,1,1,977,is not blank,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC1,20241201,1,1,988,,1,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC2,20241201,1,1,988,,2,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC3,20241201,1,1,988,,3,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC4,20241201,1,1,988,,4,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC5,20241201,1,1,988,,5,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC6,20241201,1,1,988,,6,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC7,20241201,1,1,988,,7,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC8,20241201,1,1,988,,8,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC9,20241201,1,1,988,,9,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC10,20241201,1,1,988,,10,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC11,20241201,1,1,988,,11,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC12,20241201,1,1,988,,977,not blank,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID71XTC13,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID72XTC1,20241201,1,1,988,,1,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID73XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID74XTC1,20241201,1,1,988,,1;2;3;4;5,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID81XTC1,20241201,1,1,988,,977,less than 300 characters,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID82XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID82XTC2,20241201,1,1,988,,977,is not blank,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID83XTC2,20241201,1,1,988,,1;2;3;4;977,value1,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC2,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID91XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC1,20241201,1,1,1,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC2,20241201,1,1,2,,999,,988,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID92XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID101XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID102XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID103XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID103XTC2,20241201,1,1,1,,999,,900,1,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID104XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC1,20241201,1,1,988,,999,,999,,1,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC2,20241201,1,1,988,,999,,999,,2,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC3,20241201,1,1,988,,999,,999,,3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC4,20241201,1,1,988,,999,,999,,4,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC5,20241201,1,1,988,,999,,999,,5,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC6,20241201,1,1,988,,999,,999,,6,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC7,20241201,1,1,988,,999,,999,,7,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC8,20241201,1,1,988,,999,,999,,8,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC9,20241201,1,1,988,,999,,999,,9,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC10,20241201,1,1,988,,999,,999,,10,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC11,20241201,1,1,988,,999,,999,,11,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC12,20241201,1,1,988,,999,,999,,977,not blank,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC13,20241201,1,1,988,,999,,999,,988,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID111XTC14,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID112XTC1,20241201,1,1,988,,999,,999,,1,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID112XTC2,20241201,1,1,988,,999,,999,,1;2;3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID113XTC1,20241201,1,1,988,,999,,999,,988,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID113XTC2,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID114XTC1,20241201,1,1,988,,999,,999,,1;2;3,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID121XTC1,20241201,1,1,988,,999,,999,,977,less than 300 characters,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID122XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID122XTC2,20241201,1,1,988,,999,,999,,977,is not blank,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID123XTC1,20241201,1,1,988,,999,,999,,1;2;977,value1,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC2,20241201,1,1,988,,999,,999,,999,,988,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID131XTC3,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID141XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID142XTC1,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID143XTC1,20241201,1,1,988,,999,,999,,999,,988,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID143XTC2,20241201,1,1,988,,999,,999,,999,,900,7777,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID151XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID152XTC1,20241201,1,1,988,,999,,999,,999,,999,,1,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID153XTC5,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID161XTC5,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID171XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID172XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID173XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID174XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241202,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,2,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,3,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC4,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC5,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,5,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC6,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,6,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC7,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,7,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC8,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,8,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC9,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,9,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC10,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,not blank,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID181XTC11,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC3,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID182XTC4,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC2,20241201,1,1,988,,999,,999,,999,,999,,,4,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC3,20241201,1,1,988,,999,,999,,999,,999,,7777,2,20241231,999,,999,,,,999,,,0,0,0,999,,2,2,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID183XTC4,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID184XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,999,,,,999,,,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID185XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;4,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID191XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,less than 300 characters,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID192XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXGXVID192XTC2,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,977,"is, not, blank",999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID193XTC1,20241201,1,1,988,,999,,999,,999,,999,,,3,20241231,1;2;3;977,value1,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +000TESTFIUIDDONOTUSEXBXVID201XTC1,20241201,1,1,988,,999,,999,,999,,999,,7777,1,20241231,999,,1,,,1,1,,1,0,0,0,999,,1,1,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/src/tests/test_api_functions.py b/src/tests/test_api_functions.py new file mode 100644 index 00000000..4ed32e7f --- /dev/null +++ b/src/tests/test_api_functions.py @@ -0,0 +1,120 @@ +import os + +import pytest +from fastapi.testclient import TestClient +from test_schema_functions import TestUtil + +from api.main import app + + +class TestApiSchemas: + client = TestClient(app) + util = TestUtil() + CURR_DIR = os.path.dirname(os.path.abspath(__file__)) + + def test_valid_json_data(self): + res = self.client.post("/v1/validate", json=self.util.get_data()) + assert res.json()[0] == self.util.valid_response + assert res.status_code == 200 + + def test_invalid_json_data(self): + res = self.client.post( + "/v1/validate", json=self.util.get_data({"ct_credit_product": ["989"]}) + ) + assert ( + res.json()[0]["validation"]["id"] == "ct_credit_product.invalid_enum_value" + ) + assert res.status_code == 200 + + def test_multi_invalid_json_data(self): + res = self.client.post( + "/v1/validate", + json=self.util.get_data( + { + "ct_credit_product": ["989"], + "num_principal_owners": ["1"], + "action_taken": ["2"], + } + ), + ) + assert len(res.json()) == 1 + assert ( + res.json()[0]["validation"]["id"] == "ct_credit_product.invalid_enum_value" + ) + assert res.status_code == 200 + + def test_multi_invalid_twophase_json_data(self): + res = self.client.post( + "/v1/validate", + json=self.util.get_data( + { + "num_principal_owners": ["1"], + "action_taken": ["2"], + } + ), + ) + assert len(res.json()) == 3 + assert ( + res.json()[0]["validation"]["id"] + == "amount_approved.conditional_field_conflict" + ) + assert ( + res.json()[1]["validation"]["id"] + == "pricing_charges.conditional_fieldset_conflict" + ) + assert ( + res.json()[2]["validation"]["id"] + == "num_principal_owners.conditional_field_conflict" + ) + assert res.status_code == 200 + + def test_upload_valid_file(self): + valid_file_path = os.path.join( + self.CURR_DIR, "sample_csv_files/valid_sample_data.csv" + ) + # valid_file_path = "./sample_csv_files/valid_sample_data.csv" + if os.path.isfile(valid_file_path): + files = {"file": open(valid_file_path, "rb")} + res = self.client.post("/v1/upload", files=files) + assert res.json()[0] == self.util.valid_response + assert res.status_code == 200 + else: + pytest.fail(f"{valid_file_path} does not exist.") + + def test_upload_invalid_phase1_file(self): + invalid_file_path = os.path.join( + self.CURR_DIR, "sample_csv_files/invalid_phase1_sample_data.csv" + ) + # invalid_file_path = "./sample_csv_files/invalid_phase1_sample_data.csv" + if os.path.isfile(invalid_file_path): + files = {"file": open(invalid_file_path, "rb")} + res = self.client.post("/v1/upload", files=files) + assert len(res.json()) == 1 + assert res.json()[0]["validation"]["id"] == "uid.duplicates_in_dataset" + assert res.status_code == 200 + else: + pytest.fail(f"{invalid_file_path} does not exist.") + + def test_upload_invalid_phase2_file(self): + invalid_file_path = os.path.join( + self.CURR_DIR, "sample_csv_files/invalid_phase2_sample_data.csv" + ) + if os.path.isfile(invalid_file_path): + files = {"file": open(invalid_file_path, "rb")} + res = self.client.post("/v1/upload", files=files) + assert len(res.json()) == 3 + assert ( + res.json()[0]["validation"]["id"] + == "amount_approved.conditional_field_conflict" + ) + assert ( + res.json()[1]["validation"]["id"] + == "pricing_charges.conditional_fieldset_conflict" + ) + assert ( + res.json()[2]["validation"]["id"] + == "num_principal_owners.conditional_field_conflict" + ) + assert res.status_code == 200 + else: + pytest.fail(f"{invalid_file_path} does not exist.") diff --git a/src/tests/test_check_functions.py b/src/tests/test_check_functions.py index 117e8a44..38e55698 100644 --- a/src/tests/test_check_functions.py +++ b/src/tests/test_check_functions.py @@ -1,6 +1,6 @@ import pandas as pd -from validator import global_data +from util import global_data from validator.check_functions import ( has_correct_length, has_no_conditional_field_conflict, @@ -508,32 +508,30 @@ def test_with_incorrect_length(self): class TestIsValidCode: + naics_codes = global_data.read_naics_codes() + def test_with_valid_code(self): - global_data.read_naics_codes() - result = is_valid_code("111", False, global_data.naics_codes) + result = is_valid_code("111", False, self.naics_codes) assert result is True - result = is_valid_code("111", True, global_data.naics_codes) + result = is_valid_code("111", True, self.naics_codes) assert result is True def test_with_invalid_code(self): - global_data.read_naics_codes() - result = is_valid_code("101", False, global_data.naics_codes) + result = is_valid_code("101", False, self.naics_codes) assert result is False - result = is_valid_code("101", True, global_data.naics_codes) + result = is_valid_code("101", True, self.naics_codes) assert result is False def test_with_accepted_blank(self): - global_data.read_naics_codes() - result = is_valid_code("", True, global_data.naics_codes) + result = is_valid_code("", True, self.naics_codes) assert result is True - result = is_valid_code(" ", True, global_data.naics_codes) + result = is_valid_code(" ", True, self.naics_codes) assert result is True def test_with_invalid_blank(self): - global_data.read_naics_codes() - result = is_valid_code("", False, global_data.naics_codes) + result = is_valid_code("", False, self.naics_codes) assert result is False - result = is_valid_code(" ", False, global_data.naics_codes) + result = is_valid_code(" ", False, self.naics_codes) assert result is False diff --git a/src/tests/test_schema_functions.py b/src/tests/test_schema_functions.py new file mode 100644 index 00000000..3ceb64dc --- /dev/null +++ b/src/tests/test_schema_functions.py @@ -0,0 +1,334 @@ +import pandas as pd + +from util import global_data +from validator.create_schemas import ( + get_schemas, + validate, + validate_data_list, + validate_raw_csv, +) + + +class TestGetSchemas: + naics_codes = global_data.read_naics_codes() + geoids = global_data.read_geoids() + schemas = get_schemas(naics_codes, {}) + + def test_with_valid_codes(self): + naics_phase1_checks = self.schemas[0].columns["naics_code"].checks + naics_phase2_checks = self.schemas[1].columns["naics_code"].checks + invalid_naics_format = naics_phase1_checks[0].name + invalid_naics_value = naics_phase2_checks[1].name + naics_codes = naics_phase2_checks[1]._check_kwargs["codes"] + assert self.naics_codes == naics_codes + assert invalid_naics_format == "naics_code.invalid_naics_format" + assert invalid_naics_value == "naics_code.invalid_naics_value" + + def test_with_invalid_codes(self): + geoids_phase2_checks = self.schemas[1].columns["census_tract_number"].checks + geoids = geoids_phase2_checks[1]._check_kwargs["codes"] + assert self.geoids != geoids + + +class TestUtil: + naics_codes = global_data.read_naics_codes() + geoids = global_data.read_geoids() + schemas = get_schemas(naics_codes, geoids) + valid_response = {"response": "No validations errors or warnings"} + + def get_data(self, update_data: dict[str, list[str]] = {}) -> dict[str, list[str]]: + default = { + "uid": ["000TESTFIUIDDONOTUSEXGXVID11XTC1"], + "app_date": ["20241201"], + "app_method": ["1"], + "app_recipient": ["1"], + "ct_credit_product": ["988"], + "ct_credit_product_ff": [""], + "ct_guarantee": ["999"], + "ct_guarantee_ff": [""], + "ct_loan_term_flag": ["999"], + "ct_loan_term": [""], + "credit_purpose": ["999"], + "credit_purpose_ff": [""], + "amount_applied_for_flag": ["999"], + "amount_applied_for": [""], + "amount_approved": [""], + "action_taken": ["5"], + "action_taken_date": ["20241231"], + "denial_reasons": ["999"], + "denial_reasons_ff": [""], + "pricing_interest_rate_type": ["999"], + "pricing_init_rate_period": [""], + "pricing_fixed_rate": [""], + "pricing_adj_margin": [""], + "pricing_adj_index_name": ["999"], + "pricing_adj_index_name_ff": [""], + "pricing_adj_index_value": [""], + "pricing_origination_charges": [""], + "pricing_broker_fees": [""], + "pricing_initial_charges": [""], + "pricing_mca_addcost_flag": ["999"], + "pricing_mca_addcost": [""], + "pricing_prepenalty_allowed": ["999"], + "pricing_prepenalty_exists": ["999"], + "census_tract_adr_type": ["988"], + "census_tract_number": [""], + "gross_annual_revenue_flag": ["988"], + "gross_annual_revenue": [""], + "naics_code_flag": ["988"], + "naics_code": [""], + "number_of_workers": ["988"], + "time_in_business_type": ["988"], + "time_in_business": [""], + "business_ownership_status": ["988"], + "num_principal_owners_flag": ["988"], + "num_principal_owners": [""], + "po_1_ethnicity": [""], + "po_1_ethnicity_ff": [""], + "po_1_race": [""], + "po_1_race_anai_ff": [""], + "po_1_race_asian_ff": [""], + "po_1_race_baa_ff": [""], + "po_1_race_pi_ff": [""], + "po_1_gender_flag": [""], + "po_1_gender_ff": [""], + "po_2_ethnicity": [""], + "po_2_ethnicity_ff": [""], + "po_2_race": [""], + "po_2_race_anai_ff": [""], + "po_2_race_asian_ff": [""], + "po_2_race_baa_ff": [""], + "po_2_race_pi_ff": [""], + "po_2_gender_flag": [""], + "po_2_gender_ff": [""], + "po_3_ethnicity": [""], + "po_3_ethnicity_ff": [""], + "po_3_race": [""], + "po_3_race_anai_ff": [""], + "po_3_race_asian_ff": [""], + "po_3_race_baa_ff": [""], + "po_3_race_pi_ff": [""], + "po_3_gender_flag": [""], + "po_3_gender_ff": [""], + "po_4_ethnicity": [""], + "po_4_ethnicity_ff": [""], + "po_4_race": [""], + "po_4_race_anai_ff": [""], + "po_4_race_asian_ff": [""], + "po_4_race_baa_ff": [""], + "po_4_race_pi_ff": [""], + "po_4_gender_flag": [""], + "po_4_gender_ff": [""], + } + default.update(update_data) + return default + + def get_csv_bytes(self, *update_data: str) -> bytes: + data = [ + ( + "uid,app_date,app_method,app_recipient,ct_credit_product," + "ct_credit_product_ff,ct_guarantee,ct_guarantee_ff," + "ct_loan_term_flag,ct_loan_term,credit_purpose,credit_purpose_ff," + "amount_applied_for_flag,amount_applied_for,amount_approved," + "action_taken,action_taken_date,denial_reasons,denial_reasons_ff," + "pricing_interest_rate_type,pricing_init_rate_period," + "pricing_fixed_rate,pricing_adj_margin,pricing_adj_index_name," + "pricing_adj_index_name_ff,pricing_adj_index_value," + "pricing_origination_charges,pricing_broker_fees," + "pricing_initial_charges,pricing_mca_addcost_flag," + "pricing_mca_addcost,pricing_prepenalty_allowed," + "pricing_prepenalty_exists,census_tract_adr_type," + "census_tract_number,gross_annual_revenue_flag," + "gross_annual_revenue,naics_code_flag,naics_code,number_of_workers," + "time_in_business_type,time_in_business,business_ownership_status," + "num_principal_owners_flag,num_principal_owners,po_1_ethnicity," + "po_1_ethnicity_ff,po_1_race,po_1_race_anai_ff,po_1_race_asian_ff," + "po_1_race_baa_ff,po_1_race_pi_ff,po_1_gender_flag,po_1_gender_ff," + "po_2_ethnicity,po_2_ethnicity_ff,po_2_race,po_2_race_anai_ff," + "po_2_race_asian_ff,po_2_race_baa_ff,po_2_race_pi_ff," + "po_2_gender_flag,po_2_gender_ff,po_3_ethnicity,po_3_ethnicity_ff," + "po_3_race,po_3_race_anai_ff,po_3_race_asian_ff,po_3_race_baa_ff," + "po_3_race_pi_ff,po_3_gender_flag,po_3_gender_ff,po_4_ethnicity," + "po_4_ethnicity_ff,po_4_race,po_4_race_anai_ff,po_4_race_asian_ff," + "po_4_race_baa_ff,po_4_race_pi_ff,po_4_gender_flag,po_4_gender_ff" + ) + ] + data.extend(list(update_data)) + result = "\n".join(data) + return bytes(result, "utf-8") + + +class TestValidate: + util = TestUtil() + schemas = util.schemas + phase1_schema = schemas[0] + phase2_schema = schemas[1] + + def test_with_valid_dataframe(self): + df = pd.DataFrame(data=self.util.get_data()) + result = validate(self.phase1_schema, df) + ph2_result = validate(self.phase2_schema, df) + assert len(result) == 0 + assert len(ph2_result) == 0 + + def test_with_invalid_dataframe(self): + df = pd.DataFrame(data=self.util.get_data({"ct_credit_product": ["989"]})) + result = validate(self.phase1_schema, df) + ph2_result = validate(self.phase2_schema, df) + assert len(result) == 1 + assert len(ph2_result) == 0 + + def test_with_multi_invalid_dataframe(self): + df = pd.DataFrame( + data=self.util.get_data( + { + "ct_credit_product": ["989"], + "num_principal_owners": ["1"], + "action_taken": ["2"], + } + ) + ) + result = validate(self.phase1_schema, df) + assert len(result) == 1 + + ph2_result = validate(self.phase2_schema, df) + assert len(ph2_result) == 3 + + +class TestValidateDataList: + util = TestUtil() + schemas = util.schemas + phase1_schema = schemas[0] + phase2_schema = schemas[1] + + def test_with_valid_data(self): + result = validate_data_list( + self.phase1_schema, self.phase2_schema, self.util.get_data() + ) + + assert len(result) == 1 + assert result[0] == self.util.valid_response + + def test_with_invalid_data(self): + result = validate_data_list( + self.phase1_schema, + self.phase2_schema, + self.util.get_data({"ct_credit_product": ["989"]}), + ) + + assert len(result) == 1 + assert result[0] != self.util.valid_response + + def test_with_multi_invalid_data_with_phase1(self): + result = validate_data_list( + self.phase1_schema, + self.phase2_schema, + self.util.get_data( + { + "ct_credit_product": ["989"], + "num_principal_owners": ["1"], + "action_taken": ["2"], + } + ), + ) + # should only return phase 1 validation result since phase1 failed + assert len(result) == 1 + assert result[0] != self.util.valid_response + + def test_with_multi_invalid_data_with_phase2(self): + result = validate_data_list( + self.phase1_schema, + self.phase2_schema, + self.util.get_data( + { + "num_principal_owners": ["1"], + "action_taken": ["2"], + } + ), + ) + # since the data passed phase 1 validations + # this should return phase 2 validations + assert len(result) == 3 + assert result[0] != self.util.valid_response + + +class TestValidateRawCsv: + util = TestUtil() + schemas = util.schemas + phase1_schema = schemas[0] + phase2_schema = schemas[1] + + def test_with_valid_data(self): + data = ( + "000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999," + ",999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988," + ",988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + csv_bytes = self.util.get_csv_bytes(data) + result = validate_raw_csv(self.phase1_schema, self.phase2_schema, csv_bytes) + assert len(result) == 1 + assert result[0] == self.util.valid_response + + def test_with_multi_valid_data(self): + data = ( + "000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999," + ",999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988," + ",988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + data2 = ( + "000TESTFIUIDDONOTUSEXGXVID13XTC1,20241201,1,1,988,,999,,999," + ",999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988," + ",988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + csv_bytes = self.util.get_csv_bytes(data, data2) + result = validate_raw_csv(self.phase1_schema, self.phase2_schema, csv_bytes) + assert len(result) == 1 + assert result[0] == self.util.valid_response + + def test_with_invalid_data(self): + data = ( + "000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,989,,999,,999," + ",999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988," + ",988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + data2 = ( + "000TESTFIUIDDONOTUSEXGXVID13XTC1,20241201,1,1,988,,990,,999," + ",999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988," + ",988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + csv_bytes = self.util.get_csv_bytes(data, data2) + result = validate_raw_csv(self.phase1_schema, self.phase2_schema, csv_bytes) + assert len(result) == 2 + assert result[0] != self.util.valid_response + + def test_with_multi_invalid_data_with_phase1(self): + data = ( + "000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999," + ",999,,,2,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988," + ",988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + data2 = ( + "000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999," + ",999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988," + ",988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + csv_bytes = self.util.get_csv_bytes(data, data2) + result = validate_raw_csv(self.phase1_schema, self.phase2_schema, csv_bytes) + assert len(result) == 1 + assert result[0] != self.util.valid_response + + def test_with_multi_invalid_data_with_phase2(self): + data = ( + "000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999," + ",999,,,2,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988," + ",988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + data2 = ( + "000TESTFIUIDDONOTUSEXGXVID12XTC1,20241201,1,1,988,,999,,999,,999," + ",999,,,1,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988," + ",988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,," + ) + csv_bytes = self.util.get_csv_bytes(data, data2) + result = validate_raw_csv(self.phase1_schema, self.phase2_schema, csv_bytes) + assert len(result) == 2 + assert result[0] != self.util.valid_response diff --git a/src/util/__init__.py b/src/util/__init__.py new file mode 100644 index 00000000..854f4390 --- /dev/null +++ b/src/util/__init__.py @@ -0,0 +1,9 @@ +import os +import sys + +CURR_DIR = os.path.dirname(os.path.abspath(__file__)) # noqa: E402 +PARENT_DIR = os.path.dirname(CURR_DIR) # noqa: E402 +ROOT_DIR = os.path.dirname(PARENT_DIR) # noqa: E402 +sys.path.append(CURR_DIR) # noqa: E402 +sys.path.append(PARENT_DIR) # noqa: E402 +sys.path.append(ROOT_DIR) # noqa: E402 \ No newline at end of file diff --git a/src/validator/global_data.py b/src/util/global_data.py similarity index 53% rename from src/validator/global_data.py rename to src/util/global_data.py index 69c1fc40..879fa0db 100644 --- a/src/validator/global_data.py +++ b/src/util/global_data.py @@ -1,39 +1,34 @@ import os -import re -import sys import pandas as pd +from config import CENSUS_PROCESSED_CSV_PATH, NAICS_CSV_PATH + ROOT_DIR = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -) # noqa: E402 -sys.path.append(ROOT_DIR) # noqa: E402 - -from config import CENSUS_PROCESSED_CSV_PATH, NAICS_CSV_PATH # noqa: E402 - -naics_codes = {} - -# global variable for geoids -census_geoids = {} - +) -def read_naics_codes(): +def read_naics_codes() -> dict: """ read NAICS CSV file with this format: (code, description) and populate global value: naics_codes """ - naics_codes.clear() - df = pd.read_csv(NAICS_CSV_PATH, dtype=str, na_filter=False) + naics_codes = {} + file = os.path.join(ROOT_DIR, NAICS_CSV_PATH) + df = pd.read_csv(file, dtype=str, na_filter=False) for _, row in df.iterrows(): naics_codes.update({row[0]: row[1]}) + return naics_codes -def read_geoids(): +def read_geoids() -> dict: """ read geoids CSV file with this format: (code) and populate global value: census_geoids """ - census_geoids.clear() - df = pd.read_csv(CENSUS_PROCESSED_CSV_PATH, dtype=str, na_filter=False) + census_geoids = {} + file = os.path.join(ROOT_DIR, CENSUS_PROCESSED_CSV_PATH) + df = pd.read_csv(file, dtype=str, na_filter=False) for _, row in df.iterrows(): census_geoids.update({row[0]: None}) + return census_geoids diff --git a/src/validator/__init__.py b/src/validator/__init__.py index 836099bf..854f4390 100644 --- a/src/validator/__init__.py +++ b/src/validator/__init__.py @@ -1,5 +1,9 @@ import os import sys -ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.append(ROOT_DIR) +CURR_DIR = os.path.dirname(os.path.abspath(__file__)) # noqa: E402 +PARENT_DIR = os.path.dirname(CURR_DIR) # noqa: E402 +ROOT_DIR = os.path.dirname(PARENT_DIR) # noqa: E402 +sys.path.append(CURR_DIR) # noqa: E402 +sys.path.append(PARENT_DIR) # noqa: E402 +sys.path.append(ROOT_DIR) # noqa: E402 \ No newline at end of file diff --git a/src/validator/create_schemas.py b/src/validator/create_schemas.py index 37579f4e..11a075a2 100644 --- a/src/validator/create_schemas.py +++ b/src/validator/create_schemas.py @@ -1,26 +1,168 @@ """Creates two DataFrameSchema objects by rendering the schema template with validations listed in phase 1 and phase 2.""" -from pandera import DataFrameSchema -from phase_validations import get_phase_1_and_2_validations_for_lei -from schema_template import get_template +from io import BytesIO -# Get separate schema templates for phase 1 and 2 +import pandas as pd +from pandera import DataFrameSchema +from pandera.errors import SchemaErrors +from .checks import SBLCheck +from .phase_validations import get_phase_1_and_2_validations_for_lei +from .schema_template import get_template +# Get separate schema templates for phase 1 and 2 phase_1_template = get_template() phase_2_template = get_template() -def get_schema_by_phase_for_lei(template: dict, phase: str, lei: str = None): - for column, validations in get_phase_1_and_2_validations_for_lei(lei): +def get_schemas( + naics: dict, geoids: dict, lei: str = None +) -> (DataFrameSchema, DataFrameSchema): + phase_1_schema = get_phase_1_schema_for_lei(naics, geoids, lei) + phase_2_schema = get_phase_2_schema_for_lei(naics, geoids, lei) + + return (phase_1_schema, phase_2_schema) + + +def get_schema_by_phase_for_lei( + template: dict, phase: str, naics: dict, geoids: dict, lei: str = None +): + for column, validations in get_phase_1_and_2_validations_for_lei( + naics, geoids, lei + ).items(): template[column].checks = validations[phase] return DataFrameSchema(template) -def get_phase_1_schema_for_lei(lei: str = None): - return get_schema_by_phase_for_lei(phase_1_template, "phase_1", lei) +def get_phase_1_schema_for_lei(naics: dict, geoids: dict, lei: str = None): + return get_schema_by_phase_for_lei(phase_1_template, "phase_1", naics, geoids, lei) + + +def get_phase_2_schema_for_lei(naics: dict, geoids: dict, lei: str = None): + return get_schema_by_phase_for_lei(phase_2_template, "phase_2", naics, geoids, lei) + + +def validate(schema: DataFrameSchema, df: pd.DataFrame): + """ + validate received dataframe with schema and return list of + schema errors + + Args: + schema (DataFrameSchema): schema to be used for validation + df (pd.DataFrame): data parsed into dataframe + + Returns: + list of schema error + """ + findings = [] + try: + schema(df, lazy=True) + except SchemaErrors as errors: + for schema_error in errors.schema_errors: + error = schema_error["error"] + check: SBLCheck = error.check + column_name = error.schema.name + + fields: list[str] = [column_name] + + if hasattr(check, "name"): + check_name: str = check.name + + if check.groupby: + fields += check.groupby # type: ignore + + # This will either be a boolean series or a single bool + check_output = error.check_output + else: + # This means this check's column has unique set to True. + # we shouldn't be using Unique flag as it doesn't return series of + # validation result . it returns just a printout result string/txt + raise AttributeError(f"{str(check)}") + + # Remove duplicates, but keep as `list` for JSON-friendliness + fields = list(set(fields)) + + if check_output is not None: + # `check_output` must be sorted so its index lines up with `df`'s index + check_output.sort_index(inplace=True) + + # Filter records using Pandas's boolean indexing, where all False values + # get filtered out. The `~` does the inverse since it's actually the + # False values we want to keep. + # http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#boolean-indexing + failed_check_fields_df = df[~check_output][fields].fillna("") + + # Create list of dicts representing the failed validations and the + # associated field data for each invalid record. + records = [] + for idx, row in failed_check_fields_df.iterrows(): + record = {"number": idx + 1, "field_values": {}} + for field in fields: + record["field_values"][field] = row[field] + records.append(record) + + validation_findings = { + "validation": { + "id": check_name, + "description": check.description, + "fields": fields, + "severity": "warning" if check.warning else "error", + }, + "records": records, + } + + findings.append(validation_findings) + + return findings + + +def _validate_helper( + phase1: DataFrameSchema, phase2: DataFrameSchema, df: pd.DataFrame +) -> list: + phase1_findings = validate(phase1, df) + if phase1_findings: + return phase1_findings + else: + phase2_findings = validate(phase2, df) + if phase2_findings: + return phase2_findings + else: + return [{"response": "No validations errors or warnings"}] + + +def validate_data_list( + phase1: DataFrameSchema, phase2: DataFrameSchema, data: dict +) -> list: + """ + validate data-dictionary entries + + Args: + phase1 (DataFrameSchema): phase 1 schema + phase2 (DataFrameSchema): phase 2 schema + data (dict): data dictionary + + Returns: + list: list of error validation + """ + df = pd.DataFrame.from_dict(data) + return _validate_helper(phase1, phase2, df) + + +def validate_raw_csv( + phase1: DataFrameSchema, phase2: DataFrameSchema, data: bytes +) -> list: + """ + read raw csv bytes and validate them against + phase 1 and phase 2 checks + Args: + phase1 (DataFrameSchema): phase 1 checks + phase2 (DataFrameSchema): phase 2 checks + data (bytes): csv bytes -def get_phase_2_schema_for_lei(lei: str = None): - return get_schema_by_phase_for_lei(phase_2_template, "phase_2", lei) + Returns: + list: list of error checks + """ + df = pd.read_csv(BytesIO(data), dtype=str, na_filter=False) + return _validate_helper(phase1, phase2, df) diff --git a/src/validator/main.py b/src/validator/main.py index 0a46a01d..7149692f 100644 --- a/src/validator/main.py +++ b/src/validator/main.py @@ -5,18 +5,23 @@ Run from the terminal to see the generated output. """ +import os import sys import pandas as pd from pandera.errors import SchemaErrors from schema import get_schema_for_lei +PARENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # noqa: E402 +sys.path.append(PARENT_DIR) # noqa: E402 +from util import global_data # noqa: E402 + def csv_to_df(path: str) -> pd.DataFrame: return pd.read_csv(path, dtype=str, na_filter=False) -def run_validation_on_df(df: pd.DataFrame, lei: str) -> None: +def run_validation_on_df(df: pd.DataFrame, naics: dict, geoids: dict, lei: str) -> None: """ Run validaition on the supplied dataframe and print a report to the terminal. @@ -28,32 +33,33 @@ def run_validation_on_df(df: pd.DataFrame, lei: str) -> None: print(df) print("") - sblar_schema = get_schema_for_lei(lei) - try: - sblar_schema(df, lazy=True) + schema = get_schema_for_lei(naics, geoids, lei) + schema(df, lazy=True) except SchemaErrors as errors: for error in errors.schema_errors: # Name of the column in the dataframe being checked column_name = error["error"].schema.name - + check = error["error"].check + check_output = error["error"].check_output # built in checks such as unique=True are different than custom # checks unfortunately so the name needs to be accessed differently - try: - check_name = error["error"].check.name - # This will either be a boolean series or a single bool - check_output = error["error"].check_output - except AttributeError: - check_name = error["error"].check - # this is just a string that we'd need to parse manually - check_output = error["error"].args[0] - - print(f"Validation `{check_name}` failed for column `{column_name}`") - print(check_output) - print("") + if hasattr(check, "name"): + check_name = check.name + print(f"Validation `{check_name}` failed for column `{column_name}`") + print(check_output) + print("") + else: + raise AttributeError(f"{check}") if __name__ == "__main__": + # read and populate global naics code (this should be called only once) + naics = global_data.read_naics_codes() + + # read and populate global census geoids (this should be called only once) + geoids = global_data.read_geoids() + csv_path = None lei: str = None if len(sys.argv) == 1: @@ -67,4 +73,4 @@ def run_validation_on_df(df: pd.DataFrame, lei: str) -> None: raise ValueError("correct number of args not provided") df = csv_to_df(csv_path) - run_validation_on_df(df, lei) + run_validation_on_df(df, naics, geoids, lei) diff --git a/src/validator/phase_validations.py b/src/validator/phase_validations.py index 36bda0c9..80654437 100644 --- a/src/validator/phase_validations.py +++ b/src/validator/phase_validations.py @@ -3,13 +3,7 @@ This mapping is used to populate the schema template object and create an instance of a PanderaSchema object for phase 1 and phase 2.""" -#! NOTE: "pricing_adj_margin", "pricing_adj_index_name": "pricing_adj_index_name_ff", -# and "pricing_adj_index_value" have been renamed. They used to be called -# pricing_var_xyz but are now called pricing_adj_xyz - - -import global_data -from check_functions import ( +from .check_functions import ( has_correct_length, has_no_conditional_field_conflict, has_valid_enum_pair, @@ -32,13 +26,12 @@ meets_multi_value_field_restriction, string_contains, ) -from checks import SBLCheck - -# read and populate global naics code (this should be called only once) -global_data.read_naics_codes() +from .checks import SBLCheck -def get_phase_1_and_2_validations_for_lei(lei: str = None): +def get_phase_1_and_2_validations_for_lei( + naics_codes: dict, census_geoids: dict, lei: str = None +): return { "uid": { "phase_1": [ @@ -1131,24 +1124,8 @@ def get_phase_1_and_2_validations_for_lei(lei: str = None): ], "phase_2": [], }, - "pricing_mca_addcost_flag": {"phase_1": [], "phase_2": []}, - "pricing_mca_addcost": {"phase_1": [], "phase_2": []}, - "pricing_prepenalty_allowed": { - "phase_1": [ - SBLCheck( - is_valid_enum, - name="pricing_prepenalty_allowed.invalid_enum_value", - description=( - "'Prepayment penalty could be imposed' must equal 1, 2, or 999." - ), - element_wise=True, - accepted_values=[ - "1", - "2", - "999", - ], - ), - ], + "pricing_mca_addcost_flag": { + "phase_1": [], "phase_2": [ SBLCheck( has_valid_enum_pair, @@ -1172,6 +1149,25 @@ def get_phase_1_and_2_validations_for_lei(lei: str = None): ), ], }, + "pricing_mca_addcost": {"phase_1": [], "phase_2": []}, + "pricing_prepenalty_allowed": { + "phase_1": [ + SBLCheck( + is_valid_enum, + name="pricing_prepenalty_allowed.invalid_enum_value", + description=( + "'Prepayment penalty could be imposed' must equal 1, 2, or 999." + ), + element_wise=True, + accepted_values=[ + "1", + "2", + "999", + ], + ), + ], + "phase_2": [], + }, "pricing_prepenalty_exists": { "phase_1": [ SBLCheck( @@ -1252,6 +1248,18 @@ def get_phase_1_and_2_validations_for_lei(lei: str = None): }, ], ), + SBLCheck( + is_valid_code, + name="census_tract_number.invalid_geoid", + description=( + "When present, 'census tract: tract number' " + "should be a valid census tract GEOID as defined " + "by the U.S. Census Bureau." + ), + element_wise=True, + accept_blank=True, + codes=census_geoids, + ), ], }, "gross_annual_revenue_flag": { @@ -1350,7 +1358,7 @@ def get_phase_1_and_2_validations_for_lei(lei: str = None): ), element_wise=True, accept_blank=True, - codes=global_data.naics_codes, + codes=naics_codes, ), SBLCheck( has_no_conditional_field_conflict, diff --git a/src/validator/schema.py b/src/validator/schema.py index da6e2111..383420bb 100644 --- a/src/validator/schema.py +++ b/src/validator/schema.py @@ -6,7 +6,6 @@ The only major modification from native Pandera is the use of custom Check classes to differentiate between warnings and errors. """ -import global_data from check_functions import ( has_correct_length, has_no_conditional_field_conflict, @@ -33,14 +32,8 @@ from checks import SBLCheck from pandera import Column, DataFrameSchema -# read and populate global naics code (this should be called only once) -global_data.read_naics_codes() -# read and populate global census geoids (this should be called only once) -global_data.read_geoids() - - -def get_schema_for_lei(lei: str): +def get_schema_for_lei(naics: dict, geoids: dict, lei: str): return DataFrameSchema( { "uid": Column( @@ -1353,7 +1346,7 @@ def get_schema_for_lei(lei: str): ), element_wise=True, accept_blank=True, - codes=global_data.census_geoids, + codes=geoids, ), ], ), @@ -1463,7 +1456,7 @@ def get_schema_for_lei(lei: str): ), element_wise=True, accept_blank=True, - codes=global_data.naics_codes, + codes=naics, ), SBLCheck( has_no_conditional_field_conflict, diff --git a/src/validator/schema_template.py b/src/validator/schema_template.py index 2b320047..9f86dc7a 100644 --- a/src/validator/schema_template.py +++ b/src/validator/schema_template.py @@ -14,6 +14,8 @@ from pandera import Column +# NOTE: we should not use unique flag in column definitions. +# to check for uniqueness, add `is_unique_column` check to column _schema_template = { "uid": Column( str, diff --git a/tools/__init__.py b/tools/__init__.py index e69de29b..eba2d7dc 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -0,0 +1,7 @@ +import os +import sys + +CURR_DIR = os.path.dirname(os.path.abspath(__file__)) # noqa: E402 +PARENT_DIR = os.path.dirname(CURR_DIR) # noqa: E402 +sys.path.append(CURR_DIR) # noqa: E402 +sys.path.append(PARENT_DIR) # noqa: E402 \ No newline at end of file diff --git a/tools/process_census.py b/tools/process_census.py index 0686b9c5..062e8217 100644 --- a/tools/process_census.py +++ b/tools/process_census.py @@ -5,8 +5,8 @@ import pandas as pd -ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # noqa: E402 -sys.path.append(ROOT_DIR) # noqa: E402 +PARENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # noqa: E402 +sys.path.append(PARENT_DIR) # noqa: E402 import config # noqa: E402 @@ -24,11 +24,13 @@ def _is_number(s): def _extract_census_zip_file(): CENSUS_TMP_CSV_PATH = config.CENSUS_RAW_ZIP_PATH + ".tmp.csv" # unzip and extract csv files - with zipfile.ZipFile(config.CENSUS_RAW_ZIP_PATH, "r") as zip_ref: + input_file = os.path.join(PARENT_DIR, config.CENSUS_RAW_ZIP_PATH) + output_file = os.path.join(PARENT_DIR, CENSUS_TMP_CSV_PATH) + with zipfile.ZipFile(input_file, "r") as zip_ref: for file in zip_ref.namelist(): # iterate over files in archive if file[-4:] == ".csv": - print("Extracting CSV to {}".format(CENSUS_TMP_CSV_PATH)) - with open(CENSUS_TMP_CSV_PATH, "wb") as outfile: + print("Extracting CSV to {}".format(output_file)) + with open(output_file, "wb") as outfile: outfile.write(zip_ref.read(file)) # it should only have one csv file return CENSUS_TMP_CSV_PATH @@ -85,13 +87,13 @@ def _read_census_csv(src_path: str, csv_path: str): """ if __name__ == "__main__": CSV_PATH = config.CENSUS_PROCESSED_CSV_PATH - - if os.path.isfile(CSV_PATH): - error_msg = "Output {} csv file existed".format(CSV_PATH) + file = os.path.join(PARENT_DIR, CSV_PATH) + if os.path.isfile(file): + error_msg = "Output {} csv file existed".format(file) raise FileExistsError(error_msg) tmp_census_csv_file = _extract_census_zip_file() print("Reading extracted CSV File . {}".format(tmp_census_csv_file)) - _read_census_csv(tmp_census_csv_file, CSV_PATH) + _read_census_csv(tmp_census_csv_file, file) print("Removing extracted CSV File") os.remove(tmp_census_csv_file) diff --git a/tools/process_naics.py b/tools/process_naics.py index 8ac162ae..b179effc 100644 --- a/tools/process_naics.py +++ b/tools/process_naics.py @@ -23,14 +23,16 @@ TITLE_COL = config.NAICS_TITLE_COL #check for paths - if not os.path.isfile(EXCEL_PATH): + excel_file = os.path.join(ROOT_DIR, EXCEL_PATH) + csv_file = os.path.join(ROOT_DIR, CSV_PATH) + if not os.path.isfile(excel_file): error_msg = "Input excel file not existed" raise FileNotFoundError(error_msg) - if os.path.isfile(CSV_PATH): + if os.path.isfile(csv_file): error_msg = "Output csv file existed" raise FileExistsError(error_msg) - df = pd.read_excel(EXCEL_PATH, dtype=str, na_filter=False) + df = pd.read_excel(excel_file, dtype=str, na_filter=False) #add header result = [["code", "title"]] @@ -44,7 +46,7 @@ result.append(a_row) #output data to csv file - with open(CSV_PATH, 'w') as f: + with open(csv_file, 'w') as f: writer = csv.writer(f) writer.writerows(result) \ No newline at end of file