Moved the feed data for the lookup tables from csv file into src/conf…

…ig file. Added custom tests for alembic migration

db_revisions/utils.py

@@ -1,8 +1,6 @@
 from alembic import op
 from sqlalchemy import engine_from_config
 import sqlalchemy
-from csv import DictReader
-import os
 
 
 def table_exists(table_name):
@@ -13,14 +11,3 @@ def table_exists(table_name):
     inspector = sqlalchemy.inspect(engine)
     tables = inspector.get_table_names()
     return table_name in tables
-
-
-def get_feed_data_from_file(table_name):
-    file_dir = os.path.dirname(os.path.realpath(__file__))
-    data_file_path = f"{file_dir}/feed/%s.csv" % table_name
-    data_file = open(data_file_path, "r")
-    reader = DictReader(data_file, delimiter="|")
-    output_list = list()
-    for dictionary in reader:
-        output_list.append(dictionary)
-    return output_list

db_revisions/versions/045aa502e050_update_financial_institutions_table.py

@@ -84,7 +84,9 @@ def downgrade() -> None:
     op.drop_constraint(
         constraint_name="fk_federal_regulator_financial_institutions", table_name="financial_institutions"
     )
-    op.drop_constraint(constraint_name="fk_address_state_financial_institutions", table_name="financial_institutions")
+    op.drop_constraint(
+        constraint_name="fk_address_state_code_financial_institutions", table_name="financial_institutions"
+    )
     op.drop_constraint(
         constraint_name="fk_hmda_institution_type_financial_institutions", table_name="financial_institutions"
     )

db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py

@@ -7,8 +7,8 @@
 """
 from typing import Sequence, Union
 from alembic import op
-from db_revisions.utils import get_feed_data_from_file
 from entities.models import FederalRegulatorDao
+from config import federal_regulator_feed
 
 
 # revision identifiers, used by Alembic.
@@ -19,9 +19,7 @@
 
 
 def upgrade() -> None:
-    data = get_feed_data_from_file("federal_regulator")
-
-    op.bulk_insert(FederalRegulatorDao.__table__, data)
+    op.bulk_insert(FederalRegulatorDao.__table__, federal_regulator_feed)
 
 
 def downgrade() -> None:

db_revisions/versions/7b6ff51002b5_feed_address_state_table.py

@@ -7,8 +7,8 @@
 """
 from typing import Sequence, Union
 from alembic import op
-from db_revisions.utils import get_feed_data_from_file
 from entities.models import AddressStateDao
+from config import address_state_feed
 
 
 # revision identifiers, used by Alembic.
@@ -19,9 +19,7 @@
 
 
 def upgrade() -> None:
-    data = get_feed_data_from_file("address_state")
-
-    op.bulk_insert(AddressStateDao.__table__, data)
+    op.bulk_insert(AddressStateDao.__table__, address_state_feed)
 
 
 def downgrade() -> None:

db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py

@@ -7,8 +7,8 @@
 """
 from typing import Sequence, Union
 from alembic import op
-from db_revisions.utils import get_feed_data_from_file
 from entities.models import SBLInstitutionTypeDao
+from config import sbl_institution_type_feed
 
 # revision identifiers, used by Alembic.
 revision: str = "a41281b1e109"
@@ -18,9 +18,7 @@
 
 
 def upgrade() -> None:
-    data = get_feed_data_from_file("sbl_institution_type")
-
-    op.bulk_insert(SBLInstitutionTypeDao.__table__, data)
+    op.bulk_insert(SBLInstitutionTypeDao.__table__, sbl_institution_type_feed)
 
 
 def downgrade() -> None:

db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py

@@ -7,8 +7,8 @@
 """
 from typing import Sequence, Union
 from alembic import op
-from db_revisions.utils import get_feed_data_from_file
 from entities.models import HMDAInstitutionTypeDao
+from config import hmda_institution_type_feed
 
 
 # revision identifiers, used by Alembic.
@@ -19,9 +19,7 @@
 
 
 def upgrade() -> None:
-    data = get_feed_data_from_file("hmda_institution_type")
-
-    op.bulk_insert(HMDAInstitutionTypeDao.__table__, data)
+    op.bulk_insert(HMDAInstitutionTypeDao.__table__, hmda_institution_type_feed)
 
 
 def downgrade() -> None:

src/config.py

@@ -6,6 +6,125 @@
 from pydantic.types import SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+"""
+START Feed data for the lookup tables:
+    address_state
+    federal_regulator
+    hmda_institution_type
+    sbl_institution_type
+These are accessed in db_revisions/versions/* scripts and in test/migrations/test_lookup_tables_data_feed .
+"""
+
+address_state_feed = [
+    {"code": "AL", "name": "Alabama"},
+    {"code": "AK", "name": "Alaska"},
+    {"code": "AZ", "name": "Arizona"},
+    {"code": "AR", "name": "Arkansas"},
+    {"code": "CA", "name": "California"},
+    {"code": "CO", "name": "Colorado"},
+    {"code": "CT", "name": "Connecticut"},
+    {"code": "DE", "name": "Delaware"},
+    {"code": "FL", "name": "Florida"},
+    {"code": "GA", "name": "Georgia"},
+    {"code": "HI", "name": "Hawaii"},
+    {"code": "ID", "name": "Idaho"},
+    {"code": "IL", "name": "Illinois"},
+    {"code": "IN", "name": "Indiana"},
+    {"code": "IA", "name": "Iowa"},
+    {"code": "KS", "name": "Kansas"},
+    {"code": "KY", "name": "Kentucky"},
+    {"code": "LA", "name": "Louisiana"},
+    {"code": "ME", "name": "Maine"},
+    {"code": "MD", "name": "Maryland"},
+    {"code": "MA", "name": "Massachusetts"},
+    {"code": "MI", "name": "Michigan"},
+    {"code": "MN", "name": "Minnesota"},
+    {"code": "MS", "name": "Mississippi"},
+    {"code": "MO", "name": "Missouri"},
+    {"code": "MT", "name": "Montana"},
+    {"code": "NE", "name": "Nebraska"},
+    {"code": "NV", "name": "Nevada"},
+    {"code": "NH", "name": "New Hampshire"},
+    {"code": "NJ", "name": "New Jersey"},
+    {"code": "NM", "name": "New Mexico"},
+    {"code": "NY", "name": "New York"},
+    {"code": "NC", "name": "North Carolina"},
+    {"code": "ND", "name": "North Dakota"},
+    {"code": "OH", "name": "Ohio"},
+    {"code": "OK", "name": "Oklahoma"},
+    {"code": "OR", "name": "Oregon"},
+    {"code": "PA", "name": "Pennsylvania"},
+    {"code": "RI", "name": "Rhode Island"},
+    {"code": "SC", "name": "South Carolina"},
+    {"code": "SD", "name": "South Dakota"},
+    {"code": "TN", "name": "Tennessee"},
+    {"code": "TX", "name": "Texas"},
+    {"code": "UT", "name": "Utah"},
+    {"code": "VT", "name": "Vermont"},
+    {"code": "VA", "name": "Virginia"},
+    {"code": "WA", "name": "Washington"},
+    {"code": "WV", "name": "West Virginia"},
+    {"code": "WI", "name": "Wisconsin"},
+    {"code": "WY", "name": "Wyoming"},
+    {"code": "DC", "name": "District of Columbia"},
+    {"code": "AS", "name": "American Samoa"},
+    {"code": "GU", "name": "Guam"},
+    {"code": "MP", "name": "Northern Mariana Islands"},
+    {"code": "PR", "name": "Puerto Rico"},
+    {"code": "UM", "name": "United States Minor Outlying Islands"},
+    {"code": "VI", "name": "Virgin Islands, U.S."},
+]
+
+federal_regulator_feed = [
+    {"id": "FCA", "name": "Farm Credit Administration"},
+    {"id": "FDIC", "name": "Federal Deposit Insurance Corporation"},
+    {"id": "FHFA", "name": "Federal Housing Finance Agency"},
+    {"id": "FRS", "name": "Federal Reserve System"},
+    {"id": "NCUA", "name": "National Credit Union Administration"},
+    {"id": "OCC", "name": "Office of the Comptroller of the Currency"},
+    {"id": "OTS", "name": "Office of Thrift Supervision (only valid until July 21, 2011)"},
+]
+
+sbl_institution_type_feed = [
+    {"id": "1", "name": "Bank or savings association."},
+    {"id": "2", "name": "Minority depository institution."},
+    {"id": "3", "name": "Credit union."},
+    {"id": "4", "name": "Nondepository institution."},
+    {"id": "5", "name": "Community development financial institution (CDFI)."},
+    {"id": "6", "name": "Other nonprofit financial institution."},
+    {"id": "7", "name": "Farm Credit System institution."},
+    {"id": "8", "name": "Government lender."},
+    {"id": "9", "name": "Commercial finance company."},
+    {"id": "10", "name": "Equipment finance company."},
+    {"id": "11", "name": "Industrial loan company."},
+    {"id": "12", "name": "Online lender."},
+    {"id": "13", "name": "Other"},
+]
+
+hmda_institution_type_feed = [
+    {"id": "1", "name": "National Bank (OCC supervised)"},
+    {"id": "2", "name": "State Member Bank (FRS Supervised)"},
+    {"id": "3", "name": "State non-member bank (FDIC supervised)"},
+    {"id": "4", "name": "State Chartered Thrift (FDIC supervised)"},
+    {"id": "5", "name": "Federal Chartered Thrift (OCC supervised)"},
+    {"id": "6", "name": "Credit Union (NCUA supervised)"},
+    {"id": "7", "name": "Federal Branch or Agency of Foreign Banking Organization (FBO)"},
+    {"id": "8", "name": "Branch or Agency of FBO (FRS supervised)"},
+    {"id": "9", "name": "MBS of national Bank (OCC supervised)"},
+    {"id": "10", "name": "MBS of state member bank (FRS supervised)"},
+    {"id": "11", "name": "MBS of state non-member bank (FDIC supervised)"},
+    {"id": "12", "name": "MBS of Bank Holding Company (BHC) (FRS supervised)"},
+    {"id": "13", "name": "MBS of credit union (NCUA supervised)"},
+    {"id": "14", "name": "independent MBS, no depository affiliation"},
+    {"id": "15", "name": "MBS of Savings and Loan Holding Co"},
+    {"id": "16", "name": "MBS of state chartered Thrift"},
+    {"id": "17", "name": "MBS of federally chartered thrift (OCC supervised)"},
+    {"id": "18", "name": "Affiliate of depository institution. MBS is in the same ownership org as a depository."},
+]
+
+"""
+END Feed data for the lookup tables:
+"""
 JWT_OPTS_PREFIX = "jwt_opts_"
 
 env_files_to_load = [".env"]