Features/61 alembic seed lookup tables (#65)

Co-authored-by: Nargis Sultani <[email protected]>

db_revisions/feed/address_state.csv

@@ -0,0 +1,58 @@
+code|name
+AL|Alabama
+AK|Alaska
+AZ|Arizona
+AR|Arkansas
+CA|California
+CO|Colorado
+CT|Connecticut
+DE|Delaware
+FL|Florida
+GA|Georgia
+HI|Hawaii
+ID|Idaho
+IL|Illinois
+IN|Indiana
+IA|Iowa
+KS|Kansas
+KY|Kentucky
+LA|Louisiana
+ME|Maine
+MD|Maryland
+MA|Massachusetts
+MI|Michigan
+MN|Minnesota
+MS|Mississippi
+MO|Missouri
+MT|Montana
+NE|Nebraska
+NV|Nevada
+NH|New Hampshire
+NJ|New Jersey
+NM|New Mexico
+NY|New York
+NC|North Carolina
+ND|North Dakota
+OH|Ohio
+OK|Oklahoma
+OR|Oregon
+PA|Pennsylvania
+RI|Rhode Island
+SC|South Carolina
+SD|South Dakota
+TN|Tennessee
+TX|Texas
+UT|Utah
+VT|Vermont
+VA|Virginia
+WA|Washington
+WV|West Virginia
+WI|Wisconsin
+WY|Wyoming
+DC|District of Columbia
+AS|American Samoa
+GU|Guam
+MP|Northern Mariana Islands
+PR|Puerto Rico
+UM|United States Minor Outlying Islands
+VI|Virgin Islands, U.S.

db_revisions/feed/federal_regulator.csv

@@ -0,0 +1,8 @@
+id|name
+FCA|Farm Credit Administration
+FDIC|Federal Deposit Insurance Corporation
+FHFA|Federal Housing Finance Agency
+FRS|Federal Reserve System
+NCUA|National Credit Union Administration
+OCC|Office of the Comptroller of the Currency
+OTS|Office of Thrift Supervision (only valid until July 21, 2011)

db_revisions/feed/hmda_institution_type.csv

@@ -0,0 +1,19 @@
+id|name
+1|National Bank (OCC supervised)
+2|State Member Bank (FRS Supervised):
+3|State non-member bank (FDIC supervised)
+4|State Chartered Thrift (FDIC supervised)
+5|Federal Chartered Thrift (OCC supervised)
+6|Credit Union (NCUA supervised)
+7|Federal Branch or Agency of Foreign Banking Organization (FBO)
+8|Branch or Agency of FBO (FRS supervised)
+9|MBS of national Bank (OCC supervised)
+10|MBS of state member bank (FRS supervised)
+11|MBS of state non-member bank (FDIC supervised)
+12|MBS of Bank Holding Company (BHC) (FRS supervised)
+13|MBS of credit union (NCUA supervised)
+14|independent MBS, no depository affiliation
+15|MBS of Savings and Loan Holding Co
+16|MBS of state chartered Thrift
+17|MBS of federally chartered thrift (OCC supervised)
+18|Affiliate of depository institution. MBS is in the same ownership org as a depository.

db_revisions/feed/sbl_institution_type.csv

@@ -0,0 +1,14 @@
+id|name
+1|Bank or savings association.
+2|Minority depository institution.
+3|Credit union.
+4|Nondepository institution.
+5|Community development financial institution (CDFI).
+6|Other nonprofit financial institution.
+7|Farm Credit System institution.
+8|Government lender.
+9|Commercial finance company.
+10|Equipment finance company.
+11|Industrial loan company.
+12|Online lender.
+13|Other

db_revisions/utils.py

@@ -1,6 +1,8 @@
 from alembic import op
 from sqlalchemy import engine_from_config
 import sqlalchemy
+from csv import DictReader
+import os
 
 
 def table_exists(table_name):
@@ -11,3 +13,14 @@ def table_exists(table_name):
     inspector = sqlalchemy.inspect(engine)
     tables = inspector.get_table_names()
     return table_name in tables
+
+
+def get_feed_data_from_file(table_name):
+    file_dir = os.path.dirname(os.path.realpath(__file__))
+    data_file_path = f"{file_dir}/feed/%s.csv" % table_name
+    data_file = open(data_file_path, "r")
+    reader = DictReader(data_file, delimiter="|")
+    output_list = list()
+    for dictionary in reader:
+        output_list.append(dictionary)
+    return output_list

db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py

@@ -0,0 +1,28 @@
+"""Feed Federal Regulator table
+
+Revision ID: 26a742d97ad9
+Revises: 7b6ff51002b5
+Create Date: 2023-12-14 01:23:17.872728
+
+"""
+from typing import Sequence, Union
+from alembic import op
+from db_revisions.utils import get_feed_data_from_file
+from entities.models import FederalRegulatorDao
+
+
+# revision identifiers, used by Alembic.
+revision: str = "26a742d97ad9"
+down_revision: Union[str, None] = "7b6ff51002b5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    data = get_feed_data_from_file("federal_regulator")
+
+    op.bulk_insert(FederalRegulatorDao.__table__, data)
+
+
+def downgrade() -> None:
+    op.execute(FederalRegulatorDao.__table__.delete())

db_revisions/versions/7b6ff51002b5_feed_address_state_table.py

@@ -0,0 +1,28 @@
+"""Feed Address State table
+
+Revision ID: 7b6ff51002b5
+Revises: 045aa502e050
+Create Date: 2023-12-14 01:21:48.325752
+
+"""
+from typing import Sequence, Union
+from alembic import op
+from db_revisions.utils import get_feed_data_from_file
+from entities.models import AddressStateDao
+
+
+# revision identifiers, used by Alembic.
+revision: str = "7b6ff51002b5"
+down_revision: Union[str, None] = "045aa502e050"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    data = get_feed_data_from_file("address_state")
+
+    op.bulk_insert(AddressStateDao.__table__, data)
+
+
+def downgrade() -> None:
+    op.execute(AddressStateDao.__table__.delete())

db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py

@@ -0,0 +1,27 @@
+"""Feed SBL Institution Type table
+
+Revision ID: a41281b1e109
+Revises: f4ff7d1aa6df
+Create Date: 2023-12-14 01:24:00.120073
+
+"""
+from typing import Sequence, Union
+from alembic import op
+from db_revisions.utils import get_feed_data_from_file
+from entities.models import SBLInstitutionTypeDao
+
+# revision identifiers, used by Alembic.
+revision: str = "a41281b1e109"
+down_revision: Union[str, None] = "f4ff7d1aa6df"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    data = get_feed_data_from_file("sbl_institution_type")
+
+    op.bulk_insert(SBLInstitutionTypeDao.__table__, data)
+
+
+def downgrade() -> None:
+    op.execute(SBLInstitutionTypeDao.__table__.delete())

db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py

@@ -0,0 +1,28 @@
+"""Feed HMDA Institution Type table
+
+Revision ID: f4ff7d1aa6df
+Revises: 26a742d97ad9
+Create Date: 2023-12-14 01:23:47.017878
+
+"""
+from typing import Sequence, Union
+from alembic import op
+from db_revisions.utils import get_feed_data_from_file
+from entities.models import HMDAInstitutionTypeDao
+
+
+# revision identifiers, used by Alembic.
+revision: str = "f4ff7d1aa6df"
+down_revision: Union[str, None] = "26a742d97ad9"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    data = get_feed_data_from_file("hmda_institution_type")
+
+    op.bulk_insert(HMDAInstitutionTypeDao.__table__, data)
+
+
+def downgrade() -> None:
+    op.execute(HMDAInstitutionTypeDao.__table__.delete())

tests/migrations/test_lookup_tables_data_feed.py

@@ -0,0 +1,71 @@
+import os
+import csv
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+from pytest_alembic import MigrationContext
+
+
+def data_feed_helper(table_name):
+    file_dir = os.path.dirname(os.path.realpath(__file__))
+    data_file_path = f"{file_dir}/../../db_revisions/feed/%s.csv" % table_name
+    with open(data_file_path) as f:
+        reader = csv.reader(f, delimiter="|")
+        next(reader)
+        output_list = list(tuple(line) for line in reader)
+    return output_list
+
+
+def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine):
+    # Migrate up to, but not including this new migration
+    alembic_runner.migrate_up_before("7b6ff51002b5")
+
+    # Test address_state feed
+    address_state_tablename = "address_state"
+    alembic_runner.migrate_up_one()
+    with alembic_engine.connect() as conn:
+        address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall()
+    address_state_expected = data_feed_helper(address_state_tablename)
+    assert address_state_rows == address_state_expected
+
+
+def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine):
+    # Migrate up to, but not including this new migration
+    alembic_runner.migrate_up_before("26a742d97ad9")
+
+    # Test federal_regulator feed
+    federal_regulator_tablename = "federal_regulator"
+    alembic_runner.migrate_up_one()
+    with alembic_engine.connect() as conn:
+        federal_regulator_rows = conn.execute(text("SELECT id, name from %s" % federal_regulator_tablename)).fetchall()
+    federal_regulator_expected = data_feed_helper(federal_regulator_tablename)
+    assert federal_regulator_rows == federal_regulator_expected
+
+
+def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine):
+    # Migrate up to, but not including this new migration
+    alembic_runner.migrate_up_before("f4ff7d1aa6df")
+
+    # Test hmda_institution_type feed
+    hmda_institution_type_tablename = "hmda_institution_type"
+    alembic_runner.migrate_up_one()
+    with alembic_engine.connect() as conn:
+        hmda_institution_type_rows = conn.execute(
+            text("SELECT id, name from %s" % hmda_institution_type_tablename)
+        ).fetchall()
+    hmda_institution_type_expected = data_feed_helper(hmda_institution_type_tablename)
+    assert hmda_institution_type_rows == hmda_institution_type_expected
+
+
+def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine):
+    # Migrate up to, but not including this new migration
+    alembic_runner.migrate_up_before("a41281b1e109")
+
+    # Test sbl_institution_type feed
+    sbl_institution_type_tablename = "sbl_institution_type"
+    alembic_runner.migrate_up_one()
+    with alembic_engine.connect() as conn:
+        sbl_institution_type_rows = conn.execute(
+            text("SELECT id, name from %s" % sbl_institution_type_tablename)
+        ).fetchall()
+    sbl_institution_type_expected = data_feed_helper(sbl_institution_type_tablename)
+    assert sbl_institution_type_rows == sbl_institution_type_expected