From 176ad6e3435fe3a926ce67814f67dc970c49c44b Mon Sep 17 00:00:00 2001 From: nargis-sultani <102271080+nargis-sultani@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:59:05 -0500 Subject: [PATCH] Features/61 alembic seed lookup tables (#65) Co-authored-by: Nargis Sultani --- db_revisions/feed/address_state.csv | 58 +++++++++++++++ db_revisions/feed/federal_regulator.csv | 8 +++ db_revisions/feed/hmda_institution_type.csv | 19 +++++ db_revisions/feed/sbl_institution_type.csv | 14 ++++ db_revisions/utils.py | 13 ++++ ...a742d97ad9_feed_federal_regulator_table.py | 28 ++++++++ .../7b6ff51002b5_feed_address_state_table.py | 28 ++++++++ ...1b1e109_feed_sbl_institution_type_table.py | 27 +++++++ ...1aa6df_feed_hmda_institution_type_table.py | 28 ++++++++ .../test_lookup_tables_data_feed.py | 71 +++++++++++++++++++ 10 files changed, 294 insertions(+) create mode 100644 db_revisions/feed/address_state.csv create mode 100644 db_revisions/feed/federal_regulator.csv create mode 100644 db_revisions/feed/hmda_institution_type.csv create mode 100644 db_revisions/feed/sbl_institution_type.csv create mode 100644 db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py create mode 100644 db_revisions/versions/7b6ff51002b5_feed_address_state_table.py create mode 100644 db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py create mode 100644 db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py create mode 100644 tests/migrations/test_lookup_tables_data_feed.py diff --git a/db_revisions/feed/address_state.csv b/db_revisions/feed/address_state.csv new file mode 100644 index 0000000..8cf1d0a --- /dev/null +++ b/db_revisions/feed/address_state.csv @@ -0,0 +1,58 @@ +code|name +AL|Alabama +AK|Alaska +AZ|Arizona +AR|Arkansas +CA|California +CO|Colorado +CT|Connecticut +DE|Delaware +FL|Florida +GA|Georgia +HI|Hawaii +ID|Idaho +IL|Illinois +IN|Indiana +IA|Iowa +KS|Kansas +KY|Kentucky +LA|Louisiana +ME|Maine +MD|Maryland +MA|Massachusetts +MI|Michigan +MN|Minnesota +MS|Mississippi +MO|Missouri +MT|Montana +NE|Nebraska +NV|Nevada +NH|New Hampshire +NJ|New Jersey +NM|New Mexico +NY|New York +NC|North Carolina +ND|North Dakota +OH|Ohio +OK|Oklahoma +OR|Oregon +PA|Pennsylvania +RI|Rhode Island +SC|South Carolina +SD|South Dakota +TN|Tennessee +TX|Texas +UT|Utah +VT|Vermont +VA|Virginia +WA|Washington +WV|West Virginia +WI|Wisconsin +WY|Wyoming +DC|District of Columbia +AS|American Samoa +GU|Guam +MP|Northern Mariana Islands +PR|Puerto Rico +UM|United States Minor Outlying Islands +VI|Virgin Islands, U.S. \ No newline at end of file diff --git a/db_revisions/feed/federal_regulator.csv b/db_revisions/feed/federal_regulator.csv new file mode 100644 index 0000000..a7cdab5 --- /dev/null +++ b/db_revisions/feed/federal_regulator.csv @@ -0,0 +1,8 @@ +id|name +FCA|Farm Credit Administration +FDIC|Federal Deposit Insurance Corporation +FHFA|Federal Housing Finance Agency +FRS|Federal Reserve System +NCUA|National Credit Union Administration +OCC|Office of the Comptroller of the Currency +OTS|Office of Thrift Supervision (only valid until July 21, 2011) \ No newline at end of file diff --git a/db_revisions/feed/hmda_institution_type.csv b/db_revisions/feed/hmda_institution_type.csv new file mode 100644 index 0000000..49d71d7 --- /dev/null +++ b/db_revisions/feed/hmda_institution_type.csv @@ -0,0 +1,19 @@ +id|name +1|National Bank (OCC supervised) +2|State Member Bank (FRS Supervised): +3|State non-member bank (FDIC supervised) +4|State Chartered Thrift (FDIC supervised) +5|Federal Chartered Thrift (OCC supervised) +6|Credit Union (NCUA supervised) +7|Federal Branch or Agency of Foreign Banking Organization (FBO) +8|Branch or Agency of FBO (FRS supervised) +9|MBS of national Bank (OCC supervised) +10|MBS of state member bank (FRS supervised) +11|MBS of state non-member bank (FDIC supervised) +12|MBS of Bank Holding Company (BHC) (FRS supervised) +13|MBS of credit union (NCUA supervised) +14|independent MBS, no depository affiliation +15|MBS of Savings and Loan Holding Co +16|MBS of state chartered Thrift +17|MBS of federally chartered thrift (OCC supervised) +18|Affiliate of depository institution. MBS is in the same ownership org as a depository. \ No newline at end of file diff --git a/db_revisions/feed/sbl_institution_type.csv b/db_revisions/feed/sbl_institution_type.csv new file mode 100644 index 0000000..5d150ae --- /dev/null +++ b/db_revisions/feed/sbl_institution_type.csv @@ -0,0 +1,14 @@ +id|name +1|Bank or savings association. +2|Minority depository institution. +3|Credit union. +4|Nondepository institution. +5|Community development financial institution (CDFI). +6|Other nonprofit financial institution. +7|Farm Credit System institution. +8|Government lender. +9|Commercial finance company. +10|Equipment finance company. +11|Industrial loan company. +12|Online lender. +13|Other \ No newline at end of file diff --git a/db_revisions/utils.py b/db_revisions/utils.py index d84fa39..2119987 100644 --- a/db_revisions/utils.py +++ b/db_revisions/utils.py @@ -1,6 +1,8 @@ from alembic import op from sqlalchemy import engine_from_config import sqlalchemy +from csv import DictReader +import os def table_exists(table_name): @@ -11,3 +13,14 @@ def table_exists(table_name): inspector = sqlalchemy.inspect(engine) tables = inspector.get_table_names() return table_name in tables + + +def get_feed_data_from_file(table_name): + file_dir = os.path.dirname(os.path.realpath(__file__)) + data_file_path = f"{file_dir}/feed/%s.csv" % table_name + data_file = open(data_file_path, "r") + reader = DictReader(data_file, delimiter="|") + output_list = list() + for dictionary in reader: + output_list.append(dictionary) + return output_list diff --git a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py new file mode 100644 index 0000000..5302b0d --- /dev/null +++ b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py @@ -0,0 +1,28 @@ +"""Feed Federal Regulator table + +Revision ID: 26a742d97ad9 +Revises: 7b6ff51002b5 +Create Date: 2023-12-14 01:23:17.872728 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import FederalRegulatorDao + + +# revision identifiers, used by Alembic. +revision: str = "26a742d97ad9" +down_revision: Union[str, None] = "7b6ff51002b5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file("federal_regulator") + + op.bulk_insert(FederalRegulatorDao.__table__, data) + + +def downgrade() -> None: + op.execute(FederalRegulatorDao.__table__.delete()) diff --git a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py new file mode 100644 index 0000000..5a1ed40 --- /dev/null +++ b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py @@ -0,0 +1,28 @@ +"""Feed Address State table + +Revision ID: 7b6ff51002b5 +Revises: 045aa502e050 +Create Date: 2023-12-14 01:21:48.325752 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import AddressStateDao + + +# revision identifiers, used by Alembic. +revision: str = "7b6ff51002b5" +down_revision: Union[str, None] = "045aa502e050" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file("address_state") + + op.bulk_insert(AddressStateDao.__table__, data) + + +def downgrade() -> None: + op.execute(AddressStateDao.__table__.delete()) diff --git a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py new file mode 100644 index 0000000..058087f --- /dev/null +++ b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py @@ -0,0 +1,27 @@ +"""Feed SBL Institution Type table + +Revision ID: a41281b1e109 +Revises: f4ff7d1aa6df +Create Date: 2023-12-14 01:24:00.120073 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import SBLInstitutionTypeDao + +# revision identifiers, used by Alembic. +revision: str = "a41281b1e109" +down_revision: Union[str, None] = "f4ff7d1aa6df" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file("sbl_institution_type") + + op.bulk_insert(SBLInstitutionTypeDao.__table__, data) + + +def downgrade() -> None: + op.execute(SBLInstitutionTypeDao.__table__.delete()) diff --git a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py new file mode 100644 index 0000000..1502c9e --- /dev/null +++ b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py @@ -0,0 +1,28 @@ +"""Feed HMDA Institution Type table + +Revision ID: f4ff7d1aa6df +Revises: 26a742d97ad9 +Create Date: 2023-12-14 01:23:47.017878 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import HMDAInstitutionTypeDao + + +# revision identifiers, used by Alembic. +revision: str = "f4ff7d1aa6df" +down_revision: Union[str, None] = "26a742d97ad9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file("hmda_institution_type") + + op.bulk_insert(HMDAInstitutionTypeDao.__table__, data) + + +def downgrade() -> None: + op.execute(HMDAInstitutionTypeDao.__table__.delete()) diff --git a/tests/migrations/test_lookup_tables_data_feed.py b/tests/migrations/test_lookup_tables_data_feed.py new file mode 100644 index 0000000..ed74161 --- /dev/null +++ b/tests/migrations/test_lookup_tables_data_feed.py @@ -0,0 +1,71 @@ +import os +import csv +from sqlalchemy import text +from sqlalchemy.engine import Engine +from pytest_alembic import MigrationContext + + +def data_feed_helper(table_name): + file_dir = os.path.dirname(os.path.realpath(__file__)) + data_file_path = f"{file_dir}/../../db_revisions/feed/%s.csv" % table_name + with open(data_file_path) as f: + reader = csv.reader(f, delimiter="|") + next(reader) + output_list = list(tuple(line) for line in reader) + return output_list + + +def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("7b6ff51002b5") + + # Test address_state feed + address_state_tablename = "address_state" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() + address_state_expected = data_feed_helper(address_state_tablename) + assert address_state_rows == address_state_expected + + +def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("26a742d97ad9") + + # Test federal_regulator feed + federal_regulator_tablename = "federal_regulator" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + federal_regulator_rows = conn.execute(text("SELECT id, name from %s" % federal_regulator_tablename)).fetchall() + federal_regulator_expected = data_feed_helper(federal_regulator_tablename) + assert federal_regulator_rows == federal_regulator_expected + + +def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("f4ff7d1aa6df") + + # Test hmda_institution_type feed + hmda_institution_type_tablename = "hmda_institution_type" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + hmda_institution_type_rows = conn.execute( + text("SELECT id, name from %s" % hmda_institution_type_tablename) + ).fetchall() + hmda_institution_type_expected = data_feed_helper(hmda_institution_type_tablename) + assert hmda_institution_type_rows == hmda_institution_type_expected + + +def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("a41281b1e109") + + # Test sbl_institution_type feed + sbl_institution_type_tablename = "sbl_institution_type" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + sbl_institution_type_rows = conn.execute( + text("SELECT id, name from %s" % sbl_institution_type_tablename) + ).fetchall() + sbl_institution_type_expected = data_feed_helper(sbl_institution_type_tablename) + assert sbl_institution_type_rows == sbl_institution_type_expected