diff --git a/db_revisions/feed/address_state.csv b/db_revisions/feed/address_state.csv deleted file mode 100644 index 8cf1d0a..0000000 --- a/db_revisions/feed/address_state.csv +++ /dev/null @@ -1,58 +0,0 @@ -code|name -AL|Alabama -AK|Alaska -AZ|Arizona -AR|Arkansas -CA|California -CO|Colorado -CT|Connecticut -DE|Delaware -FL|Florida -GA|Georgia -HI|Hawaii -ID|Idaho -IL|Illinois -IN|Indiana -IA|Iowa -KS|Kansas -KY|Kentucky -LA|Louisiana -ME|Maine -MD|Maryland -MA|Massachusetts -MI|Michigan -MN|Minnesota -MS|Mississippi -MO|Missouri -MT|Montana -NE|Nebraska -NV|Nevada -NH|New Hampshire -NJ|New Jersey -NM|New Mexico -NY|New York -NC|North Carolina -ND|North Dakota -OH|Ohio -OK|Oklahoma -OR|Oregon -PA|Pennsylvania -RI|Rhode Island -SC|South Carolina -SD|South Dakota -TN|Tennessee -TX|Texas -UT|Utah -VT|Vermont -VA|Virginia -WA|Washington -WV|West Virginia -WI|Wisconsin -WY|Wyoming -DC|District of Columbia -AS|American Samoa -GU|Guam -MP|Northern Mariana Islands -PR|Puerto Rico -UM|United States Minor Outlying Islands -VI|Virgin Islands, U.S. \ No newline at end of file diff --git a/db_revisions/feed/federal_regulator.csv b/db_revisions/feed/federal_regulator.csv deleted file mode 100644 index a7cdab5..0000000 --- a/db_revisions/feed/federal_regulator.csv +++ /dev/null @@ -1,8 +0,0 @@ -id|name -FCA|Farm Credit Administration -FDIC|Federal Deposit Insurance Corporation -FHFA|Federal Housing Finance Agency -FRS|Federal Reserve System -NCUA|National Credit Union Administration -OCC|Office of the Comptroller of the Currency -OTS|Office of Thrift Supervision (only valid until July 21, 2011) \ No newline at end of file diff --git a/db_revisions/feed/hmda_institution_type.csv b/db_revisions/feed/hmda_institution_type.csv deleted file mode 100644 index 49d71d7..0000000 --- a/db_revisions/feed/hmda_institution_type.csv +++ /dev/null @@ -1,19 +0,0 @@ -id|name -1|National Bank (OCC supervised) -2|State Member Bank (FRS Supervised): -3|State non-member bank (FDIC supervised) -4|State Chartered Thrift (FDIC supervised) -5|Federal Chartered Thrift (OCC supervised) -6|Credit Union (NCUA supervised) -7|Federal Branch or Agency of Foreign Banking Organization (FBO) -8|Branch or Agency of FBO (FRS supervised) -9|MBS of national Bank (OCC supervised) -10|MBS of state member bank (FRS supervised) -11|MBS of state non-member bank (FDIC supervised) -12|MBS of Bank Holding Company (BHC) (FRS supervised) -13|MBS of credit union (NCUA supervised) -14|independent MBS, no depository affiliation -15|MBS of Savings and Loan Holding Co -16|MBS of state chartered Thrift -17|MBS of federally chartered thrift (OCC supervised) -18|Affiliate of depository institution. MBS is in the same ownership org as a depository. \ No newline at end of file diff --git a/db_revisions/feed/sbl_institution_type.csv b/db_revisions/feed/sbl_institution_type.csv deleted file mode 100644 index 5d150ae..0000000 --- a/db_revisions/feed/sbl_institution_type.csv +++ /dev/null @@ -1,14 +0,0 @@ -id|name -1|Bank or savings association. -2|Minority depository institution. -3|Credit union. -4|Nondepository institution. -5|Community development financial institution (CDFI). -6|Other nonprofit financial institution. -7|Farm Credit System institution. -8|Government lender. -9|Commercial finance company. -10|Equipment finance company. -11|Industrial loan company. -12|Online lender. -13|Other \ No newline at end of file diff --git a/db_revisions/utils.py b/db_revisions/utils.py index 2119987..d84fa39 100644 --- a/db_revisions/utils.py +++ b/db_revisions/utils.py @@ -1,8 +1,6 @@ from alembic import op from sqlalchemy import engine_from_config import sqlalchemy -from csv import DictReader -import os def table_exists(table_name): @@ -13,14 +11,3 @@ def table_exists(table_name): inspector = sqlalchemy.inspect(engine) tables = inspector.get_table_names() return table_name in tables - - -def get_feed_data_from_file(table_name): - file_dir = os.path.dirname(os.path.realpath(__file__)) - data_file_path = f"{file_dir}/feed/%s.csv" % table_name - data_file = open(data_file_path, "r") - reader = DictReader(data_file, delimiter="|") - output_list = list() - for dictionary in reader: - output_list.append(dictionary) - return output_list diff --git a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py index 7932387..4a4122f 100644 --- a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py +++ b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py @@ -84,7 +84,9 @@ def downgrade() -> None: op.drop_constraint( constraint_name="fk_federal_regulator_financial_institutions", table_name="financial_institutions" ) - op.drop_constraint(constraint_name="fk_address_state_financial_institutions", table_name="financial_institutions") + op.drop_constraint( + constraint_name="fk_address_state_code_financial_institutions", table_name="financial_institutions" + ) op.drop_constraint( constraint_name="fk_hmda_institution_type_financial_institutions", table_name="financial_institutions" ) diff --git a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py index 5302b0d..8a0e2bd 100644 --- a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py +++ b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py @@ -7,8 +7,8 @@ """ from typing import Sequence, Union from alembic import op -from db_revisions.utils import get_feed_data_from_file from entities.models import FederalRegulatorDao +from config import federal_regulator_feed # revision identifiers, used by Alembic. @@ -19,9 +19,7 @@ def upgrade() -> None: - data = get_feed_data_from_file("federal_regulator") - - op.bulk_insert(FederalRegulatorDao.__table__, data) + op.bulk_insert(FederalRegulatorDao.__table__, federal_regulator_feed) def downgrade() -> None: diff --git a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py index 5a1ed40..2a6f109 100644 --- a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py +++ b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py @@ -7,8 +7,8 @@ """ from typing import Sequence, Union from alembic import op -from db_revisions.utils import get_feed_data_from_file from entities.models import AddressStateDao +from config import address_state_feed # revision identifiers, used by Alembic. @@ -19,9 +19,7 @@ def upgrade() -> None: - data = get_feed_data_from_file("address_state") - - op.bulk_insert(AddressStateDao.__table__, data) + op.bulk_insert(AddressStateDao.__table__, address_state_feed) def downgrade() -> None: diff --git a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py index 058087f..0de6ddc 100644 --- a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py +++ b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py @@ -7,8 +7,8 @@ """ from typing import Sequence, Union from alembic import op -from db_revisions.utils import get_feed_data_from_file from entities.models import SBLInstitutionTypeDao +from config import sbl_institution_type_feed # revision identifiers, used by Alembic. revision: str = "a41281b1e109" @@ -18,9 +18,7 @@ def upgrade() -> None: - data = get_feed_data_from_file("sbl_institution_type") - - op.bulk_insert(SBLInstitutionTypeDao.__table__, data) + op.bulk_insert(SBLInstitutionTypeDao.__table__, sbl_institution_type_feed) def downgrade() -> None: diff --git a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py index 1502c9e..7725962 100644 --- a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py +++ b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py @@ -7,8 +7,8 @@ """ from typing import Sequence, Union from alembic import op -from db_revisions.utils import get_feed_data_from_file from entities.models import HMDAInstitutionTypeDao +from config import hmda_institution_type_feed # revision identifiers, used by Alembic. @@ -19,9 +19,7 @@ def upgrade() -> None: - data = get_feed_data_from_file("hmda_institution_type") - - op.bulk_insert(HMDAInstitutionTypeDao.__table__, data) + op.bulk_insert(HMDAInstitutionTypeDao.__table__, hmda_institution_type_feed) def downgrade() -> None: diff --git a/src/config.py b/src/config.py index d8a3f4e..1ee950b 100644 --- a/src/config.py +++ b/src/config.py @@ -6,6 +6,125 @@ from pydantic.types import SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict +""" +START Feed data for the lookup tables: + address_state + federal_regulator + hmda_institution_type + sbl_institution_type +These are accessed in db_revisions/versions/* scripts and in test/migrations/test_lookup_tables_data_feed . +""" + +address_state_feed = [ + {"code": "AL", "name": "Alabama"}, + {"code": "AK", "name": "Alaska"}, + {"code": "AZ", "name": "Arizona"}, + {"code": "AR", "name": "Arkansas"}, + {"code": "CA", "name": "California"}, + {"code": "CO", "name": "Colorado"}, + {"code": "CT", "name": "Connecticut"}, + {"code": "DE", "name": "Delaware"}, + {"code": "FL", "name": "Florida"}, + {"code": "GA", "name": "Georgia"}, + {"code": "HI", "name": "Hawaii"}, + {"code": "ID", "name": "Idaho"}, + {"code": "IL", "name": "Illinois"}, + {"code": "IN", "name": "Indiana"}, + {"code": "IA", "name": "Iowa"}, + {"code": "KS", "name": "Kansas"}, + {"code": "KY", "name": "Kentucky"}, + {"code": "LA", "name": "Louisiana"}, + {"code": "ME", "name": "Maine"}, + {"code": "MD", "name": "Maryland"}, + {"code": "MA", "name": "Massachusetts"}, + {"code": "MI", "name": "Michigan"}, + {"code": "MN", "name": "Minnesota"}, + {"code": "MS", "name": "Mississippi"}, + {"code": "MO", "name": "Missouri"}, + {"code": "MT", "name": "Montana"}, + {"code": "NE", "name": "Nebraska"}, + {"code": "NV", "name": "Nevada"}, + {"code": "NH", "name": "New Hampshire"}, + {"code": "NJ", "name": "New Jersey"}, + {"code": "NM", "name": "New Mexico"}, + {"code": "NY", "name": "New York"}, + {"code": "NC", "name": "North Carolina"}, + {"code": "ND", "name": "North Dakota"}, + {"code": "OH", "name": "Ohio"}, + {"code": "OK", "name": "Oklahoma"}, + {"code": "OR", "name": "Oregon"}, + {"code": "PA", "name": "Pennsylvania"}, + {"code": "RI", "name": "Rhode Island"}, + {"code": "SC", "name": "South Carolina"}, + {"code": "SD", "name": "South Dakota"}, + {"code": "TN", "name": "Tennessee"}, + {"code": "TX", "name": "Texas"}, + {"code": "UT", "name": "Utah"}, + {"code": "VT", "name": "Vermont"}, + {"code": "VA", "name": "Virginia"}, + {"code": "WA", "name": "Washington"}, + {"code": "WV", "name": "West Virginia"}, + {"code": "WI", "name": "Wisconsin"}, + {"code": "WY", "name": "Wyoming"}, + {"code": "DC", "name": "District of Columbia"}, + {"code": "AS", "name": "American Samoa"}, + {"code": "GU", "name": "Guam"}, + {"code": "MP", "name": "Northern Mariana Islands"}, + {"code": "PR", "name": "Puerto Rico"}, + {"code": "UM", "name": "United States Minor Outlying Islands"}, + {"code": "VI", "name": "Virgin Islands, U.S."}, +] + +federal_regulator_feed = [ + {"id": "FCA", "name": "Farm Credit Administration"}, + {"id": "FDIC", "name": "Federal Deposit Insurance Corporation"}, + {"id": "FHFA", "name": "Federal Housing Finance Agency"}, + {"id": "FRS", "name": "Federal Reserve System"}, + {"id": "NCUA", "name": "National Credit Union Administration"}, + {"id": "OCC", "name": "Office of the Comptroller of the Currency"}, + {"id": "OTS", "name": "Office of Thrift Supervision (only valid until July 21, 2011)"}, +] + +sbl_institution_type_feed = [ + {"id": "1", "name": "Bank or savings association."}, + {"id": "2", "name": "Minority depository institution."}, + {"id": "3", "name": "Credit union."}, + {"id": "4", "name": "Nondepository institution."}, + {"id": "5", "name": "Community development financial institution (CDFI)."}, + {"id": "6", "name": "Other nonprofit financial institution."}, + {"id": "7", "name": "Farm Credit System institution."}, + {"id": "8", "name": "Government lender."}, + {"id": "9", "name": "Commercial finance company."}, + {"id": "10", "name": "Equipment finance company."}, + {"id": "11", "name": "Industrial loan company."}, + {"id": "12", "name": "Online lender."}, + {"id": "13", "name": "Other"}, +] + +hmda_institution_type_feed = [ + {"id": "1", "name": "National Bank (OCC supervised)"}, + {"id": "2", "name": "State Member Bank (FRS Supervised)"}, + {"id": "3", "name": "State non-member bank (FDIC supervised)"}, + {"id": "4", "name": "State Chartered Thrift (FDIC supervised)"}, + {"id": "5", "name": "Federal Chartered Thrift (OCC supervised)"}, + {"id": "6", "name": "Credit Union (NCUA supervised)"}, + {"id": "7", "name": "Federal Branch or Agency of Foreign Banking Organization (FBO)"}, + {"id": "8", "name": "Branch or Agency of FBO (FRS supervised)"}, + {"id": "9", "name": "MBS of national Bank (OCC supervised)"}, + {"id": "10", "name": "MBS of state member bank (FRS supervised)"}, + {"id": "11", "name": "MBS of state non-member bank (FDIC supervised)"}, + {"id": "12", "name": "MBS of Bank Holding Company (BHC) (FRS supervised)"}, + {"id": "13", "name": "MBS of credit union (NCUA supervised)"}, + {"id": "14", "name": "independent MBS, no depository affiliation"}, + {"id": "15", "name": "MBS of Savings and Loan Holding Co"}, + {"id": "16", "name": "MBS of state chartered Thrift"}, + {"id": "17", "name": "MBS of federally chartered thrift (OCC supervised)"}, + {"id": "18", "name": "Affiliate of depository institution. MBS is in the same ownership org as a depository."}, +] + +""" +END Feed data for the lookup tables: +""" JWT_OPTS_PREFIX = "jwt_opts_" env_files_to_load = [".env"] diff --git a/tests/migrations/test_lookup_tables_data_feed.py b/tests/migrations/test_lookup_tables_data_feed.py index ed74161..33de741 100644 --- a/tests/migrations/test_lookup_tables_data_feed.py +++ b/tests/migrations/test_lookup_tables_data_feed.py @@ -1,18 +1,13 @@ -import os -import csv from sqlalchemy import text from sqlalchemy.engine import Engine from pytest_alembic import MigrationContext - -def data_feed_helper(table_name): - file_dir = os.path.dirname(os.path.realpath(__file__)) - data_file_path = f"{file_dir}/../../db_revisions/feed/%s.csv" % table_name - with open(data_file_path) as f: - reader = csv.reader(f, delimiter="|") - next(reader) - output_list = list(tuple(line) for line in reader) - return output_list +from config import ( + address_state_feed, + federal_regulator_feed, + hmda_institution_type_feed, + sbl_institution_type_feed, +) def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): @@ -24,7 +19,8 @@ def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engin alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() - address_state_expected = data_feed_helper(address_state_tablename) + address_state_expected = [tuple(dict.values()) for dict in address_state_feed] + assert address_state_rows == address_state_expected @@ -37,7 +33,8 @@ def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_e alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: federal_regulator_rows = conn.execute(text("SELECT id, name from %s" % federal_regulator_tablename)).fetchall() - federal_regulator_expected = data_feed_helper(federal_regulator_tablename) + federal_regulator_expected = [tuple(dict.values()) for dict in federal_regulator_feed] + assert federal_regulator_rows == federal_regulator_expected @@ -52,7 +49,8 @@ def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alemb hmda_institution_type_rows = conn.execute( text("SELECT id, name from %s" % hmda_institution_type_tablename) ).fetchall() - hmda_institution_type_expected = data_feed_helper(hmda_institution_type_tablename) + hmda_institution_type_expected = [tuple(dict.values()) for dict in hmda_institution_type_feed] + assert hmda_institution_type_rows == hmda_institution_type_expected @@ -67,5 +65,6 @@ def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembi sbl_institution_type_rows = conn.execute( text("SELECT id, name from %s" % sbl_institution_type_tablename) ).fetchall() - sbl_institution_type_expected = data_feed_helper(sbl_institution_type_tablename) + sbl_institution_type_expected = [tuple(dict.values()) for dict in sbl_institution_type_feed] + assert sbl_institution_type_rows == sbl_institution_type_expected diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py index 2a27197..5059002 100644 --- a/tests/migrations/test_migrations.py +++ b/tests/migrations/test_migrations.py @@ -10,15 +10,32 @@ from pytest_alembic import MigrationContext -def test_tables_exist_after_migration(alembic_runner: MigrationContext, alembic_engine: Engine): +def test_tables_exist_migrate_up_to_045aa502e050(alembic_runner: MigrationContext, alembic_engine: Engine): alembic_runner.migrate_up_to("045aa502e050") inspector = sqlalchemy.inspect(alembic_engine) tables = inspector.get_table_names() - assert "denied_domains" in tables - assert "financial_institutions" in tables - assert "financial_institution_domains" in tables assert "address_state" in tables assert "federal_regulator" in tables assert "hmda_institution_type" in tables assert "sbl_institution_type" in tables + + +def test_tables_exist_migrate_up_to_20e0d51d8be9(alembic_runner: MigrationContext, alembic_engine: Engine): + alembic_runner.migrate_up_to("20e0d51d8be9") + + inspector = sqlalchemy.inspect(alembic_engine) + tables = inspector.get_table_names() + assert "denied_domains" in tables + assert "financial_institutions" in tables + assert "financial_institution_domains" in tables + + +def test_tables_not_exist_migrate_down_to_base(alembic_runner: MigrationContext, alembic_engine: Engine): + alembic_runner.migrate_down_to("base") + + inspector = sqlalchemy.inspect(alembic_engine) + tables = inspector.get_table_names() + assert "denied_domains" not in tables + assert "financial_institutions" not in tables + assert "financial_institution_domains" not in tables diff --git a/tests/migrations/test_schema.py b/tests/migrations/test_schema.py new file mode 100644 index 0000000..09b6ac4 --- /dev/null +++ b/tests/migrations/test_schema.py @@ -0,0 +1,56 @@ +import sqlalchemy +from sqlalchemy.engine import Engine + +from pytest_alembic import MigrationContext + + +def test_financial_institutions_schema_migrate_up_to_045aa502e050( + alembic_runner: MigrationContext, alembic_engine: Engine +): + alembic_runner.migrate_up_to("045aa502e050") + + inspector = sqlalchemy.inspect(alembic_engine) + expexted_columns = [ + "lei", + "name", + "event_time", + "tax_id", + "rssd_id", + "primary_federal_regulator_id", + "hmda_institution_type_id", + "sbl_institution_type_id", + "hq_address_street_1", + "hq_address_street_2", + "hq_address_city", + "hq_address_state_code", + "hq_address_zip", + "parent_lei", + "parent_legal_name", + "parent_rssd_id", + "top_holder_lei", + "top_holder_legal_name", + "top_holder_rssd_id", + ] + + columns = inspector.get_columns("financial_institutions") + columns_names = [column.get("name") for column in columns] + + assert columns_names == expexted_columns + + +def test_financial_institutions_schema_migrate_up_to_20e0d51d8be9( + alembic_runner: MigrationContext, alembic_engine: Engine +): + alembic_runner.migrate_up_to("20e0d51d8be9") + + inspector = sqlalchemy.inspect(alembic_engine) + expexted_columns = [ + "lei", + "name", + "event_time", + ] + + columns = inspector.get_columns("financial_institutions") + columns_names = [column.get("name") for column in columns] + + assert columns_names == expexted_columns