diff --git a/README.md b/README.md index 307dbc6..00bd9c3 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,14 @@ $ docker compose up -d pg keycloak ``` ![Docker](images/sbl_project_svcs.png) +--- +## Resetting DB and Seeding mock data +On app startup, alembic creates all the tables and seeds the lookup tables. +Running the below script with the 'reset' argument will reset the db: +db_revisions/dev_setup.sh reset +Passing the 'reset-then-seed' argument to the script will reset the db and then seed the lookup tables: +db_revisions/dev_setup.sh reset-then-seed + --- ## Running the app Once the [Dependencies](#dependencies), and [Pre-requisites](#pre-requisites) have been satisfied: diff --git a/db_revisions/__init__.py b/db_revisions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db_revisions/dev_setup.sh b/db_revisions/dev_setup.sh new file mode 100755 index 0000000..87fcd05 --- /dev/null +++ b/db_revisions/dev_setup.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +if [[ $1 == "" ]] +then + echo "No arguments passed!" + echo "The argument must be either reset or reset-then-seed" + exit 0 +fi + +ACTION=$1 + +if [ $ACTION == "reset" ] +then + #If only need to reset db + poetry run alembic downgrade base +elif [ $ACTION == "reset-then-seed" ] +then + #First reset the db + poetry run alembic downgrade base + #Then upgrade it to head + poetry run alembic upgrade head +fi \ No newline at end of file diff --git a/db_revisions/feed/address_state.csv b/db_revisions/feed/address_state.csv deleted file mode 100644 index 8cf1d0a..0000000 --- a/db_revisions/feed/address_state.csv +++ /dev/null @@ -1,58 +0,0 @@ -code|name -AL|Alabama -AK|Alaska -AZ|Arizona -AR|Arkansas -CA|California -CO|Colorado -CT|Connecticut -DE|Delaware -FL|Florida -GA|Georgia -HI|Hawaii -ID|Idaho -IL|Illinois -IN|Indiana -IA|Iowa -KS|Kansas -KY|Kentucky -LA|Louisiana -ME|Maine -MD|Maryland -MA|Massachusetts -MI|Michigan -MN|Minnesota -MS|Mississippi -MO|Missouri -MT|Montana -NE|Nebraska -NV|Nevada -NH|New Hampshire -NJ|New Jersey -NM|New Mexico -NY|New York -NC|North Carolina -ND|North Dakota -OH|Ohio -OK|Oklahoma -OR|Oregon -PA|Pennsylvania -RI|Rhode Island -SC|South Carolina -SD|South Dakota -TN|Tennessee -TX|Texas -UT|Utah -VT|Vermont -VA|Virginia -WA|Washington -WV|West Virginia -WI|Wisconsin -WY|Wyoming -DC|District of Columbia -AS|American Samoa -GU|Guam -MP|Northern Mariana Islands -PR|Puerto Rico -UM|United States Minor Outlying Islands -VI|Virgin Islands, U.S. \ No newline at end of file diff --git a/db_revisions/feed/federal_regulator.csv b/db_revisions/feed/federal_regulator.csv deleted file mode 100644 index a7cdab5..0000000 --- a/db_revisions/feed/federal_regulator.csv +++ /dev/null @@ -1,8 +0,0 @@ -id|name -FCA|Farm Credit Administration -FDIC|Federal Deposit Insurance Corporation -FHFA|Federal Housing Finance Agency -FRS|Federal Reserve System -NCUA|National Credit Union Administration -OCC|Office of the Comptroller of the Currency -OTS|Office of Thrift Supervision (only valid until July 21, 2011) \ No newline at end of file diff --git a/db_revisions/feed/hmda_institution_type.csv b/db_revisions/feed/hmda_institution_type.csv deleted file mode 100644 index 49d71d7..0000000 --- a/db_revisions/feed/hmda_institution_type.csv +++ /dev/null @@ -1,19 +0,0 @@ -id|name -1|National Bank (OCC supervised) -2|State Member Bank (FRS Supervised): -3|State non-member bank (FDIC supervised) -4|State Chartered Thrift (FDIC supervised) -5|Federal Chartered Thrift (OCC supervised) -6|Credit Union (NCUA supervised) -7|Federal Branch or Agency of Foreign Banking Organization (FBO) -8|Branch or Agency of FBO (FRS supervised) -9|MBS of national Bank (OCC supervised) -10|MBS of state member bank (FRS supervised) -11|MBS of state non-member bank (FDIC supervised) -12|MBS of Bank Holding Company (BHC) (FRS supervised) -13|MBS of credit union (NCUA supervised) -14|independent MBS, no depository affiliation -15|MBS of Savings and Loan Holding Co -16|MBS of state chartered Thrift -17|MBS of federally chartered thrift (OCC supervised) -18|Affiliate of depository institution. MBS is in the same ownership org as a depository. \ No newline at end of file diff --git a/db_revisions/feed/sbl_institution_type.csv b/db_revisions/feed/sbl_institution_type.csv deleted file mode 100644 index 5d150ae..0000000 --- a/db_revisions/feed/sbl_institution_type.csv +++ /dev/null @@ -1,14 +0,0 @@ -id|name -1|Bank or savings association. -2|Minority depository institution. -3|Credit union. -4|Nondepository institution. -5|Community development financial institution (CDFI). -6|Other nonprofit financial institution. -7|Farm Credit System institution. -8|Government lender. -9|Commercial finance company. -10|Equipment finance company. -11|Industrial loan company. -12|Online lender. -13|Other \ No newline at end of file diff --git a/db_revisions/utils.py b/db_revisions/utils.py index 2119987..2e26400 100644 --- a/db_revisions/utils.py +++ b/db_revisions/utils.py @@ -1,8 +1,7 @@ +from typing import Any, List, Dict from alembic import op -from sqlalchemy import engine_from_config +from sqlalchemy import MetaData, Table, engine_from_config import sqlalchemy -from csv import DictReader -import os def table_exists(table_name): @@ -15,12 +14,12 @@ def table_exists(table_name): return table_name in tables -def get_feed_data_from_file(table_name): - file_dir = os.path.dirname(os.path.realpath(__file__)) - data_file_path = f"{file_dir}/feed/%s.csv" % table_name - data_file = open(data_file_path, "r") - reader = DictReader(data_file, delimiter="|") - output_list = list() - for dictionary in reader: - output_list.append(dictionary) - return output_list +def get_table_by_name(table_name): + meta = MetaData() + meta.reflect(op.get_bind()) + table = Table(table_name, meta) + return table + + +def get_indices_from_collection(data: List[Dict[Any, Any]], key: str) -> List[Any]: + return [e[key] for e in data] diff --git a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py deleted file mode 100644 index 5302b0d..0000000 --- a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Feed Federal Regulator table - -Revision ID: 26a742d97ad9 -Revises: 7b6ff51002b5 -Create Date: 2023-12-14 01:23:17.872728 - -""" -from typing import Sequence, Union -from alembic import op -from db_revisions.utils import get_feed_data_from_file -from entities.models import FederalRegulatorDao - - -# revision identifiers, used by Alembic. -revision: str = "26a742d97ad9" -down_revision: Union[str, None] = "7b6ff51002b5" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - data = get_feed_data_from_file("federal_regulator") - - op.bulk_insert(FederalRegulatorDao.__table__, data) - - -def downgrade() -> None: - op.execute(FederalRegulatorDao.__table__.delete()) diff --git a/db_revisions/versions/26a742d97ad9_seed_federal_regulator_table.py b/db_revisions/versions/26a742d97ad9_seed_federal_regulator_table.py new file mode 100644 index 0000000..6301b27 --- /dev/null +++ b/db_revisions/versions/26a742d97ad9_seed_federal_regulator_table.py @@ -0,0 +1,41 @@ +"""Seed Federal Regulator table + +Revision ID: 26a742d97ad9 +Revises: 7b6ff51002b5 +Create Date: 2023-12-14 01:23:17.872728 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_table_by_name, get_indices_from_collection + + +# revision identifiers, used by Alembic. +revision: str = "26a742d97ad9" +down_revision: Union[str, None] = "7b6ff51002b5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +seed_data = [ + {"id": "FCA", "name": "Farm Credit Administration"}, + {"id": "FDIC", "name": "Federal Deposit Insurance Corporation"}, + {"id": "FHFA", "name": "Federal Housing Finance Agency"}, + {"id": "FRS", "name": "Federal Reserve System"}, + {"id": "NCUA", "name": "National Credit Union Administration"}, + {"id": "OCC", "name": "Office of the Comptroller of the Currency"}, + {"id": "OTS", "name": "Office of Thrift Supervision (only valid until July 21, 2011)"}, +] + + +def upgrade() -> None: + table = get_table_by_name("federal_regulator") + + op.bulk_insert(table, seed_data) + + +def downgrade() -> None: + table = get_table_by_name("federal_regulator") + + ids = get_indices_from_collection(seed_data, "id") + + op.execute(table.delete().where(table.c.id.in_(ids))) diff --git a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py deleted file mode 100644 index 5a1ed40..0000000 --- a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Feed Address State table - -Revision ID: 7b6ff51002b5 -Revises: 045aa502e050 -Create Date: 2023-12-14 01:21:48.325752 - -""" -from typing import Sequence, Union -from alembic import op -from db_revisions.utils import get_feed_data_from_file -from entities.models import AddressStateDao - - -# revision identifiers, used by Alembic. -revision: str = "7b6ff51002b5" -down_revision: Union[str, None] = "045aa502e050" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - data = get_feed_data_from_file("address_state") - - op.bulk_insert(AddressStateDao.__table__, data) - - -def downgrade() -> None: - op.execute(AddressStateDao.__table__.delete()) diff --git a/db_revisions/versions/7b6ff51002b5_seed_address_state_table.py b/db_revisions/versions/7b6ff51002b5_seed_address_state_table.py new file mode 100644 index 0000000..a559876 --- /dev/null +++ b/db_revisions/versions/7b6ff51002b5_seed_address_state_table.py @@ -0,0 +1,91 @@ +"""Seed Address State table + +Revision ID: 7b6ff51002b5 +Revises: 045aa502e050 +Create Date: 2023-12-14 01:21:48.325752 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_table_by_name, get_indices_from_collection + + +# revision identifiers, used by Alembic. +revision: str = "7b6ff51002b5" +down_revision: Union[str, None] = "045aa502e050" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +seed_data = [ + {"code": "AL", "name": "Alabama"}, + {"code": "AK", "name": "Alaska"}, + {"code": "AZ", "name": "Arizona"}, + {"code": "AR", "name": "Arkansas"}, + {"code": "CA", "name": "California"}, + {"code": "CO", "name": "Colorado"}, + {"code": "CT", "name": "Connecticut"}, + {"code": "DE", "name": "Delaware"}, + {"code": "FL", "name": "Florida"}, + {"code": "GA", "name": "Georgia"}, + {"code": "HI", "name": "Hawaii"}, + {"code": "ID", "name": "Idaho"}, + {"code": "IL", "name": "Illinois"}, + {"code": "IN", "name": "Indiana"}, + {"code": "IA", "name": "Iowa"}, + {"code": "KS", "name": "Kansas"}, + {"code": "KY", "name": "Kentucky"}, + {"code": "LA", "name": "Louisiana"}, + {"code": "ME", "name": "Maine"}, + {"code": "MD", "name": "Maryland"}, + {"code": "MA", "name": "Massachusetts"}, + {"code": "MI", "name": "Michigan"}, + {"code": "MN", "name": "Minnesota"}, + {"code": "MS", "name": "Mississippi"}, + {"code": "MO", "name": "Missouri"}, + {"code": "MT", "name": "Montana"}, + {"code": "NE", "name": "Nebraska"}, + {"code": "NV", "name": "Nevada"}, + {"code": "NH", "name": "New Hampshire"}, + {"code": "NJ", "name": "New Jersey"}, + {"code": "NM", "name": "New Mexico"}, + {"code": "NY", "name": "New York"}, + {"code": "NC", "name": "North Carolina"}, + {"code": "ND", "name": "North Dakota"}, + {"code": "OH", "name": "Ohio"}, + {"code": "OK", "name": "Oklahoma"}, + {"code": "OR", "name": "Oregon"}, + {"code": "PA", "name": "Pennsylvania"}, + {"code": "RI", "name": "Rhode Island"}, + {"code": "SC", "name": "South Carolina"}, + {"code": "SD", "name": "South Dakota"}, + {"code": "TN", "name": "Tennessee"}, + {"code": "TX", "name": "Texas"}, + {"code": "UT", "name": "Utah"}, + {"code": "VT", "name": "Vermont"}, + {"code": "VA", "name": "Virginia"}, + {"code": "WA", "name": "Washington"}, + {"code": "WV", "name": "West Virginia"}, + {"code": "WI", "name": "Wisconsin"}, + {"code": "WY", "name": "Wyoming"}, + {"code": "DC", "name": "District of Columbia"}, + {"code": "AS", "name": "American Samoa"}, + {"code": "GU", "name": "Guam"}, + {"code": "MP", "name": "Northern Mariana Islands"}, + {"code": "PR", "name": "Puerto Rico"}, + {"code": "UM", "name": "United States Minor Outlying Islands"}, + {"code": "VI", "name": "Virgin Islands, U.S."}, +] + + +def upgrade() -> None: + table = get_table_by_name("address_state") + + op.bulk_insert(table, seed_data) + + +def downgrade() -> None: + table = get_table_by_name("address_state") + + codes = get_indices_from_collection(seed_data, "code") + + op.execute(table.delete().where(table.c.code.in_(codes))) diff --git a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py deleted file mode 100644 index 058087f..0000000 --- a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Feed SBL Institution Type table - -Revision ID: a41281b1e109 -Revises: f4ff7d1aa6df -Create Date: 2023-12-14 01:24:00.120073 - -""" -from typing import Sequence, Union -from alembic import op -from db_revisions.utils import get_feed_data_from_file -from entities.models import SBLInstitutionTypeDao - -# revision identifiers, used by Alembic. -revision: str = "a41281b1e109" -down_revision: Union[str, None] = "f4ff7d1aa6df" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - data = get_feed_data_from_file("sbl_institution_type") - - op.bulk_insert(SBLInstitutionTypeDao.__table__, data) - - -def downgrade() -> None: - op.execute(SBLInstitutionTypeDao.__table__.delete()) diff --git a/db_revisions/versions/a41281b1e109_seed_sbl_institution_type_table.py b/db_revisions/versions/a41281b1e109_seed_sbl_institution_type_table.py new file mode 100644 index 0000000..2abb309 --- /dev/null +++ b/db_revisions/versions/a41281b1e109_seed_sbl_institution_type_table.py @@ -0,0 +1,46 @@ +"""Seed SBL Institution Type table + +Revision ID: a41281b1e109 +Revises: f4ff7d1aa6df +Create Date: 2023-12-14 01:24:00.120073 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_table_by_name, get_indices_from_collection + +# revision identifiers, used by Alembic. +revision: str = "a41281b1e109" +down_revision: Union[str, None] = "f4ff7d1aa6df" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +seed_data = [ + {"id": "1", "name": "Bank or savings association."}, + {"id": "2", "name": "Minority depository institution."}, + {"id": "3", "name": "Credit union."}, + {"id": "4", "name": "Nondepository institution."}, + {"id": "5", "name": "Community development financial institution (CDFI)."}, + {"id": "6", "name": "Other nonprofit financial institution."}, + {"id": "7", "name": "Farm Credit System institution."}, + {"id": "8", "name": "Government lender."}, + {"id": "9", "name": "Commercial finance company."}, + {"id": "10", "name": "Equipment finance company."}, + {"id": "11", "name": "Industrial loan company."}, + {"id": "12", "name": "Online lender."}, + {"id": "13", "name": "Other"}, +] + + +def upgrade() -> None: + table = get_table_by_name("sbl_institution_type") + + op.bulk_insert(table, seed_data) + + +def downgrade() -> None: + table = get_table_by_name("sbl_institution_type") + + ids = get_indices_from_collection(seed_data, "id") + + op.execute(table.delete().where(table.c.id.in_(ids))) diff --git a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py deleted file mode 100644 index 1502c9e..0000000 --- a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Feed HMDA Institution Type table - -Revision ID: f4ff7d1aa6df -Revises: 26a742d97ad9 -Create Date: 2023-12-14 01:23:47.017878 - -""" -from typing import Sequence, Union -from alembic import op -from db_revisions.utils import get_feed_data_from_file -from entities.models import HMDAInstitutionTypeDao - - -# revision identifiers, used by Alembic. -revision: str = "f4ff7d1aa6df" -down_revision: Union[str, None] = "26a742d97ad9" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - data = get_feed_data_from_file("hmda_institution_type") - - op.bulk_insert(HMDAInstitutionTypeDao.__table__, data) - - -def downgrade() -> None: - op.execute(HMDAInstitutionTypeDao.__table__.delete()) diff --git a/db_revisions/versions/f4ff7d1aa6df_seed_hmda_institution_type_table.py b/db_revisions/versions/f4ff7d1aa6df_seed_hmda_institution_type_table.py new file mode 100644 index 0000000..c49efad --- /dev/null +++ b/db_revisions/versions/f4ff7d1aa6df_seed_hmda_institution_type_table.py @@ -0,0 +1,52 @@ +"""Seed HMDA Institution Type table + +Revision ID: f4ff7d1aa6df +Revises: 26a742d97ad9 +Create Date: 2023-12-14 01:23:47.017878 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_table_by_name, get_indices_from_collection + + +# revision identifiers, used by Alembic. +revision: str = "f4ff7d1aa6df" +down_revision: Union[str, None] = "26a742d97ad9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +seed_data = [ + {"id": "1", "name": "National Bank (OCC supervised)"}, + {"id": "2", "name": "State Member Bank (FRS Supervised)"}, + {"id": "3", "name": "State non-member bank (FDIC supervised)"}, + {"id": "4", "name": "State Chartered Thrift (FDIC supervised)"}, + {"id": "5", "name": "Federal Chartered Thrift (OCC supervised)"}, + {"id": "6", "name": "Credit Union (NCUA supervised)"}, + {"id": "7", "name": "Federal Branch or Agency of Foreign Banking Organization (FBO)"}, + {"id": "8", "name": "Branch or Agency of FBO (FRS supervised)"}, + {"id": "9", "name": "MBS of national Bank (OCC supervised)"}, + {"id": "10", "name": "MBS of state member bank (FRS supervised)"}, + {"id": "11", "name": "MBS of state non-member bank (FDIC supervised)"}, + {"id": "12", "name": "MBS of Bank Holding Company (BHC) (FRS supervised)"}, + {"id": "13", "name": "MBS of credit union (NCUA supervised)"}, + {"id": "14", "name": "independent MBS, no depository affiliation"}, + {"id": "15", "name": "MBS of Savings and Loan Holding Co"}, + {"id": "16", "name": "MBS of state chartered Thrift"}, + {"id": "17", "name": "MBS of federally chartered thrift (OCC supervised)"}, + {"id": "18", "name": "Affiliate of depository institution. MBS is in the same ownership org as a depository."}, +] + + +def upgrade() -> None: + table = get_table_by_name("hmda_institution_type") + + op.bulk_insert(table, seed_data) + + +def downgrade() -> None: + table = get_table_by_name("hmda_institution_type") + + ids = get_indices_from_collection(seed_data, "id") + + op.execute(table.delete().where(table.c.id.in_(ids))) diff --git a/pyproject.toml b/pyproject.toml index cf6aeff..53de168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ pytest-alembic = "^0.10.7" [tool.pytest.ini_options] asyncio_mode = "auto" -pythonpath = ["src"] +pythonpath = ["src","db_revisions"] addopts = [ "--cov-report=term-missing", "--cov-branch", @@ -47,8 +47,11 @@ addopts = [ ] testpaths = ["tests"] env = [ - "INST_CONN=postgresql+asyncpg://localhost", "INST_DB_SCHEMA=main", + "INST_DB_USER=user", + "INST_DB_PWD=user", + "INST_DB_HOST=localhost:5432", + "INST_DB_NAME=fi", "KC_URL=http://localhost", "KC_REALM=", "KC_ADMIN_CLIENT_ID=", diff --git a/src/.env.local b/src/.env.local index ecc4a82..69a344b 100644 --- a/src/.env.local +++ b/src/.env.local @@ -13,7 +13,6 @@ INST_DB_USER=fi INST_DB_PWD=fi INST_DB_HOST=localhost:5432 INST_DB_SCHEMA=public -INST_CONN=postgresql+asyncpg://${INST_DB_USER}:${INST_DB_PWD}@${INST_DB_HOST}/${INST_DB_NAME} JWT_OPTS_VERIFY_AT_HASH="false" JWT_OPTS_VERIFY_AUD="false" JWT_OPTS_VERIFY_ISS="false" \ No newline at end of file diff --git a/src/.env.template b/src/.env.template index 7ccb51a..b5c39a5 100644 --- a/src/.env.template +++ b/src/.env.template @@ -13,4 +13,4 @@ INST_DB_USER= INST_DB_PWD= INST_DB_HOST= INST_DB_SCHEMA= -INST_CONN=postgresql+asyncpg://${INST_DB_USER}:${INST_DB_PWD}@${INST_DB_HOST}/${INST_DB_NAME} \ No newline at end of file +# INST_DB_SCHEME This can be added if needing to override the default of 'postgresql+asyncpg' \ No newline at end of file diff --git a/src/config.py b/src/config.py index d8a3f4e..f11e602 100644 --- a/src/config.py +++ b/src/config.py @@ -1,11 +1,13 @@ import os +from urllib import parse from typing import Dict, Any -from pydantic import TypeAdapter +from pydantic import TypeAdapter, field_validator, ValidationInfo from pydantic.networks import HttpUrl, PostgresDsn from pydantic.types import SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict + JWT_OPTS_PREFIX = "jwt_opts_" env_files_to_load = [".env"] @@ -14,8 +16,13 @@ class Settings(BaseSettings): - inst_conn: PostgresDsn inst_db_schema: str = "public" + inst_db_name: str + inst_db_user: str + inst_db_pwd: str + inst_db_host: str + inst_db_scheme: str = "postgresql+asyncpg" + inst_conn: PostgresDsn | None = None auth_client: str auth_url: HttpUrl token_url: HttpUrl @@ -31,6 +38,18 @@ def __init__(self, **data): super().__init__(**data) self.set_jwt_opts() + @field_validator("inst_conn", mode="before") + @classmethod + def build_postgres_dsn(cls, postgres_dsn, info: ValidationInfo) -> Any: + postgres_dsn = PostgresDsn.build( + scheme=info.data.get("inst_db_scheme"), + username=info.data.get("inst_db_user"), + password=parse.quote(info.data.get("inst_db_pwd"), safe=""), + host=info.data.get("inst_db_host"), + path=info.data.get("inst_db_name"), + ) + return str(postgres_dsn) + def set_jwt_opts(self) -> None: """ Converts `jwt_opts_` prefixed settings, and env vars into JWT options dictionary. diff --git a/src/entities/models/__init__.py b/src/entities/models/__init__.py index e43e50e..0394759 100644 --- a/src/entities/models/__init__.py +++ b/src/entities/models/__init__.py @@ -16,9 +16,7 @@ "SBLInstitutionTypeDao", "AddressStateDao", "FederalRegulatorDto", - # "HMDAInstitutionTypeDto", "InstitutionTypeDto", - # "SBLInstitutionTypeDto", "AddressStateDto", ] @@ -42,8 +40,6 @@ UserProfile, AuthenticatedUser, FederalRegulatorDto, - # HMDAInstitutionTypeDto, InstitutionTypeDto, - # SBLInstitutionTypeDto, AddressStateDto, ) diff --git a/src/entities/models/dao.py b/src/entities/models/dao.py index 4c41b23..3cf3bb7 100644 --- a/src/entities/models/dao.py +++ b/src/entities/models/dao.py @@ -73,16 +73,17 @@ class FederalRegulatorDao(AuditMixin, Base): name: Mapped[str] = mapped_column(unique=True, nullable=False) -class HMDAInstitutionTypeDao(AuditMixin, Base): - __tablename__ = "hmda_institution_type" +class InstitutionTypeMixin(AuditMixin): id: Mapped[str] = mapped_column(index=True, primary_key=True, unique=True) name: Mapped[str] = mapped_column(unique=True) -class SBLInstitutionTypeDao(AuditMixin, Base): +class HMDAInstitutionTypeDao(InstitutionTypeMixin, Base): + __tablename__ = "hmda_institution_type" + + +class SBLInstitutionTypeDao(InstitutionTypeMixin, Base): __tablename__ = "sbl_institution_type" - id: Mapped[str] = mapped_column(index=True, primary_key=True, unique=True) - name: Mapped[str] = mapped_column(unique=True, nullable=False) class AddressStateDao(AuditMixin, Base): diff --git a/src/entities/models/dto.py b/src/entities/models/dto.py index dad9d31..315b6b8 100644 --- a/src/entities/models/dto.py +++ b/src/entities/models/dto.py @@ -81,14 +81,6 @@ class Config: from_attributes = True -class HMDAInstitutionTypeDto(InstitutionTypeDto): - pass - - -class SBLInstitutionTypeDto(InstitutionTypeDto): - pass - - class AddressStateBase(BaseModel): code: str @@ -102,8 +94,8 @@ class Config: class FinancialInstitutionWithRelationsDto(FinancialInstitutionDto): primary_federal_regulator: FederalRegulatorDto | None = None - hmda_institution_type: HMDAInstitutionTypeDto | None = None - sbl_institution_type: SBLInstitutionTypeDto | None = None + hmda_institution_type: InstitutionTypeDto | None = None + sbl_institution_type: InstitutionTypeDto | None = None hq_address_state: AddressStateDto domains: List[FinancialInsitutionDomainDto] = [] diff --git a/src/entities/repos/institutions_repo.py b/src/entities/repos/institutions_repo.py index d65e897..fb891cb 100644 --- a/src/entities/repos/institutions_repo.py +++ b/src/entities/repos/institutions_repo.py @@ -4,6 +4,8 @@ from sqlalchemy.orm import joinedload from sqlalchemy.ext.asyncio import AsyncSession +from .repo_utils import query_type + from entities.models import ( FinancialInstitutionDao, FinancialInstitutionDomainDao, @@ -49,32 +51,20 @@ async def get_institution(session: AsyncSession, lei: str) -> FinancialInstituti return await session.scalar(stmt) -async def get_sbl_types(session: AsyncSession) -> SBLInstitutionTypeDao: - async with session.begin(): - stmt = select(SBLInstitutionTypeDao) - res = await session.scalars(stmt) - return res.all() +async def get_sbl_types(session: AsyncSession) -> List[SBLInstitutionTypeDao]: + return await query_type(session, SBLInstitutionTypeDao) -async def get_hmda_types(session: AsyncSession) -> HMDAInstitutionTypeDao: - async with session.begin(): - stmt = select(HMDAInstitutionTypeDao) - res = await session.scalars(stmt) - return res.all() +async def get_hmda_types(session: AsyncSession) -> List[HMDAInstitutionTypeDao]: + return await query_type(session, HMDAInstitutionTypeDao) -async def get_address_states(session: AsyncSession) -> AddressStateDao: - async with session.begin(): - stmt = select(AddressStateDao) - res = await session.scalars(stmt) - return res.all() +async def get_address_states(session: AsyncSession) -> List[AddressStateDao]: + return await query_type(session, AddressStateDao) -async def get_federal_regulators(session: AsyncSession) -> FederalRegulatorDao: - async with session.begin(): - stmt = select(FederalRegulatorDao) - res = await session.scalars(stmt) - return res.all() +async def get_federal_regulators(session: AsyncSession) -> List[FederalRegulatorDao]: + return await query_type(session, FederalRegulatorDao) async def upsert_institution(session: AsyncSession, fi: FinancialInstitutionDto) -> FinancialInstitutionDao: diff --git a/src/entities/repos/repo_utils.py b/src/entities/repos/repo_utils.py new file mode 100644 index 0000000..faa48be --- /dev/null +++ b/src/entities/repos/repo_utils.py @@ -0,0 +1,12 @@ +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from typing import List, TypeVar + +T = TypeVar("T") + + +async def query_type(session: AsyncSession, type: T) -> List[T]: + async with session.begin(): + stmt = select(type) + res = await session.scalars(stmt) + return res.all() diff --git a/src/routers/institutions.py b/src/routers/institutions.py index b207217..63961ba 100644 --- a/src/routers/institutions.py +++ b/src/routers/institutions.py @@ -12,9 +12,7 @@ FinancialInsitutionDomainDto, FinancialInsitutionDomainCreate, FinanicialInstitutionAssociationDto, - # HMDAInstitutionTypeDto, InstitutionTypeDto, - # SBLInstitutionTypeDto, AuthenticatedUser, AddressStateDto, FederalRegulatorDto, @@ -73,10 +71,11 @@ async def get_associated_institutions(request: Request): @router.get("/types/{type}", response_model=List[InstitutionTypeDto]) @requires("authenticated") async def get_institution_types(request: Request, type: InstitutionType): - if type == "sbl": - return await repo.get_sbl_types(request.state.db_session) - else: - return await repo.get_hmda_types(request.state.db_session) + match type: + case "sbl": + return await repo.get_sbl_types(request.state.db_session) + case "hmda": + return await repo.get_hmda_types(request.state.db_session) @router.get("/address-states", response_model=List[AddressStateDto]) diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 9336bc1..eeea71c 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -2,6 +2,18 @@ from config import Settings +def test_postgres_dsn_building(): + mock_config = { + "inst_db_name": "test", + "inst_db_user": "user", + "inst_db_pwd": "\\z9-/tgb76#@", + "inst_db_host": "test:5432", + "inst_db_scehma": "test", + } + settings = Settings(**mock_config) + assert str(settings.inst_conn) == "postgresql+asyncpg://user:%5Cz9-%2Ftgb76%23%40@test:5432/test" + + def test_jwt_opts_valid_values(): mock_config = { "jwt_opts_test1": "true", diff --git a/tests/migrations/test_lookup_tables_data_feed.py b/tests/migrations/test_lookup_tables_data_feed.py deleted file mode 100644 index ed74161..0000000 --- a/tests/migrations/test_lookup_tables_data_feed.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -import csv -from sqlalchemy import text -from sqlalchemy.engine import Engine -from pytest_alembic import MigrationContext - - -def data_feed_helper(table_name): - file_dir = os.path.dirname(os.path.realpath(__file__)) - data_file_path = f"{file_dir}/../../db_revisions/feed/%s.csv" % table_name - with open(data_file_path) as f: - reader = csv.reader(f, delimiter="|") - next(reader) - output_list = list(tuple(line) for line in reader) - return output_list - - -def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("7b6ff51002b5") - - # Test address_state feed - address_state_tablename = "address_state" - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() - address_state_expected = data_feed_helper(address_state_tablename) - assert address_state_rows == address_state_expected - - -def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("26a742d97ad9") - - # Test federal_regulator feed - federal_regulator_tablename = "federal_regulator" - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - federal_regulator_rows = conn.execute(text("SELECT id, name from %s" % federal_regulator_tablename)).fetchall() - federal_regulator_expected = data_feed_helper(federal_regulator_tablename) - assert federal_regulator_rows == federal_regulator_expected - - -def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("f4ff7d1aa6df") - - # Test hmda_institution_type feed - hmda_institution_type_tablename = "hmda_institution_type" - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - hmda_institution_type_rows = conn.execute( - text("SELECT id, name from %s" % hmda_institution_type_tablename) - ).fetchall() - hmda_institution_type_expected = data_feed_helper(hmda_institution_type_tablename) - assert hmda_institution_type_rows == hmda_institution_type_expected - - -def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("a41281b1e109") - - # Test sbl_institution_type feed - sbl_institution_type_tablename = "sbl_institution_type" - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - sbl_institution_type_rows = conn.execute( - text("SELECT id, name from %s" % sbl_institution_type_tablename) - ).fetchall() - sbl_institution_type_expected = data_feed_helper(sbl_institution_type_tablename) - assert sbl_institution_type_rows == sbl_institution_type_expected diff --git a/tests/migrations/test_lookup_tables_data_seed.py b/tests/migrations/test_lookup_tables_data_seed.py new file mode 100644 index 0000000..335c40b --- /dev/null +++ b/tests/migrations/test_lookup_tables_data_seed.py @@ -0,0 +1,109 @@ +import pytest +from sqlalchemy import text +from sqlalchemy.engine import Engine +from pytest_alembic import MigrationContext + + +@pytest.fixture +def alembic_config(): + return { + "at_revision_data": { + "7b6ff51002b5": {"__tablename__": "address_state", "code": "ZZ", "name": "TestState"}, + "26a742d97ad9": {"__tablename__": "federal_regulator", "id": "ZZZ", "name": "TestRegulator"}, + "f4ff7d1aa6df": {"__tablename__": "hmda_institution_type", "id": "00", "name": "TestHmdaInstitutionType"}, + "a41281b1e109": {"__tablename__": "sbl_institution_type", "id": "00", "name": "TestSblInstitutionType"}, + } + } + + +def test_address_state_data_seed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("7b6ff51002b5") + + # Test address_state seed + address_state_tablename = "address_state" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + address_state_rows = conn.execute( + text("SELECT code, name from %s where code = :code" % address_state_tablename), (dict(code="AL")) + ).fetchall() + address_state_expected = [("AL", "Alabama")] + assert address_state_rows == address_state_expected + + alembic_runner.migrate_down_one() + with alembic_engine.connect() as conn: + address_state_before_seed = conn.execute(text("SELECT code, name FROM %s" % address_state_tablename)).fetchall() + assert address_state_before_seed == [("ZZ", "TestState")] + + +def test_federal_regulator_data_seed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("26a742d97ad9") + + # Test federal_regulator seed + federal_regulator_tablename = "federal_regulator" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + federal_regulator_rows = conn.execute( + text("SELECT id, name from %s where id = :id" % federal_regulator_tablename), (dict(id="FCA")) + ).fetchall() + federal_regulator_expected = [ + ("FCA", "Farm Credit Administration"), + ] + + assert federal_regulator_rows == federal_regulator_expected + + alembic_runner.migrate_down_one() + with alembic_engine.connect() as conn: + federal_regulator_before_seed = conn.execute( + text("SELECT id, name FROM %s" % federal_regulator_tablename) + ).fetchall() + assert federal_regulator_before_seed == [("ZZZ", "TestRegulator")] + + +def test_hmda_institution_type_data_seed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("f4ff7d1aa6df") + + # Test hmda_institution_type seed + hmda_institution_type_tablename = "hmda_institution_type" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + hmda_institution_type_rows = conn.execute( + # text("SELECT id, name from %s" % hmda_institution_type_tablename) + text("SELECT id, name from %s where id = :id" % hmda_institution_type_tablename), + (dict(id="1")), + ).fetchall() + hmda_institution_type_expected = [("1", "National Bank (OCC supervised)")] + + assert hmda_institution_type_rows == hmda_institution_type_expected + + alembic_runner.migrate_down_one() + with alembic_engine.connect() as conn: + hmda_institution_type_before_seed = conn.execute( + text("SELECT id, name FROM %s" % hmda_institution_type_tablename) + ).fetchall() + assert hmda_institution_type_before_seed == [("00", "TestHmdaInstitutionType")] + + +def test_sbl_institution_type_data_seed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("a41281b1e109") + + # Test sbl_institution_type seed + sbl_institution_type_tablename = "sbl_institution_type" + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + sbl_institution_type_rows = conn.execute( + text("SELECT id, name from %s where id = :id " % sbl_institution_type_tablename), (dict(id="1")) + ).fetchall() + sbl_institution_type_expected = [("1", "Bank or savings association.")] + + assert sbl_institution_type_rows == sbl_institution_type_expected + + alembic_runner.migrate_down_one() + with alembic_engine.connect() as conn: + sbl_institution_type_before_seed = conn.execute( + text("SELECT id, name FROM %s" % sbl_institution_type_tablename) + ).fetchall() + assert sbl_institution_type_before_seed == [("00", "TestSblInstitutionType")] diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py index 2a27197..5059002 100644 --- a/tests/migrations/test_migrations.py +++ b/tests/migrations/test_migrations.py @@ -10,15 +10,32 @@ from pytest_alembic import MigrationContext -def test_tables_exist_after_migration(alembic_runner: MigrationContext, alembic_engine: Engine): +def test_tables_exist_migrate_up_to_045aa502e050(alembic_runner: MigrationContext, alembic_engine: Engine): alembic_runner.migrate_up_to("045aa502e050") inspector = sqlalchemy.inspect(alembic_engine) tables = inspector.get_table_names() - assert "denied_domains" in tables - assert "financial_institutions" in tables - assert "financial_institution_domains" in tables assert "address_state" in tables assert "federal_regulator" in tables assert "hmda_institution_type" in tables assert "sbl_institution_type" in tables + + +def test_tables_exist_migrate_up_to_20e0d51d8be9(alembic_runner: MigrationContext, alembic_engine: Engine): + alembic_runner.migrate_up_to("20e0d51d8be9") + + inspector = sqlalchemy.inspect(alembic_engine) + tables = inspector.get_table_names() + assert "denied_domains" in tables + assert "financial_institutions" in tables + assert "financial_institution_domains" in tables + + +def test_tables_not_exist_migrate_down_to_base(alembic_runner: MigrationContext, alembic_engine: Engine): + alembic_runner.migrate_down_to("base") + + inspector = sqlalchemy.inspect(alembic_engine) + tables = inspector.get_table_names() + assert "denied_domains" not in tables + assert "financial_institutions" not in tables + assert "financial_institution_domains" not in tables diff --git a/tests/migrations/test_schema.py b/tests/migrations/test_schema.py new file mode 100644 index 0000000..09b6ac4 --- /dev/null +++ b/tests/migrations/test_schema.py @@ -0,0 +1,56 @@ +import sqlalchemy +from sqlalchemy.engine import Engine + +from pytest_alembic import MigrationContext + + +def test_financial_institutions_schema_migrate_up_to_045aa502e050( + alembic_runner: MigrationContext, alembic_engine: Engine +): + alembic_runner.migrate_up_to("045aa502e050") + + inspector = sqlalchemy.inspect(alembic_engine) + expexted_columns = [ + "lei", + "name", + "event_time", + "tax_id", + "rssd_id", + "primary_federal_regulator_id", + "hmda_institution_type_id", + "sbl_institution_type_id", + "hq_address_street_1", + "hq_address_street_2", + "hq_address_city", + "hq_address_state_code", + "hq_address_zip", + "parent_lei", + "parent_legal_name", + "parent_rssd_id", + "top_holder_lei", + "top_holder_legal_name", + "top_holder_rssd_id", + ] + + columns = inspector.get_columns("financial_institutions") + columns_names = [column.get("name") for column in columns] + + assert columns_names == expexted_columns + + +def test_financial_institutions_schema_migrate_up_to_20e0d51d8be9( + alembic_runner: MigrationContext, alembic_engine: Engine +): + alembic_runner.migrate_up_to("20e0d51d8be9") + + inspector = sqlalchemy.inspect(alembic_engine) + expexted_columns = [ + "lei", + "name", + "event_time", + ] + + columns = inspector.get_columns("financial_institutions") + columns_names = [column.get("name") for column in columns] + + assert columns_names == expexted_columns