From 540b3df697ba1c3b02e8627a6ff9cd1a594df49f Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Wed, 29 Nov 2023 12:30:25 -0500 Subject: [PATCH] Ticket 48: Update Dao and Create alembic script for institutions table update --- ...050_update_financial_institutions_table.py | 89 +++++++++++++++++++ ...1f6c33f20a2e_create_address_state_table.py | 37 ++++++++ ...612bf1c9_create_federal_regulator_table.py | 40 +++++++++ ...cd2d4_create_sbl_institution_type_table.py | 42 +++++++++ ...275b_create_hmda_institution_type_table.py | 41 +++++++++ src/entities/models/__init__.py | 16 ++++ src/entities/models/dao.py | 4 +- src/entities/models/dto.py | 42 +++++++-- src/entities/repos/institutions_repo.py | 21 ++++- 9 files changed, 324 insertions(+), 8 deletions(-) create mode 100644 db_revisions/versions/045aa502e050_update_financial_institutions_table.py create mode 100644 db_revisions/versions/1f6c33f20a2e_create_address_state_table.py create mode 100644 db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py create mode 100644 db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py create mode 100644 db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py diff --git a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py new file mode 100644 index 0000000..9f2aca2 --- /dev/null +++ b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py @@ -0,0 +1,89 @@ +"""Update Financial Institutions Table + +Revision ID: 045aa502e050 +Revises: 20e0d51d8be9 +Create Date: 2023-11-29 11:55:10.328766 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "045aa502e050" +down_revision: Union[str, None] = "20e0d51d8be9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on = ["1f6c33f20a2e", "8b1ba6a3275b", "56ef0b5cd2d4", "549c612bf1c9"] + + +def upgrade() -> None: + op.add_column("financial_institutions", sa.Column("tax_id", sa.String(length=9), nullable=False)) + op.add_column("financial_institutions", sa.Column("rssd_id", sa.Integer(), nullable=False)) + op.add_column( + "financial_institutions", sa.Column("primary_federal_regulator_id", sa.String(length=4), nullable=False) + ) + op.add_column("financial_institutions", sa.Column("hmda_institution_type_id", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("sbl_institution_type_id", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("hq_address_street_1", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("hq_address_street_2", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("hq_address_city", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("hq_address_state", sa.String(length=2), nullable=False)) + op.add_column("financial_institutions", sa.Column("hq_address_zip", sa.String(length=5), nullable=False)) + op.add_column("financial_institutions", sa.Column("parent_lei", sa.String(length=20), nullable=False)) + op.add_column("financial_institutions", sa.Column("parent_legal_name", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("parent_rssd_id", sa.Integer(), nullable=False)) + op.add_column("financial_institutions", sa.Column("top_holder_lei", sa.String(length=20), nullable=False)) + op.add_column("financial_institutions", sa.Column("top_holder_legal_name", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("top_holder_rssd_id", sa.Integer(), nullable=False)) + op.create_index( + op.f("ix_financial_institutions_hmda_institution_type_id"), + "financial_institutions", + ["hmda_institution_type_id"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_hq_address_state"), "financial_institutions", ["hq_address_state"], unique=False + ) + op.create_index( + op.f("ix_financial_institutions_primary_federal_regulator_id"), + "financial_institutions", + ["primary_federal_regulator_id"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_sbl_institution_type_id"), + "financial_institutions", + ["sbl_institution_type_id"], + unique=False, + ) + op.create_unique_constraint(None, "financial_institutions", ["rssd_id"]) + op.create_unique_constraint(None, "financial_institutions", ["tax_id"]) + op.create_foreign_key(None, "financial_institutions", "federal_regulator", ["primary_federal_regulator_id"], ["id"]) + op.create_foreign_key(None, "financial_institutions", "address_state", ["hq_address_state"], ["code"]) + op.create_foreign_key(None, "financial_institutions", "hmda_institution_type", ["hmda_institution_type_id"], ["id"]) + op.create_foreign_key(None, "financial_institutions", "sbl_institution_type", ["sbl_institution_type_id"], ["id"]) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.drop_column("financial_institutions", "top_holder_rssd_id") + op.drop_column("financial_institutions", "top_holder_legal_name") + op.drop_column("financial_institutions", "top_holder_lei") + op.drop_column("financial_institutions", "parent_rssd_id") + op.drop_column("financial_institutions", "parent_legal_name") + op.drop_column("financial_institutions", "parent_lei") + op.drop_column("financial_institutions", "hq_address_zip") + op.drop_column("financial_institutions", "hq_address_state") + op.drop_column("financial_institutions", "hq_address_city") + op.drop_column("financial_institutions", "hq_address_street_2") + op.drop_column("financial_institutions", "hq_address_street_1") + op.drop_column("financial_institutions", "sbl_institution_type_id") + op.drop_column("financial_institutions", "hmda_institution_type_id") + op.drop_column("financial_institutions", "primary_federal_regulator_id") + op.drop_column("financial_institutions", "rssd_id") + op.drop_column("financial_institutions", "tax_id") + # ### end Alembic commands ### diff --git a/db_revisions/versions/1f6c33f20a2e_create_address_state_table.py b/db_revisions/versions/1f6c33f20a2e_create_address_state_table.py new file mode 100644 index 0000000..621e667 --- /dev/null +++ b/db_revisions/versions/1f6c33f20a2e_create_address_state_table.py @@ -0,0 +1,37 @@ +"""Create Address State Table + +Revision ID: 1f6c33f20a2e +Revises: 20e0d51d8be9 +Create Date: 2023-11-29 12:03:41.737864 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "1f6c33f20a2e" +down_revision: Union[str, None] = "20e0d51d8be9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "address_state", + sa.Column("code", sa.String(length=2), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("code"), + sa.UniqueConstraint("name"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("address_state") + # ### end Alembic commands ### diff --git a/db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py b/db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py new file mode 100644 index 0000000..886f588 --- /dev/null +++ b/db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py @@ -0,0 +1,40 @@ +"""Create Federal Regulator Table + +Revision ID: 549c612bf1c9 +Revises: 20e0d51d8be9 +Create Date: 2023-11-29 12:09:20.012400 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "549c612bf1c9" +down_revision: Union[str, None] = "20e0d51d8be9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.create_table( + "federal_regulator", + sa.Column("id", sa.String(length=4), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("federal_regulator") + + # ### end Alembic commands ### diff --git a/db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py b/db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py new file mode 100644 index 0000000..3830e00 --- /dev/null +++ b/db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py @@ -0,0 +1,42 @@ +"""Create SBL Institution Type Table + +Revision ID: 56ef0b5cd2d4 +Revises: 20e0d51d8be9 +Create Date: 2023-11-29 12:20:05.593826 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "56ef0b5cd2d4" +down_revision: Union[str, None] = "20e0d51d8be9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.create_table( + "sbl_institution_type", + sa.Column("id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + ) + op.create_index(op.f("ix_sbl_institution_type_id"), "sbl_institution_type", ["id"], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.drop_table("sbl_institution_type") + + # ### end Alembic commands ### diff --git a/db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py b/db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py new file mode 100644 index 0000000..efb0ced --- /dev/null +++ b/db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py @@ -0,0 +1,41 @@ +"""Create HMDA Institution Type Table + +Revision ID: 8b1ba6a3275b +Revises: 20e0d51d8be9 +Create Date: 2023-11-29 12:14:16.694281 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "8b1ba6a3275b" +down_revision: Union[str, None] = "20e0d51d8be9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.create_table( + "hmda_institution_type", + sa.Column("id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + ) + op.create_index(op.f("ix_hmda_institution_type_id"), "hmda_institution_type", ["id"], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("hmda_institution_type") + + # ### end Alembic commands ### diff --git a/src/entities/models/__init__.py b/src/entities/models/__init__.py index 3bd37af..f4b5b33 100644 --- a/src/entities/models/__init__.py +++ b/src/entities/models/__init__.py @@ -11,6 +11,14 @@ "DeniedDomainDto", "UserProfile", "AuthenticatedUser", + "FederalRegulatorDao", + "FederalRegulatorDto", + "HMDAInstitutionTypeDao", + "HMDAInstitutionTypeDto", + "SBLInstitutionTypeDao", + "SBLInstitutionTypeDto", + "AddressStateDao", + "AddressStateDto", ] from .dao import ( @@ -18,6 +26,10 @@ FinancialInstitutionDao, FinancialInstitutionDomainDao, DeniedDomainDao, + FederalRegulatorDao, + HMDAInstitutionTypeDao, + SBLInstitutionTypeDao, + AddressStateDao, ) from .dto import ( FinancialInstitutionDto, @@ -28,4 +40,8 @@ DeniedDomainDto, UserProfile, AuthenticatedUser, + FederalRegulatorDto, + HMDAInstitutionTypeDto, + SBLInstitutionTypeDto, + AddressStateDto, ) diff --git a/src/entities/models/dao.py b/src/entities/models/dao.py index 8926118..8e8abaa 100644 --- a/src/entities/models/dao.py +++ b/src/entities/models/dao.py @@ -17,11 +17,11 @@ class AuditMixin(object): class FinancialInstitutionDao(AuditMixin, Base): __tablename__ = "financial_institutions" lei: Mapped[str] = mapped_column(unique=True, index=True, primary_key=True) - legal_name: Mapped[str] = mapped_column(index=True) + name: Mapped[str] = mapped_column(index=True) domains: Mapped[List["FinancialInstitutionDomainDao"]] = relationship( "FinancialInstitutionDomainDao", back_populates="fi" ) - tax_id: Mapped[str] = mapped_column(unique=True) + tax_id: Mapped[str] = mapped_column(String(9), unique=True) rssd_id: Mapped[int] = mapped_column(unique=True) primary_federal_regulator_id: Mapped[str] = mapped_column( ForeignKey("federal_regulator.id"), index=True, primary_key=True diff --git a/src/entities/models/dto.py b/src/entities/models/dto.py index a7ae988..1015126 100644 --- a/src/entities/models/dto.py +++ b/src/entities/models/dto.py @@ -19,11 +19,27 @@ class Config: class FinancialInstitutionBase(BaseModel): - legal_name: str + name: str class FinancialInstitutionDto(FinancialInstitutionBase): lei: str + tax_id: str + rssd_id: int + primary_federal_regulator_id: str + hmda_institution_type_id: str + sbl_institution_type_id: str + hq_address_street_1: str + hq_address_street_2: str + hq_address_city: str + hq_address_state: str + hq_address_zip: str + parent_lei: str + parent_legal_name: str + parent_rssd_id: int + top_holder_lei: str + top_holder_legal_name: str + top_holder_rssd_id: int class Config: from_attributes = True @@ -40,30 +56,46 @@ class Config: from_attributes = True -class FederalRegulatorDto(BaseModel): +class FederalRegulatorBase(BaseModel): id: str + +class FederalRegulatorDto(FederalRegulatorBase): + name: str + class Config: from_attributes = True -class HMDAInstitutionTypeDto(BaseModel): +class HMDAInstitutionTypeBase(BaseModel): id: str + +class HMDAInstitutionTypeDto(HMDAInstitutionTypeBase): + name: str + class Config: from_attributes = True -class SBLInstitutionTypeDto(BaseModel): +class SBLInstitutionTypeBase(BaseModel): id: str + +class SBLInstitutionTypeDto(SBLInstitutionTypeBase): + name: str + class Config: from_attributes = True -class AddressStateDto(BaseModel): +class AddressStateBase(BaseModel): code: str + +class AddressStateDto(AddressStateBase): + name: str + class Config: from_attributes = True diff --git a/src/entities/repos/institutions_repo.py b/src/entities/repos/institutions_repo.py index 8605ce6..6455e2d 100644 --- a/src/entities/repos/institutions_repo.py +++ b/src/entities/repos/institutions_repo.py @@ -55,7 +55,26 @@ async def upsert_institution(session: AsyncSession, fi: FinancialInstitutionDto) res = await session.execute(stmt) db_fi = res.scalar_one_or_none() if db_fi is None: - db_fi = FinancialInstitutionDao(lei=fi.lei, name=fi.name) + db_fi = FinancialInstitutionDao( + lei=fi.lei, + name=fi.name, + tax_id=fi.tax_id, + rssd_id=fi.rssd_id, + primary_federal_regulator_id=fi.primary_federal_regulator_id, + hmda_institution_type_id=fi.hmda_institution_type_id, + sbl_institution_type_id=fi.sbl_institution_type_id, + hq_address_street_1=fi.hq_address_street_1, + hq_address_street_2=fi.hq_address_street_2, + hq_address_city=fi.hq_address_city, + hq_address_state=fi.hq_address_state, + hq_address_zip=fi.hq_address_zip, + parent_lei=fi.parent_lei, + parent_legal_name=fi.parent_legal_name, + parent_rssd_id=fi.parent_rssd_id, + top_holder_lei=fi.top_holder_lei, + top_holder_legal_name=fi.top_holder_legal_name, + top_holder_rssd_id=fi.top_holder_rssd_id, + ) session.add(db_fi) else: db_fi.name = fi.name