Add Template_Categories table (#2193)

* Draft migration to add TemplateCategories table

- Initial work on the models

* Fix prop logic for template_process_type

* Add indexes, unique constraints

- Fix typos
- Update sqlalchemy model

* Add CRUD methods for TemplateCategories

- Added template_category_id to the template_history table
- Adjusted model class name TemplateCategories -> TemplateCategory
- WIP: Added some basic tests for the TemplateCategory dao

* Insert low, med, high default categories during migration

* Fix prop logic for template_process_type again

* WIP: Add API endpoints for interacting with TemplateCategories

- Added a schema for TemplateCategory
- Added from_json and serialize to the TemplateCategory model
- Added the template/category Blueprint
- Add some initial CRUD api endpoints

* Implement dao and api to update process_type

* Address PR comments

- Changed the category relationship in Template from TemplateCategory to template_category
- dao_get_template_category_by_template_id now simply selects the template, and returns the associated template_category instead of using an expensive join

* Finish adding needed api endpoints

- Moved the default categories into our config file
- Added dao_update_template_category
- Added dao_delete_template_category_by_id
- Added routes to: get and delete a template category
- Added a route to get all template categories
- Updated the migration file to use default category uuid's from the config file

* Chore: logic cleanup

* First batch of unit tests & bug fixes

* Implement filtering when fetching all categories

- Add tests

* Add lazy join on TemplateCategory

* Clean up dao tests

- Squash more bugs
- Fix formatting
- Added a couple more tests for the filters on dao_get_all_template_categories

* Add tests for deleting a template category

- Excluded the template_category table from deletion in notify_db_session to preserve the 3 generic template categories between test runs
- Fixed inserts in the migration, apparently alembic / sqlalchemy doesn't like multi-line f-strings
- Made a few tests shorter by excluding the description_en and description_fr columns as they are optional

* Add API tests, squash bugs

- Allow passing of a uuid to dao_create_template_category
- Fixed issues with get_template_categories and delete_template_category filters / flags
- Added a fixture to re-populate the template_category table with generic categories and removed template_categories from the list of tables that are excluded from the post-test db clear

* Fix pre-existing tests

- formatting

* Misc. fixes

- Fix incorrectly named prop call in dao
- Added category as a declared attribute in TemplateBase model
- Added a proper route to get a category by template id
- Added a foreign key to templates_history for the category id
- Fix a couple more tests

* We definitely didn't want that FK on templatehistory...

* Logic cleanups

- Tweak tests
- No longer excluding template_category_id in template_category_schema
- Updated migration revision as 0453 was added for pinpoint work

* Rename migration

- Remove NOT NULL constraint on templates.process_type

* Add tests for models

- Simplify the dao delete logic

* Add tests that were missed for template rest and dao

* Rename /template/category to /template-category

- Adjust tests
- formatting

* various fixes

* Re-word dao_delete_template_category_by_id

Clarify that new category assignments aren't done at random but are chosen from one of the default categories.

* Add created_at and updated_at fields

- Set a default category id if a category is not passed to create_sample_template

* Add default value for created at, fix tests

* Quick fix

* Fix column defaults

- Create a template category when creating a sample template and do not pass in a category as we cannot rely on the defaults existing in the test db

* Fix silly typo..

* Remove unneeded assert

---------

Co-authored-by: Jumana B <[email protected]>

app/__init__.py

@@ -201,6 +201,7 @@ def register_blueprint(application):
     from app.service.rest import service_blueprint
     from app.status.healthcheck import status as status_blueprint
     from app.template.rest import template_blueprint
+    from app.template.template_category_rest import template_category_blueprint
     from app.template_folder.rest import template_folder_blueprint
     from app.template_statistics.rest import (
         template_statistics as template_statistics_blueprint,
@@ -259,6 +260,8 @@ def register_blueprint(application):
 
     register_notify_blueprint(application, letter_branding_blueprint, requires_admin_auth)
 
+    register_notify_blueprint(application, template_category_blueprint, requires_admin_auth)
+
 
 def register_v2_blueprints(application):
     from app.authentication.auth import requires_auth

app/config.py

@@ -340,6 +340,9 @@ class Config(object):
     HEARTBEAT_TEMPLATE_SMS_LOW = "ab3a603b-d602-46ea-8c83-e05cb280b950"
     HEARTBEAT_TEMPLATE_SMS_MEDIUM = "a48b54ce-40f6-4e4a-abe8-1e2fa389455b"
     HEARTBEAT_TEMPLATE_SMS_HIGH = "4969a9e9-ddfd-476e-8b93-6231e6f1be4a"
+    DEFAULT_TEMPLATE_CATEGORY_LOW = "0dda24c2-982a-4f44-9749-0e38b2607e89"
+    DEFAULT_TEMPLATE_CATEGORY_MEDIUM = "f75d6706-21b7-437e-b93a-2c0ab771e28e"
+    DEFAULT_TEMPLATE_CATEGORY_HIGH = "c4f87d7c-a55b-4c0f-91fe-e56c65bb1871"
 
     # Allowed service IDs able to send HTML through their templates.
     ALLOW_HTML_SERVICE_IDS: List[str] = [id.strip() for id in os.getenv("ALLOW_HTML_SERVICE_IDS", "").split(",")]

app/dao/template_categories_dao.py

@@ -0,0 +1,86 @@
+import uuid
+from datetime import datetime
+
+from flask import current_app
+
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import Template, TemplateCategory
+
+
+@transactional
+def dao_create_template_category(template_category: TemplateCategory):
+    if template_category.id is None:
+        template_category.id = uuid.uuid4()
+    db.session.add(template_category)
+
+
+def dao_get_template_category_by_id(template_category_id) -> TemplateCategory:
+    return TemplateCategory.query.filter_by(id=template_category_id).one()
+
+
+def dao_get_template_category_by_template_id(template_id) -> TemplateCategory:
+    return Template.query.filter_by(id=template_id).one().category
+
+
+# TODO: Add filters: Select all template categories used by at least 1 sms/email template
+def dao_get_all_template_categories(template_type=None, hidden=None):
+    query = TemplateCategory.query
+
+    if template_type is not None:
+        query = query.join(Template).filter(Template.template_type == template_type)
+
+    if hidden is not None:
+        query = query.filter(TemplateCategory.hidden == hidden)
+
+    return query.all()
+
+
+@transactional
+def dao_update_template_category(template_category: TemplateCategory):
+    db.session.add(template_category)
+    db.session.commit()
+
+
+@transactional
+def dao_delete_template_category_by_id(template_category_id, cascade=False):
+    """
+    Deletes a `TemplateCategory`. By default, if the `TemplateCategory` is associated with any `Template`, it will not be deleted.
+    If the `cascade` option is specified then the category will be forcible removed:
+    1. The `Category` will be dissociated from templates that use it
+    2. The `Template` is assigned to one of the default categories that matches the priority of the deleted category
+    3. Finally the `Category` will be deleted
+
+    Args:
+        template_category_id (str): The id of the template_category to delete
+        cascade (bool, optional): Specify whether to dissociate the category from templates that use it to force removal. Defaults to False.
+    """
+    template_category = dao_get_template_category_by_id(template_category_id)
+    templates = Template.query.filter_by(template_category_id=template_category_id).all()
+
+    if not templates or cascade:
+        # When there are templates and we are cascading, we set the category to a default
+        # that matches the template's previous category's priority
+        if cascade:
+            for template in templates:
+                # Get the a default category that matches the previous priority of the template, based on template type
+                default_category_id = _get_default_category_id(
+                    template_category.sms_process_type
+                    if template.template_type == "sms"
+                    else template_category.email_process_type
+                )
+                template.category = dao_get_template_category_by_id(default_category_id)
+                template.updated_at = datetime.utcnow()
+                db.session.add(template)
+
+        db.session.delete(template_category)
+        db.session.commit()
+
+
+def _get_default_category_id(process_type):
+    default_categories = {
+        "bulk": current_app.config["DEFAULT_TEMPLATE_CATEGORY_LOW"],
+        "normal": current_app.config["DEFAULT_TEMPLATE_CATEGORY_MEDIUM"],
+        "priority": current_app.config["DEFAULT_TEMPLATE_CATEGORY_HIGH"],
+    }
+    return default_categories.get(process_type, current_app.config["DEFAULT_TEMPLATE_CATEGORY_LOW"])

app/dao/templates_dao.py

@@ -78,6 +78,71 @@ def dao_update_template_reply_to(template_id, reply_to):
     return template
 
 
+@transactional
+def dao_update_template_process_type(template_id, process_type):
+    Template.query.filter_by(id=template_id).update(
+        {
+            "process_type": process_type,
+        }
+    )
+    template = Template.query.filter_by(id=template_id).one()
+
+    history = TemplateHistory(
+        **{
+            "id": template.id,
+            "name": template.name,
+            "template_type": template.template_type,
+            "created_at": template.created_at,
+            "updated_at": template.updated_at,
+            "content": template.content,
+            "service_id": template.service_id,
+            "subject": template.subject,
+            "postage": template.postage,
+            "created_by_id": template.created_by_id,
+            "version": template.version,
+            "archived": template.archived,
+            "process_type": template.process_type,
+            "service_letter_contact_id": template.service_letter_contact_id,
+        }
+    )
+    db.session.add(history)
+    return template
+
+
+@transactional
+def dao_update_template_category(template_id, category_id):
+    Template.query.filter_by(id=template_id).update(
+        {
+            "template_category_id": category_id,
+            "updated_at": datetime.utcnow(),
+            "version": Template.version + 1,
+        }
+    )
+
+    template = Template.query.filter_by(id=template_id).one()
+
+    history = TemplateHistory(
+        **{
+            "id": template.id,
+            "name": template.name,
+            "template_type": template.template_type,
+            "created_at": template.created_at,
+            "updated_at": template.updated_at,
+            "content": template.content,
+            "service_id": template.service_id,
+            "subject": template.subject,
+            "postage": template.postage,
+            "created_by_id": template.created_by_id,
+            "version": template.version,
+            "archived": template.archived,
+            "process_type": template.process_type,
+            "service_letter_contact_id": template.service_letter_contact_id,
+        }
+    )
+    db.session.add(history)
+    return template
+
+
 @transactional
 def dao_redact_template(template, user_id):
     template.template_redacted.redact_personalisation = True

app/models.py

@@ -1033,6 +1033,40 @@ def get_users_with_permission(self):
 PRECOMPILED_TEMPLATE_NAME = "Pre-compiled PDF"
 
 
+class TemplateCategory(BaseModel):
+    __tablename__ = "template_categories"
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    name_en = db.Column(db.String(255), unique=True, nullable=False)
+    name_fr = db.Column(db.String(255), unique=True, nullable=False)
+    description_en = db.Column(db.String(200), nullable=True)
+    description_fr = db.Column(db.String(200), nullable=True)
+    sms_process_type = db.Column(db.String(200), nullable=False)
+    email_process_type = db.Column(db.String(200), nullable=False)
+    hidden = db.Column(db.Boolean, nullable=False, default=False)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+    updated_at = db.Column(db.DateTime, onupdate=datetime.datetime.utcnow)
+
+    def serialize(self):
+        return {
+            "id": self.id,
+            "name_en": self.name_en,
+            "name_fr": self.name_fr,
+            "description_en": self.description_en,
+            "description_fr": self.description_fr,
+            "sms_process_type": self.sms_process_type,
+            "email_process_type": self.email_process_type,
+            "hidden": self.hidden,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+        }
+
+    @classmethod
+    def from_json(cls, data):
+        fields = data.copy()
+        return cls(**fields)
+
+
 class TemplateBase(BaseModel):
     __abstract__ = True
 
@@ -1078,6 +1112,14 @@ def service_id(cls):
     def created_by_id(cls):
         return db.Column(UUID(as_uuid=True), db.ForeignKey("users.id"), index=True, nullable=False)
 
+    @declared_attr
+    def template_category_id(cls):
+        return db.Column(UUID(as_uuid=True), db.ForeignKey("template_categories.id"), index=True, nullable=True)
+
+    @declared_attr
+    def category(cls):
+        return db.relationship("TemplateCategory")
+
     @declared_attr
     def created_by(cls):
         return db.relationship("User")
@@ -1179,6 +1221,7 @@ class Template(TemplateBase):
 
     service = db.relationship("Service", backref="templates")
     version = db.Column(db.Integer, default=0, nullable=False)
+    category = db.relationship("TemplateCategory", lazy="joined", backref="templates")
 
     folder = db.relationship(
         "TemplateFolder",
@@ -1198,6 +1241,17 @@ def get_link(self):
             _external=True,
         )
 
+    @property
+    def template_process_type(self):
+        """By default we use the process_type from TemplateCategory, but allow admins to override it on a per-template basis.
+        Only when overriden do we use the process_type from the template itself.
+        """
+        if self.template_type == SMS_TYPE:
+            return self.process_type if self.process_type else self.template_categories.sms_process_type
+        elif self.template_type == EMAIL_TYPE:
+            return self.process_type if self.process_type else self.template_categories.email_process_type
+        return self.process_type
+
     @classmethod
     def from_json(cls, data, folder=None):
         """
@@ -1246,6 +1300,7 @@ class TemplateHistory(TemplateBase):
 
     service = db.relationship("Service")
     version = db.Column(db.Integer, primary_key=True, nullable=False)
+    category = db.relationship("TemplateCategory")
 
     @classmethod
     def from_json(cls, data):

app/schemas.py

@@ -107,6 +107,31 @@ def make_instance(self, data, **kwargs):
         return super(BaseSchema, self).make_instance(data)
 
 
+class TemplateCategorySchema(BaseSchema):
+    class Meta(BaseSchema.Meta):
+        model = models.TemplateCategory
+
+    @validates("name_en")
+    def validate_name_en(self, value):
+        if not value:
+            raise ValidationError("Invalid name")
+
+    @validates("name_fr")
+    def validate_name_fr(self, value):
+        if not value:
+            raise ValidationError("Invalid name")
+
+    @validates("sms_process_type")
+    def validate_sms_process_type(self, value):
+        if value not in models.TEMPLATE_PROCESS_TYPE:
+            raise ValidationError("Invalid SMS process type")
+
+    @validates("email_process_type")
+    def validate_email_process_type(self, value):
+        if value not in models.TEMPLATE_PROCESS_TYPE:
+            raise ValidationError("Invalid email process type")
+
+
 class UserSchema(BaseSchema):
     permissions = fields.Method("user_permissions", dump_only=True)
     password_changed_at = field_for(models.User, "password_changed_at", format="%Y-%m-%d %H:%M:%S.%f")
@@ -805,6 +830,7 @@ def validate_archived(self, data, **kwargs):
 service_history_schema = ServiceHistorySchema()
 api_key_history_schema = ApiKeyHistorySchema()
 template_history_schema = TemplateHistorySchema()
+template_category_schema = TemplateCategorySchema()
 event_schema = EventSchema()
 provider_details_schema = ProviderDetailsSchema()
 provider_details_history_schema = ProviderDetailsHistorySchema()

app/template/rest.py

@@ -25,6 +25,8 @@
     dao_get_template_versions,
     dao_redact_template,
     dao_update_template,
+    dao_update_template_category,
+    dao_update_template_process_type,
     dao_update_template_reply_to,
     get_precompiled_letter_template,
 )
@@ -132,6 +134,24 @@ def create_template(service_id):
     return jsonify(data=template_schema.dump(new_template)), 201
 
 
+@template_blueprint.route("/<uuid:template_id>/category/<uuid:template_category_id>", methods=["POST"])
+def update_templates_category(service_id, template_id, template_category_id):
+    updated = dao_update_template_category(template_id, template_category_id)
+    return jsonify(data=template_schema.dump(updated)), 200
+
+
+@template_blueprint.route("/<uuid:template_id>/process-type", methods=["POST"])
+def update_template_process_type(template_id):
+    data = request.get_json()
+    if "process_type" not in data:
+        message = "Field is required"
+        errors = {"process_type": [message]}
+        raise InvalidRequest(errors, status_code=400)
+
+    updated = dao_update_template_process_type(template_id=template_id, process_type=data.get("process_type"))
+    return jsonify(data=template_schema.dump(updated)), 200
+
+
 @template_blueprint.route("/<uuid:template_id>", methods=["POST"])
 def update_template(service_id, template_id):
     fetched_template = dao_get_template_by_id_and_service_id(template_id=template_id, service_id=service_id)