diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f65007b2b..dec0b9fac 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -48,7 +48,7 @@ All other changes require just a single approving review.--> - [ ] Added at least 2 developers as PR reviewers (only 1 will need to approve) - [ ] Messaged on Slack or in standup to notify the team that a PR is ready for review - [ ] Changes to “how we do things” are documented in READMEs and or onboarding guide -- [ ] If any model was updated to modify/add/delete columns, makemigrations was ran and the assoicated migrations file has been commited. +- [ ] If any model was updated to modify/add/delete columns, makemigrations was ran and the associated migrations file has been commited. #### Ensured code standards are met (Original Developer) @@ -72,7 +72,7 @@ All other changes require just a single approving review.--> - [ ] Reviewed this code and left comments - [ ] Checked that all code is adequately covered by tests - [ ] Made it clear which comments need to be addressed before this work is merged -- [ ] If any model was updated to modify/add/delete columns, makemigrations was ran and the assoicated migrations file has been commited. +- [ ] If any model was updated to modify/add/delete columns, makemigrations was ran and the associated migrations file has been commited. #### Ensured code standards are met (Code reviewer) diff --git a/.github/workflows/deploy-development.yaml b/.github/workflows/deploy-development.yaml index 562b2b11f..686635c20 100644 --- a/.github/workflows/deploy-development.yaml +++ b/.github/workflows/deploy-development.yaml @@ -38,3 +38,11 @@ jobs: cf_org: cisa-dotgov cf_space: development push_arguments: "-f ops/manifests/manifest-development.yaml" + - name: Run Django migrations + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_DEVELOPMENT_USERNAME }} + cf_password: ${{ secrets.CF_DEVELOPMENT_PASSWORD }} + cf_org: cisa-dotgov + cf_space: development + cf_command: "run-task getgov-development --command 'python manage.py migrate' --name migrate" diff --git a/docs/operations/README.md b/docs/operations/README.md index 0e7b7a432..0bd55ab51 100644 --- a/docs/operations/README.md +++ b/docs/operations/README.md @@ -237,3 +237,7 @@ Bugs on production software need to be documented quickly and triaged to determi 3. In the case where the engineering lead is is unresponsive or unavailable to assign the ticket immediately, the product team will make sure an engineer volunteers or is assigned to the ticket/PR review ASAP. 4. Once done, the developer must make a PR and should tag the assigned PR reviewers in our Slack dev channel stating that the PR is now waiting on their review. These reviewers should drop other tasks in order to review this promptly. 5. See the the section above on [Making bug fixes on stable](#making-bug-fixes-on-stable-during-production) for how to push changes to stable once the PR is approved + +# Investigating and monitoring the health of the Registrar + +Sometimes, we may want individuals to routinely monitor the Registrar's health, such as after big feature launches. The cadence of such monitoring and what we look for is subject to change and is instead documented in [Checklist for production verification document](https://docs.google.com/document/d/15b_qwEZMiL76BHeRHnznV1HxDQcxNRt--vPSEfixBOI). All project team members should feel free to suggest edits to this document and should refer to it if production-level monitoring is underway. \ No newline at end of file diff --git a/src/registrar/admin.py b/src/registrar/admin.py index bd5555805..325081575 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -20,6 +20,8 @@ from auditlog.models import LogEntry # type: ignore from auditlog.admin import LogEntryAdmin # type: ignore from django_fsm import TransitionNotAllowed # type: ignore +from django.utils.safestring import mark_safe +from django.utils.html import escape logger = logging.getLogger(__name__) @@ -452,6 +454,60 @@ def get_readonly_fields(self, request, obj=None): readonly_fields.extend([field for field in self.analyst_readonly_fields]) return readonly_fields # Read-only fields for analysts + def change_view(self, request, object_id, form_url="", extra_context=None): + """Extend the change_view for Contact objects in django admin. + Customize to display related objects to the Contact. These will be passed + through the messages construct to the template for display to the user.""" + + # Fetch the Contact instance + contact = models.Contact.objects.get(pk=object_id) + + # initialize related_objects array + related_objects = [] + # for all defined fields in the model + for related_field in contact._meta.get_fields(): + # if the field is a relation to another object + if related_field.is_relation: + # Check if the related field is not None + related_manager = getattr(contact, related_field.name) + if related_manager is not None: + # Check if it's a ManyToManyField/reverse ForeignKey or a OneToOneField + # Do this by checking for get_queryset method on the related_manager + if hasattr(related_manager, "get_queryset"): + # Handles ManyToManyRel and ManyToOneRel + queryset = related_manager.get_queryset() + else: + # Handles OneToOne rels, ie. User + queryset = [related_manager] + + for obj in queryset: + # for each object, build the edit url in this view and add as tuple + # to the related_objects array + app_label = obj._meta.app_label + model_name = obj._meta.model_name + obj_id = obj.id + change_url = reverse("admin:%s_%s_change" % (app_label, model_name), args=[obj_id]) + related_objects.append((change_url, obj)) + + if related_objects: + message = "
" + if len(related_objects) > 5: + related_objects_over_five = len(related_objects) - 5 + message += f"And {related_objects_over_five} more...
" + + message_html = mark_safe(message) # nosec + messages.warning( + request, + message_html, + ) + + return super().change_view(request, object_id, form_url, extra_context=extra_context) + class WebsiteAdmin(ListHeaderAdmin): """Custom website admin class.""" @@ -1239,7 +1295,7 @@ class DraftDomainAdmin(ListHeaderAdmin): search_help_text = "Search by draft domain name." -class VeryImportantPersonAdmin(ListHeaderAdmin): +class VerifiedByStaffAdmin(ListHeaderAdmin): list_display = ("email", "requestor", "truncated_notes", "created_at") search_fields = ["email"] search_help_text = "Search by email." @@ -1282,4 +1338,4 @@ def save_model(self, request, obj, form, change): admin.site.register(models.PublicContact, AuditedAdmin) admin.site.register(models.DomainApplication, DomainApplicationAdmin) admin.site.register(models.TransitionDomain, TransitionDomainAdmin) -admin.site.register(models.VeryImportantPerson, VeryImportantPersonAdmin) +admin.site.register(models.VerifiedByStaff, VerifiedByStaffAdmin) diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index de7ef6172..587b95305 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -130,7 +130,7 @@ function inlineToast(el, id, style, msg) { } } -function _checkDomainAvailability(el) { +function checkDomainAvailability(el) { const callback = (response) => { toggleInputValidity(el, (response && response.available), msg=response.message); announce(el.id, response.message); @@ -154,9 +154,6 @@ function _checkDomainAvailability(el) { fetchJSON(`available/?domain=${el.value}`, callback); } -/** Call the API to see if the domain is good. */ -const checkDomainAvailability = debounce(_checkDomainAvailability); - /** Hides the toast message and clears the aira live region. */ function clearDomainAvailability(el) { el.classList.remove('usa-input--success'); @@ -206,13 +203,33 @@ function handleInputValidation(e) { } /** On button click, handles running any associated validators. */ -function handleValidationClick(e) { +function validateFieldInput(e) { const attribute = e.target.getAttribute("validate-for") || ""; if (!attribute.length) return; const input = document.getElementById(attribute); + removeFormErrors(input, true); runValidators(input); } + +function validateFormsetInputs(e, availabilityButton) { + + // Collect input IDs from the repeatable forms + let inputs = Array.from(document.querySelectorAll('.repeatable-form input')) + + // Run validators for each input + inputs.forEach(input => { + removeFormErrors(input, true); + runValidators(input); + }); + + // Set the validate-for attribute on the button with the collected input IDs + // Not needed for functionality but nice for accessibility + inputs = inputs.map(input => input.id).join(', '); + availabilityButton.setAttribute('validate-for', inputs); + +} + // <<>><<>><<>><<>><<>><<>><<>><<>><<>><<>><<>><<>><<>><<>><<>> // Initialization code. @@ -232,14 +249,64 @@ function handleValidationClick(e) { for(const input of needsValidation) { input.addEventListener('input', handleInputValidation); } + const alternativeDomainsAvailability = document.getElementById('validate-alt-domains-availability'); const activatesValidation = document.querySelectorAll('[validate-for]'); + for(const button of activatesValidation) { - button.addEventListener('click', handleValidationClick); + // Adds multi-field validation for alternative domains + if (button === alternativeDomainsAvailability) { + button.addEventListener('click', (e) => { + validateFormsetInputs(e, alternativeDomainsAvailability) + }); + } else { + button.addEventListener('click', validateFieldInput); + } } })(); /** - * Delete method for formsets that diff in the view and delete in the model (Nameservers, DS Data) + * Removes form errors surrounding a form input + */ +function removeFormErrors(input, removeStaleAlerts=false){ + // Remove error message + let errorMessage = document.getElementById(`${input.id}__error-message`); + if (errorMessage) { + errorMessage.remove(); + }else{ + return + } + + // Remove error classes + if (input.classList.contains('usa-input--error')) { + input.classList.remove('usa-input--error'); + } + + // Get the form label + let label = document.querySelector(`label[for="${input.id}"]`); + if (label) { + label.classList.remove('usa-label--error'); + + // Remove error classes from parent div + let parentDiv = label.parentElement; + if (parentDiv) { + parentDiv.classList.remove('usa-form-group--error'); + } + } + + if (removeStaleAlerts){ + let staleAlerts = document.querySelectorAll(".usa-alert--error") + for (let alert of staleAlerts){ + // Don't remove the error associated with the input + if (alert.id !== `${input.id}--toast`) { + alert.remove() + } + } + } +} + +/** + * Prepare the namerservers and DS data forms delete buttons + * We will call this on the forms init, and also every time we add a form * */ function removeForm(e, formLabel, isNameserversForm, addButton, formIdentifier){ @@ -460,6 +527,7 @@ function hideDeletedForms() { let isNameserversForm = document.querySelector(".nameservers-form"); let isOtherContactsForm = document.querySelector(".other-contacts-form"); let isDsDataForm = document.querySelector(".ds-data-form"); + let isDotgovDomain = document.querySelector(".dotgov-domain-form"); // The Nameservers formset features 2 required and 11 optionals if (isNameserversForm) { cloneIndex = 2; @@ -472,6 +540,8 @@ function hideDeletedForms() { formLabel = "Organization contact"; container = document.querySelector("#other-employees"); formIdentifier = "other_contacts" + } else if (isDotgovDomain) { + formIdentifier = "dotgov_domain" } let totalForms = document.querySelector(`#id_${formIdentifier}-TOTAL_FORMS`); @@ -554,6 +624,7 @@ function hideDeletedForms() { // Reset the values of each input to blank inputs.forEach((input) => { input.classList.remove("usa-input--error"); + input.classList.remove("usa-input--success"); if (input.type === "text" || input.type === "number" || input.type === "password" || input.type === "email" || input.type === "tel") { input.value = ""; // Set the value to an empty string @@ -566,22 +637,25 @@ function hideDeletedForms() { let selects = newForm.querySelectorAll("select"); selects.forEach((select) => { select.classList.remove("usa-input--error"); + select.classList.remove("usa-input--success"); select.selectedIndex = 0; // Set the value to an empty string }); let labels = newForm.querySelectorAll("label"); labels.forEach((label) => { label.classList.remove("usa-label--error"); + label.classList.remove("usa-label--success"); }); let usaFormGroups = newForm.querySelectorAll(".usa-form-group"); usaFormGroups.forEach((usaFormGroup) => { usaFormGroup.classList.remove("usa-form-group--error"); + usaFormGroup.classList.remove("usa-form-group--success"); }); - // Remove any existing error messages - let usaErrorMessages = newForm.querySelectorAll(".usa-error-message"); - usaErrorMessages.forEach((usaErrorMessage) => { + // Remove any existing error and success messages + let usaMessages = newForm.querySelectorAll(".usa-error-message, .usa-alert"); + usaMessages.forEach((usaErrorMessage) => { let parentDiv = usaErrorMessage.closest('div'); if (parentDiv) { parentDiv.remove(); // Remove the parent div if it exists @@ -592,7 +666,8 @@ function hideDeletedForms() { // Attach click event listener on the delete buttons of the new form let newDeleteButton = newForm.querySelector(".delete-record"); - prepareNewDeleteButton(newDeleteButton, formLabel); + if (newDeleteButton) + prepareNewDeleteButton(newDeleteButton, formLabel); // Disable the add more button if we have 13 forms if (isNameserversForm && formNum == 13) { diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index a3d631243..760c4f13a 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -143,7 +143,7 @@ h1, h2, h3, .module h3 { padding: 0; - color: var(--primary); + color: var(--link-fg); margin: units(2) 0 units(1) 0; } @@ -258,3 +258,15 @@ h1, h2, h3, #select2-id_user-results { width: 100%; } + +// Content list inside of a DjA alert, unstyled +.messagelist_content-list--unstyled { + padding-left: 0; + li { + font-family: "Source Sans Pro Web", "Helvetica Neue", Helvetica, Roboto, Arial, sans-serif; + font-size: 13.92px!important; + background: none!important; + padding: 0!important; + margin: 0!important; + } +} diff --git a/src/registrar/assets/sass/_theme/_lists.scss b/src/registrar/assets/sass/_theme/_lists.scss new file mode 100644 index 000000000..b97abd569 --- /dev/null +++ b/src/registrar/assets/sass/_theme/_lists.scss @@ -0,0 +1,14 @@ +@use "uswds-core" as *; + +dt { + color: color('primary-dark'); + margin-top: units(2); + font-weight: font-weight('semibold'); + // The units mixin can only get us close, so it's between + // hardcoding the value and using in markup + font-size: 16.96px; +} +dd { + margin-left: 0; +} + diff --git a/src/registrar/assets/sass/_theme/_register-form.scss b/src/registrar/assets/sass/_theme/_register-form.scss index 6d268d155..7c93f0a10 100644 --- a/src/registrar/assets/sass/_theme/_register-form.scss +++ b/src/registrar/assets/sass/_theme/_register-form.scss @@ -6,12 +6,14 @@ margin-top: units(-1); } - //Tighter spacing when H2 is immediatly after H1 +// Tighter spacing when h2 is immediatly after h1 .register-form-step .usa-fieldset:first-of-type h2:first-of-type, .register-form-step h1 + h2 { margin-top: units(1); } +// register-form-review-header is used on the summary page and +// should not be styled like the register form headers .register-form-step h3 { color: color('primary-dark'); letter-spacing: $letter-space--xs; @@ -23,6 +25,16 @@ } } +h3.register-form-review-header { + color: color('primary-dark'); + margin-top: units(2); + margin-bottom: 0; + font-weight: font-weight('semibold'); + // The units mixin can only get us close, so it's between + // hardcoding the value and using in markup + font-size: 16.96px; +} + .register-form-step h4 { margin-bottom: 0; diff --git a/src/registrar/assets/sass/_theme/styles.scss b/src/registrar/assets/sass/_theme/styles.scss index 8a2e1d2d3..0239199e7 100644 --- a/src/registrar/assets/sass/_theme/styles.scss +++ b/src/registrar/assets/sass/_theme/styles.scss @@ -10,6 +10,7 @@ --- Custom Styles ---------------------------------*/ @forward "base"; @forward "typography"; +@forward "lists"; @forward "buttons"; @forward "forms"; @forward "fieldsets"; diff --git a/src/registrar/forms/application_wizard.py b/src/registrar/forms/application_wizard.py index 85ce28bb6..1ee7e0036 100644 --- a/src/registrar/forms/application_wizard.py +++ b/src/registrar/forms/application_wizard.py @@ -420,7 +420,7 @@ def clean_alternative_domain(self): alternative_domain = forms.CharField( required=False, - label="", + label="Alternative domain", ) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 755c9b98a..c082552eb 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -182,8 +182,6 @@ def update_transition_domain_models(self): # STEP 5: Parse creation and expiration data updated_transition_domain = self.parse_creation_expiration_data(domain_name, transition_domain) - # Check if the instance has changed before saving - updated_transition_domain.save() updated_transition_domains.append(updated_transition_domain) logger.info(f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" f"{TerminalColors.ENDC}") @@ -199,6 +197,28 @@ def update_transition_domain_models(self): ) failed_transition_domains.append(domain_name) + updated_fields = [ + "organization_name", + "organization_type", + "federal_type", + "federal_agency", + "first_name", + "middle_name", + "last_name", + "email", + "phone", + "epp_creation_date", + "epp_expiration_date", + ] + + batch_size = 1000 + # Create a Paginator object. Bulk_update on the full dataset + # is too memory intensive for our current app config, so we can chunk this data instead. + paginator = Paginator(updated_transition_domains, batch_size) + for page_num in paginator.page_range: + page = paginator.page(page_num) + TransitionDomain.objects.bulk_update(page.object_list, updated_fields) + failed_count = len(failed_transition_domains) if failed_count == 0: if self.debug: diff --git a/src/registrar/migrations/0065_create_groups_v06.py b/src/registrar/migrations/0065_create_groups_v06.py new file mode 100644 index 000000000..d2cb32cee --- /dev/null +++ b/src/registrar/migrations/0065_create_groups_v06.py @@ -0,0 +1,37 @@ +# This migration creates the create_full_access_group and create_cisa_analyst_group groups +# It is dependent on 0035 (which populates ContentType and Permissions) +# If permissions on the groups need changing, edit CISA_ANALYST_GROUP_PERMISSIONS +# in the user_group model then: +# [NOT RECOMMENDED] +# step 1: docker-compose exec app ./manage.py migrate --fake registrar 0035_contenttypes_permissions +# step 2: docker-compose exec app ./manage.py migrate registrar 0036_create_groups +# step 3: fake run the latest migration in the migrations list +# [RECOMMENDED] +# Alternatively: +# step 1: duplicate the migration that loads data +# step 2: docker-compose exec app ./manage.py migrate + +from django.db import migrations +from registrar.models import UserGroup +from typing import Any + + +# For linting: RunPython expects a function reference, +# so let's give it one +def create_groups(apps, schema_editor) -> Any: + UserGroup.create_cisa_analyst_group(apps, schema_editor) + UserGroup.create_full_access_group(apps, schema_editor) + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0064_alter_domainapplication_address_line1_and_more"), + ] + + operations = [ + migrations.RunPython( + create_groups, + reverse_code=migrations.RunPython.noop, + atomic=True, + ), + ] diff --git a/src/registrar/migrations/0066_rename_veryimportantperson_verifiedbystaff_and_more.py b/src/registrar/migrations/0066_rename_veryimportantperson_verifiedbystaff_and_more.py new file mode 100644 index 000000000..d167334a0 --- /dev/null +++ b/src/registrar/migrations/0066_rename_veryimportantperson_verifiedbystaff_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.7 on 2024-01-29 22:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0065_create_groups_v06"), + ] + + operations = [ + migrations.RenameModel( + old_name="VeryImportantPerson", + new_name="VerifiedByStaff", + ), + migrations.AlterModelOptions( + name="verifiedbystaff", + options={"verbose_name_plural": "Verified by staff"}, + ), + ] diff --git a/src/registrar/migrations/0067_create_groups_v07.py b/src/registrar/migrations/0067_create_groups_v07.py new file mode 100644 index 000000000..85138d4af --- /dev/null +++ b/src/registrar/migrations/0067_create_groups_v07.py @@ -0,0 +1,37 @@ +# This migration creates the create_full_access_group and create_cisa_analyst_group groups +# It is dependent on 0035 (which populates ContentType and Permissions) +# If permissions on the groups need changing, edit CISA_ANALYST_GROUP_PERMISSIONS +# in the user_group model then: +# [NOT RECOMMENDED] +# step 1: docker-compose exec app ./manage.py migrate --fake registrar 0035_contenttypes_permissions +# step 2: docker-compose exec app ./manage.py migrate registrar 0036_create_groups +# step 3: fake run the latest migration in the migrations list +# [RECOMMENDED] +# Alternatively: +# step 1: duplicate the migration that loads data +# step 2: docker-compose exec app ./manage.py migrate + +from django.db import migrations +from registrar.models import UserGroup +from typing import Any + + +# For linting: RunPython expects a function reference, +# so let's give it one +def create_groups(apps, schema_editor) -> Any: + UserGroup.create_cisa_analyst_group(apps, schema_editor) + UserGroup.create_full_access_group(apps, schema_editor) + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0066_rename_veryimportantperson_verifiedbystaff_and_more"), + ] + + operations = [ + migrations.RunPython( + create_groups, + reverse_code=migrations.RunPython.noop, + atomic=True, + ), + ] diff --git a/src/registrar/models/__init__.py b/src/registrar/models/__init__.py index 90cb2e286..d9ccd64cb 100644 --- a/src/registrar/models/__init__.py +++ b/src/registrar/models/__init__.py @@ -13,7 +13,7 @@ from .user_group import UserGroup from .website import Website from .transition_domain import TransitionDomain -from .very_important_person import VeryImportantPerson +from .verified_by_staff import VerifiedByStaff __all__ = [ "Contact", @@ -30,7 +30,7 @@ "UserGroup", "Website", "TransitionDomain", - "VeryImportantPerson", + "VerifiedByStaff", ] auditlog.register(Contact) @@ -47,4 +47,4 @@ auditlog.register(UserGroup, m2m_fields=["permissions"]) auditlog.register(Website) auditlog.register(TransitionDomain) -auditlog.register(VeryImportantPerson) +auditlog.register(VerifiedByStaff) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 1a581a4ec..27a8364bc 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -910,10 +910,15 @@ def renew(self): raise NotImplementedError() def get_security_email(self): - logger.info("get_security_email-> getting the contact ") - secContact = self.security_contact - if secContact is not None: - return secContact.email + logger.info("get_security_email-> getting the contact") + + security = PublicContact.ContactTypeChoices.SECURITY + security_contact = self.generic_contact_getter(security) + + # If we get a valid value for security_contact, pull its email + # Otherwise, just return nothing + if security_contact is not None and isinstance(security_contact, PublicContact): + return security_contact.email else: return None @@ -1121,7 +1126,6 @@ def generic_contact_getter(self, contact_type_choice: PublicContact.ContactTypeC If you wanted to setup getter logic for Security, you would call: cache_contact_helper(PublicContact.ContactTypeChoices.SECURITY), or cache_contact_helper("security"). - """ # registrant_contact(s) are an edge case. They exist on # the "registrant" property as opposed to contacts. diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index 269569bfe..bf904a044 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -7,7 +7,7 @@ from .domain_invitation import DomainInvitation from .transition_domain import TransitionDomain -from .very_important_person import VeryImportantPerson +from .verified_by_staff import VerifiedByStaff from .domain import Domain from phonenumber_field.modelfields import PhoneNumberField # type: ignore @@ -91,7 +91,7 @@ def needs_identity_verification(cls, email, uuid): return False # New users flagged by Staff to bypass ial2 - if VeryImportantPerson.objects.filter(email=email).exists(): + if VerifiedByStaff.objects.filter(email=email).exists(): return False # A new incoming user who is being invited to be a domain manager (that is, diff --git a/src/registrar/models/user_group.py b/src/registrar/models/user_group.py index a733239c7..a32406a05 100644 --- a/src/registrar/models/user_group.py +++ b/src/registrar/models/user_group.py @@ -66,6 +66,11 @@ def create_cisa_analyst_group(apps, schema_editor): "model": "userdomainrole", "permissions": ["view_userdomainrole", "delete_userdomainrole"], }, + { + "app_label": "registrar", + "model": "verifiedbystaff", + "permissions": ["add_verifiedbystaff", "change_verifiedbystaff", "delete_verifiedbystaff"], + }, ] # Avoid error: You can't execute queries until the end diff --git a/src/registrar/models/utility/domain_helper.py b/src/registrar/models/utility/domain_helper.py index a808ef803..3267f0c93 100644 --- a/src/registrar/models/utility/domain_helper.py +++ b/src/registrar/models/utility/domain_helper.py @@ -57,6 +57,9 @@ def _validate_domain_string(domain, blank_ok): # If blank ok is true, just return the domain return domain + if domain.startswith("www."): + domain = domain[4:] + if domain.endswith(".gov"): domain = domain[:-4] diff --git a/src/registrar/models/very_important_person.py b/src/registrar/models/verified_by_staff.py similarity index 85% rename from src/registrar/models/very_important_person.py rename to src/registrar/models/verified_by_staff.py index 9134cb893..4c9e76e9d 100644 --- a/src/registrar/models/very_important_person.py +++ b/src/registrar/models/verified_by_staff.py @@ -3,7 +3,7 @@ from .utility.time_stamped_model import TimeStampedModel -class VeryImportantPerson(TimeStampedModel): +class VerifiedByStaff(TimeStampedModel): """emails that get added to this table will bypass ial2 on login.""" @@ -28,5 +28,8 @@ class VeryImportantPerson(TimeStampedModel): help_text="Notes", ) + class Meta: + verbose_name_plural = "Verified by staff" + def __str__(self): return self.email diff --git a/src/registrar/templates/application_dotgov_domain.html b/src/registrar/templates/application_dotgov_domain.html index 1838f33f4..39f9935c2 100644 --- a/src/registrar/templates/application_dotgov_domain.html +++ b/src/registrar/templates/application_dotgov_domain.html @@ -48,16 +48,15 @@Federally-recognized tribe
{% endif %} - {% if application.state_recognized_tribe %}State-recognized tribe
{% endif %} - {% endif %} - {% if step == Step.ORGANIZATION_FEDERAL %} - {{ application.get_federal_type_display|default:"Incomplete" }} - {% endif %} - {% if step == Step.ORGANIZATION_ELECTION %} - {{ application.is_election_board|yesno:"Yes,No,Incomplete" }} - {% endif %} - {% if step == Step.ORGANIZATION_CONTACT %} - {% if application.organization_name %} - {% include "includes/organization_address.html" with organization=application %} - {% else %} - Incomplete - {% endif %} +{{ application.about_your_organization|default:"Incomplete" }}
+ {% endif %} + + {% if step == Step.TRIBAL_GOVERNMENT %} + {% namespaced_url 'application' step as application_url %} + {% with title=form_titles|get_item:step value=application.tribe_name|default:"Incomplete" %} + {% include "includes/summary_item.html" with title=title value=value heading_level=heading_level editable=True edit_link=application_url %} + {% endwith %} + {% if application.federally_recognized_tribe %}Federally-recognized tribe
{% endif %} + {% if application.state_recognized_tribe %}State-recognized tribe
{% endif %} + {% endif %} + + + {% if step == Step.ORGANIZATION_FEDERAL %} + {% namespaced_url 'application' step as application_url %} + {% with title=form_titles|get_item:step value=application.get_federal_type_display|default:"Incomplete" %} + {% include "includes/summary_item.html" with title=title value=value heading_level=heading_level editable=True edit_link=application_url %} + {% endwith %} + {% endif %} + + {% if step == Step.ORGANIZATION_ELECTION %} + {% namespaced_url 'application' step as application_url %} + {% with title=form_titles|get_item:step value=application.is_election_board|yesno:"Yes,No,Incomplete" %} + {% include "includes/summary_item.html" with title=title value=value heading_level=heading_level editable=True edit_link=application_url %} + {% endwith %} + {% endif %} + + {% if step == Step.ORGANIZATION_CONTACT %} + {% namespaced_url 'application' step as application_url %} + {% if application.organization_name %} + {% with title=form_titles|get_item:step value=application %} + {% include "includes/summary_item.html" with title=title value=value heading_level=heading_level editable=True edit_link=application_url address='true' %} + {% endwith %} + {% else %} + {% with title=form_titles|get_item:step value='Incomplete' %} + {% include "includes/summary_item.html" with title=title value=value heading_level=heading_level editable=True edit_link=application_url %} + {% endwith %} {% endif %} - {% if step == Step.AUTHORIZING_OFFICIAL %} - {% if application.authorizing_official %} -Contact {{ forloop.counter }}
- {% include "includes/contact.html" with contact=other %} -No other employees from your organization?
- {{ application.no_other_contacts_rationale|default:"Incomplete" }} -- Contact {{forloop.counter}} -
- {% include "includes/contact.html" with contact=item %}+ None +
+ {% endif %} {% endif %} {% else %} {% include "includes/contact.html" with contact=value %} @@ -57,10 +62,10 @@ {% endspaceless %}) {% endif %} {% else %} -{{ value | first }}
+{{ value | first }}
{% endif %} {% else %} -And 1 more...
", + ) + def tearDown(self): + DomainApplication.objects.all().delete() + Contact.objects.all().delete() User.objects.all().delete() -class VeryImportantPersonAdminTestCase(TestCase): +class VerifiedByStaffAdminTestCase(TestCase): def setUp(self): self.superuser = create_superuser() self.factory = RequestFactory() @@ -1750,13 +1829,13 @@ def test_save_model_sets_user_field(self): self.client.force_login(self.superuser) # Create an instance of the admin class - admin_instance = VeryImportantPersonAdmin(model=VeryImportantPerson, admin_site=None) + admin_instance = VerifiedByStaffAdmin(model=VerifiedByStaff, admin_site=None) - # Create a VeryImportantPerson instance - vip_instance = VeryImportantPerson(email="test@example.com", notes="Test Notes") + # Create a VerifiedByStaff instance + vip_instance = VerifiedByStaff(email="test@example.com", notes="Test Notes") # Create a request object - request = self.factory.post("/admin/yourapp/veryimportantperson/add/") + request = self.factory.post("/admin/yourapp/VerifiedByStaff/add/") request.user = self.superuser # Call the save_model method diff --git a/src/registrar/tests/test_emails.py b/src/registrar/tests/test_emails.py index a4f32bfcf..f2a94a186 100644 --- a/src/registrar/tests/test_emails.py +++ b/src/registrar/tests/test_emails.py @@ -102,9 +102,9 @@ def test_submission_confirmation_no_other_contacts_spacing(self): application.submit() _, kwargs = self.mock_client.send_email.call_args body = kwargs["Content"]["Simple"]["Body"]["Text"]["Data"] - self.assertNotIn("Other employees from your organization:", body) # spacing should be right between adjacent elements - self.assertRegex(body, r"5556\n\nAnything else") + self.assertRegex(body, r"5556\n\nOther employees") + self.assertRegex(body, r"None\n\nAnything else") @boto3_mocking.patching def test_submission_confirmation_alternative_govdomain_spacing(self): @@ -117,7 +117,7 @@ def test_submission_confirmation_alternative_govdomain_spacing(self): body = kwargs["Content"]["Simple"]["Body"]["Text"]["Data"] self.assertIn("city1.gov", body) # spacing should be right between adjacent elements - self.assertRegex(body, r"city.gov\ncity1.gov\n\nPurpose of your domain:") + self.assertRegex(body, r"city.gov\n\nAlternative domains:\ncity1.gov\n\nPurpose of your domain:") @boto3_mocking.patching def test_submission_confirmation_no_alternative_govdomain_spacing(self): diff --git a/src/registrar/tests/test_forms.py b/src/registrar/tests/test_forms.py index a8d84ba7b..c9dd3c4f0 100644 --- a/src/registrar/tests/test_forms.py +++ b/src/registrar/tests/test_forms.py @@ -64,6 +64,12 @@ def test_requested_domain_valid(self): form = DotGovDomainForm(data={"requested_domain": "top-level-agency"}) self.assertEqual(len(form.errors), 0) + def test_requested_domain_starting_www(self): + """Test a valid domain name with .www at the beginning.""" + form = DotGovDomainForm(data={"requested_domain": "www.top-level-agency"}) + self.assertEqual(len(form.errors), 0) + self.assertEqual(form.cleaned_data["requested_domain"], "top-level-agency") + def test_requested_domain_ending_dotgov(self): """Just a valid domain name with .gov at the end.""" form = DotGovDomainForm(data={"requested_domain": "top-level-agency.gov"}) diff --git a/src/registrar/tests/test_migrations.py b/src/registrar/tests/test_migrations.py index cc9d379e5..773a885c1 100644 --- a/src/registrar/tests/test_migrations.py +++ b/src/registrar/tests/test_migrations.py @@ -43,6 +43,9 @@ def test_groups_created(self): "change_user", "delete_userdomainrole", "view_userdomainrole", + "add_verifiedbystaff", + "change_verifiedbystaff", + "delete_verifiedbystaff", "change_website", ] diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index ef6522747..d0005cbd5 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -16,7 +16,7 @@ import boto3_mocking from registrar.models.transition_domain import TransitionDomain -from registrar.models.very_important_person import VeryImportantPerson # type: ignore +from registrar.models.verified_by_staff import VerifiedByStaff # type: ignore from .common import MockSESClient, less_console_noise, completed_application from django_fsm import TransitionNotAllowed @@ -656,7 +656,7 @@ def test_identity_verification_with_transition_user(self): def test_identity_verification_with_very_important_person(self): """A Very Important Person should return False when tested with class method needs_identity_verification""" - VeryImportantPerson.objects.get_or_create(email=self.user.email) + VerifiedByStaff.objects.get_or_create(email=self.user.email) self.assertFalse(User.needs_identity_verification(self.user.email, self.user.username)) def test_identity_verification_with_invited_user(self): diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 3bbcfcf01..dcb6ba9e0 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -712,7 +712,7 @@ def test_application_form_submission(self): # Review page contains all the previously entered data # Let's make sure the long org name is displayed - self.assertContains(review_page, "Federal: an agency of the U.S. government") + self.assertContains(review_page, "Federal") self.assertContains(review_page, "Executive") self.assertContains(review_page, "Testorg") self.assertContains(review_page, "address 1") @@ -2360,18 +2360,6 @@ def test_long_org_name_in_application(self): self.assertContains(type_page, "Federal: an agency of the U.S. government") - def test_long_org_name_in_application_manage(self): - """ - Make sure the long name is displaying in the application summary - page (manage your application) - """ - completed_application(status=DomainApplication.ApplicationStatus.SUBMITTED, user=self.user) - home_page = self.app.get("/") - self.assertContains(home_page, "city.gov") - # click the "Edit" link - detail_page = home_page.click("Manage", index=0) - self.assertContains(detail_page, "Federal: an agency of the U.S. government") - def test_submit_modal_no_domain_text_fallback(self): """When user clicks on submit your domain request and the requested domain is null (possible through url direct access to the review page), present diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 3924c03c4..f9608f553 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -3,10 +3,13 @@ from datetime import datetime from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation -from registrar.models.public_contact import PublicContact -from django.db.models import Value -from django.db.models.functions import Coalesce from django.utils import timezone +from django.core.paginator import Paginator +from django.db.models import F, Value, CharField +from django.db.models.functions import Concat, Coalesce + +from registrar.models.public_contact import PublicContact + logger = logging.getLogger(__name__) @@ -20,50 +23,77 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): - domain_infos = DomainInformation.objects.filter(**filter_condition).order_by(*sort_fields) - return domain_infos + domain_infos = ( + DomainInformation.objects.select_related("domain", "authorizing_official") + .filter(**filter_condition) + .order_by(*sort_fields) + ) + # Do a mass concat of the first and last name fields for authorizing_official. + # The old operation was computationally heavy for some reason, so if we precompute + # this here, it is vastly more efficient. + domain_infos_cleaned = domain_infos.annotate( + ao=Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ) + ) + return domain_infos_cleaned -def write_row(writer, columns, domain_info): - security_contacts = domain_info.domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) - # For linter - ao = " " - if domain_info.authorizing_official: - first_name = domain_info.authorizing_official.first_name or "" - last_name = domain_info.authorizing_official.last_name or "" - ao = first_name + " " + last_name +def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None): + """Given a set of columns, generate a new row from cleaned column data""" - security_email = " " - if security_contacts: - security_email = security_contacts[0].email + # Domain should never be none when parsing this information + if domain_info.domain is None: + raise ValueError("Domain is none") + + domain = domain_info.domain # type: ignore + + # Grab the security email from a preset dictionary. + # If nothing exists in the dictionary, grab from .contacts. + if security_emails_dict is not None and domain.name in security_emails_dict: + _email = security_emails_dict.get(domain.name) + security_email = _email if _email is not None else " " + else: + # If the dictionary doesn't contain that data, lets filter for it manually. + # This is a last resort as this is a more expensive operation. + security_contacts = domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) + _email = security_contacts[0].email if security_contacts else None + security_email = _email if _email is not None else " " - invalid_emails = {"registrar@dotgov.gov", "dotgov@cisa.dhs.gov"} # These are default emails that should not be displayed in the csv report - if security_email is not None and security_email.lower() in invalid_emails: + invalid_emails = {"registrar@dotgov.gov", "dotgov@cisa.dhs.gov"} + if security_email.lower() in invalid_emails: security_email = "(blank)" + if domain_info.federal_type: + domain_type = f"{domain_info.get_organization_type_display()} - {domain_info.get_federal_type_display()}" + else: + domain_type = domain_info.get_organization_type_display() + # create a dictionary of fields which can be included in output FIELDS = { - "Domain name": domain_info.domain.name, - "Domain type": domain_info.get_organization_type_display() + " - " + domain_info.get_federal_type_display() - if domain_info.federal_type - else domain_info.get_organization_type_display(), + "Domain name": domain.name, + "Domain type": domain_type, "Agency": domain_info.federal_agency, "Organization name": domain_info.organization_name, "City": domain_info.city, "State": domain_info.state_territory, - "AO": ao, + "AO": domain_info.ao, # type: ignore "AO email": domain_info.authorizing_official.email if domain_info.authorizing_official else " ", "Security contact email": security_email, - "Status": domain_info.domain.get_state_display(), - "Expiration date": domain_info.domain.expiration_date, - "Created at": domain_info.domain.created_at, - "First ready": domain_info.domain.first_ready, - "Deleted": domain_info.domain.deleted, + "Status": domain.get_state_display(), + "Expiration date": domain.expiration_date, + "Created at": domain.created_at, + "First ready": domain.first_ready, + "Deleted": domain.deleted, } - writer.writerow([FIELDS.get(column, "") for column in columns]) + row = [FIELDS.get(column, "") for column in columns] + return row def write_body( @@ -78,13 +108,41 @@ def write_body( """ # Get the domainInfos - domain_infos = get_domain_infos(filter_condition, sort_fields) - - all_domain_infos = list(domain_infos) + all_domain_infos = get_domain_infos(filter_condition, sort_fields) + + # Store all security emails to avoid epp calls or excessive filters + sec_contact_ids = all_domain_infos.values_list("domain__security_contact_registry_id", flat=True) + security_emails_dict = {} + public_contacts = ( + PublicContact.objects.only("email", "domain__name") + .select_related("domain") + .filter(registry_id__in=sec_contact_ids) + ) - # Write rows to CSV - for domain_info in all_domain_infos: - write_row(writer, columns, domain_info) + # Populate a dictionary of domain names and their security contacts + for contact in public_contacts: + domain: Domain = contact.domain + if domain is not None and domain.name not in security_emails_dict: + security_emails_dict[domain.name] = contact.email + else: + logger.warning("csv_export -> Domain was none for PublicContact") + + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_domain_infos, 1000) + for page_num in paginator.page_range: + page = paginator.page(page_num) + rows = [] + for domain_info in page.object_list: + try: + row = parse_row(columns, domain_info, security_emails_dict) + rows.append(row) + except ValueError: + # This should not happen. If it does, just skip this row. + # It indicates that DomainInformation.domain is None. + logger.error("csv_export -> Error when parsing row, domain was None") + continue + + writer.writerows(rows) def export_data_type_to_csv(csv_file):