From b0d1bc26da85e0e6ac8bae4746d529089a6b61de Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:15:13 -0700 Subject: [PATCH 01/14] revert csv-export --- src/registrar/tests/test_reports.py | 138 ++++---- src/registrar/utility/csv_export.py | 519 ++++++---------------------- 2 files changed, 181 insertions(+), 476 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index cafaff7b1..f91c5b299 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -71,8 +71,8 @@ def test_generate_federal_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), ] @@ -93,8 +93,8 @@ def test_generate_full_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("zdomain12.gov,Interstate,,,,,(blank)\r\n"), @@ -251,35 +251,32 @@ def test_domain_data_type(self): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Status,First ready on,Expiration date,Domain type,Agency," - "Organization name,City,State,SO,SO email," - "Security contact email,Domain managers,Invited domain managers\n" - "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive," - "Portfolio 1 Federal Agency,,,, ,,(blank)," - "meoward@rocks.com,squeaker@rocks.com\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive," - "Portfolio 1 Federal Agency,,,, ,,(blank)," - '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' - "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive," - "World War I Centennial Commission,,,, ,,(blank)," + "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," + "SO email,Security contact email,Domain managers,Invited domain managers\n" + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,,(blank),,," "meoward@rocks.com,\n" - "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),," + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," + ',,,(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' + "woofwardthethird@rocks.com\n" + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,," "squeaker@rocks.com\n" - "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "ddomain3.gov,On hold,(blank),2023-11-15,Federal," - "Armed Forces Retirement Home,,,, ,,security@mail.gov,,\n" - "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,(blank),meoward@rocks.com,\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,,,," + "security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,,(blank),,," + "meoward@rocks.com,squeaker@rocks.com\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,,(blank),,,meoward@rocks.com,\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -315,17 +312,20 @@ def test_domain_data_type_user(self): # We expect only domains associated with the user expected_content = ( "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name," - "City,State,SO,SO email,Security contact email,Domain managers,Invited domain managers\n" - "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," + "City,State,SO,SO email," + "Security contact email,Domain managers,Invited domain managers\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,,, ,," + '(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' + "woofwardthethird@rocks.com\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,(blank)," '"info@example.com, meoward@rocks.com",squeaker@rocks.com\n' - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," - '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -493,17 +493,17 @@ def test_domain_data_full(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" - "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" - "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" "zdomain12.gov,Interstate,,,,,(blank)\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -533,16 +533,16 @@ def test_domain_data_federal(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" - "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" - "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -587,13 +587,13 @@ def test_domain_growth(self): expected_content = ( "Domain name,Domain type,Agency,Organization name,City," "State,Status,Expiration date, Deleted\n" - "cdomain1.gov,Federal-Executive,Portfolio1FederalAgency,Ready,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,Ready,(blank)\n" - "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank)\n" - "zdomain12.gov,Interstate,Ready,(blank)\n" + "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" + "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" + "zdomain12.govInterstateReady(blank)\n" "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" - "sdomain8.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" - "xdomain7.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace @@ -611,6 +611,7 @@ def test_domain_managed(self): squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). She should show twice in this report but not in test_DomainManaged.""" + self.maxDiff = None # Create a CSV file in memory csv_file = StringIO() # Call the export functions @@ -645,6 +646,7 @@ def test_domain_managed(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -681,6 +683,7 @@ def test_domain_unmanaged(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -718,9 +721,10 @@ def test_domain_request_growth(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # @less_console_noise_decorator + @less_console_noise_decorator def test_domain_request_data_full(self): """Tests the full domain request report.""" # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data @@ -762,34 +766,35 @@ def test_domain_request_data_full(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - expected_content = ( # Header - "Domain request,Status,Domain type,Federal type,Federal agency,Organization name,Election office," - "City,State/territory,Region,Creator first name,Creator last name,Creator email," - "Creator approved domains count,Creator active requests count,Alternative domains,SO first name," - "SO last name,SO email,SO title/role,Request purpose,Request additional details,Other contacts," + "Domain request,Status,Domain type,Federal type," + "Federal agency,Organization name,Election office,City,State/territory," + "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," + "Creator active requests count,Alternative domains,SO first name,SO last name,SO email," + "SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content - "city5.gov,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," + "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city2.gov,In review,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," - "Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city3.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1," - '"cheeseville.gov, city1.gov, igorville.gov",,,,,Purpose of the site,CISA-first-name CISA-last-name | ' - 'There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, ' - 'Testy Tester testy2@town.com",' - 'test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' - "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy," - "Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com," - "cisaRep@igorville.gov,city.com,\n" - "city6.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," - "Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," + "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," + "testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' + 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' + "CISA-last-name " + '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' + 'testy2@town.com"' + ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com" + ",cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com," "cisaRep@igorville.gov,city.com,\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() @@ -857,6 +862,7 @@ def test_member_export(self): # Create a request and add the user to the request request = self.factory.get("/") request.user = self.user + self.maxDiff = None # Add portfolio to session request = GenericTestHelper._mock_user_request_for_factory(request) request.session["portfolio"] = self.portfolio_1 diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 97feae20c..a03e51de5 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -414,11 +414,8 @@ def get_model_annotation_dict(cls, request=None, **kwargs): ) .values(*shared_columns) ) - # Adding a order_by increases output predictability. - # Doesn't matter as much for normal use, but makes tests easier. - # We should also just be ordering by default anyway. - members = permissions.union(invitations).order_by("email_display") - return convert_queryset_to_dict(members, is_model=False) + + return convert_queryset_to_dict(permissions.union(invitations), is_model=False) @classmethod def get_invited_by_query(cls, object_id_query): @@ -528,115 +525,6 @@ def model(cls): # Return the model class that this export handles return DomainInformation - @classmethod - def get_computed_fields(cls, **kwargs): - """ - Get a dict of computed fields. - """ - # NOTE: These computed fields imitate @Property functions in the Domain model and Portfolio model where needed. - # This is for performance purposes. Since we are working with dictionary values and not - # model objects as we export data, trying to reinstate model objects in order to grab @property - # values negatively impacts performance. Therefore, we will follow best practice and use annotations - return { - "converted_generic_org_type": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__organization_type")), - # Otherwise, return the natively assigned value - default=F("generic_org_type"), - output_field=CharField(), - ), - "converted_federal_agency": Case( - # When portfolio is present, use its value instead - When( - Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), - then=F("portfolio__federal_agency__agency"), - ), - # Otherwise, return the natively assigned value - default=F("federal_agency__agency"), - output_field=CharField(), - ), - "converted_federal_type": Case( - # When portfolio is present, use its value instead - # NOTE: this is an @Property funciton in portfolio. - When( - Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), - then=F("portfolio__federal_agency__federal_type"), - ), - # Otherwise, return the natively assigned value - default=F("federal_type"), - output_field=CharField(), - ), - "converted_organization_name": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__organization_name")), - # Otherwise, return the natively assigned value - default=F("organization_name"), - output_field=CharField(), - ), - "converted_city": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__city")), - # Otherwise, return the natively assigned value - default=F("city"), - output_field=CharField(), - ), - "converted_state_territory": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__state_territory")), - # Otherwise, return the natively assigned value - default=F("state_territory"), - output_field=CharField(), - ), - "converted_so_email": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__email")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__email"), - output_field=CharField(), - ), - "converted_senior_official_last_name": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__last_name")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__last_name"), - output_field=CharField(), - ), - "converted_senior_official_first_name": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__first_name")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__first_name"), - output_field=CharField(), - ), - "converted_senior_official_title": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__title")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__title"), - output_field=CharField(), - ), - "converted_so_name": Case( - # When portfolio is present, use that senior official instead - When( - Q(portfolio__isnull=False) & Q(portfolio__senior_official__isnull=False), - then=Concat( - Coalesce(F("portfolio__senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("portfolio__senior_official__last_name"), Value("")), - output_field=CharField(), - ), - ), - # Otherwise, return the natively assigned senior official - default=Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - output_field=CharField(), - ), - } - @classmethod def update_queryset(cls, queryset, **kwargs): """ @@ -726,10 +614,10 @@ def parse_row(cls, columns, model): if first_ready_on is None: first_ready_on = "(blank)" - # organization_type has organization_type AND is_election - domain_org_type = model.get("converted_generic_org_type") + # organization_type has generic_org_type AND is_election + domain_org_type = model.get("organization_type") human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) - domain_federal_type = model.get("converted_federal_type") + domain_federal_type = model.get("federal_type") human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) domain_type = human_readable_domain_org_type if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: @@ -752,12 +640,12 @@ def parse_row(cls, columns, model): "First ready on": first_ready_on, "Expiration date": expiration_date, "Domain type": domain_type, - "Agency": model.get("converted_federal_agency"), - "Organization name": model.get("converted_organization_name"), - "City": model.get("converted_city"), - "State": model.get("converted_state_territory"), - "SO": model.get("converted_so_name"), - "SO email": model.get("converted_so_email"), + "Agency": model.get("federal_agency__agency"), + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State": model.get("state_territory"), + "SO": model.get("so_name"), + "SO email": model.get("senior_official__email"), "Security contact email": security_contact_email, "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), @@ -766,23 +654,8 @@ def parse_row(cls, columns, model): } row = [FIELDS.get(column, "") for column in columns] - return row - def get_filtered_domain_infos_by_org(domain_infos_to_filter, org_to_filter_by): - """Returns a list of Domain Requests that has been filtered by the given organization value.""" - - annotated_queryset = domain_infos_to_filter.annotate( - converted_generic_org_type=Case( - # Recreate the logic of the converted_generic_org_type property - # here in annotations - When(portfolio__isnull=False, then=F("portfolio__organization_type")), - default=F("generic_org_type"), - output_field=CharField(), - ) - ) - return annotated_queryset.filter(converted_generic_org_type=org_to_filter_by) - @classmethod def get_sliced_domains(cls, filter_condition): """Get filtered domains counts sliced by org type and election office. @@ -790,51 +663,23 @@ def get_sliced_domains(cls, filter_condition): when a domain has more that one manager. """ - domain_informations = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domain_informations.count() - federal = ( - cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.FEDERAL) - .distinct() - .count() - ) - interstate = cls.get_filtered_domain_infos_by_org( - domain_informations, DomainRequest.OrganizationChoices.INTERSTATE - ).count() + domains = DomainInformation.objects.all().filter(**filter_condition).distinct() + domains_count = domains.count() + federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() state_or_territory = ( - cls.get_filtered_domain_infos_by_org( - domain_informations, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY - ) - .distinct() - .count() - ) - tribal = ( - cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.TRIBAL) - .distinct() - .count() - ) - county = ( - cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.COUNTY) - .distinct() - .count() - ) - city = ( - cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.CITY) - .distinct() - .count() + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() ) + tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - cls.get_filtered_domain_infos_by_org( - domain_informations, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT - ) - .distinct() - .count() + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() ) school_district = ( - cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT) - .distinct() - .count() + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() ) - election_board = domain_informations.filter(is_election_board=True).distinct().count() + election_board = domains.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -861,7 +706,6 @@ def get_columns(cls): """ Overrides the columns for CSV export specific to DomainExport. """ - return [ "Domain name", "Status", @@ -879,13 +723,6 @@ def get_columns(cls): "Invited domain managers", ] - @classmethod - def get_annotations_for_sort(cls): - """ - Get a dict of annotations to make available for sorting. - """ - return cls.get_computed_fields() - @classmethod def get_sort_fields(cls): """ @@ -893,9 +730,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "converted_generic_org_type", - Coalesce("converted_federal_type", Value("ZZZZZ")), - "converted_federal_agency", + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", "domain__name", ] @@ -936,6 +773,20 @@ def get_prefetch_related(cls): """ return ["domain__permissions"] + @classmethod + def get_computed_fields(cls, delimiter=", ", **kwargs): + """ + Get a dict of computed fields. + """ + return { + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("senior_official__last_name"), Value("")), + output_field=CharField(), + ), + } + @classmethod def get_related_table_fields(cls): """ @@ -1041,7 +892,7 @@ def exporting_dr_data_to_csv(cls, response, request=None): cls.safe_get(getattr(request, "region_field", None)), request.status, cls.safe_get(getattr(request, "election_office", None)), - request.converted_federal_type, + request.federal_type, cls.safe_get(getattr(request, "domain_type", None)), cls.safe_get(getattr(request, "additional_details", None)), cls.safe_get(getattr(request, "creator_approved_domains_count", None)), @@ -1092,13 +943,6 @@ def get_columns(cls): "Security contact email", ] - @classmethod - def get_annotations_for_sort(cls, delimiter=", "): - """ - Get a dict of annotations to make available for sorting. - """ - return cls.get_computed_fields() - @classmethod def get_sort_fields(cls): """ @@ -1106,9 +950,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "converted_generic_org_type", - Coalesce("converted_federal_type", Value("ZZZZZ")), - "converted_federal_agency", + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", "domain__name", ] @@ -1146,6 +990,20 @@ def get_filter_conditions(cls, **kwargs): ], ) + @classmethod + def get_computed_fields(cls, delimiter=", ", **kwargs): + """ + Get a dict of computed fields. + """ + return { + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("senior_official__last_name"), Value("")), + output_field=CharField(), + ), + } + @classmethod def get_related_table_fields(cls): """ @@ -1179,13 +1037,6 @@ def get_columns(cls): "Security contact email", ] - @classmethod - def get_annotations_for_sort(cls, delimiter=", "): - """ - Get a dict of annotations to make available for sorting. - """ - return cls.get_computed_fields() - @classmethod def get_sort_fields(cls): """ @@ -1193,9 +1044,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "converted_generic_org_type", - Coalesce("converted_federal_type", Value("ZZZZZ")), - "converted_federal_agency", + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", "domain__name", ] @@ -1234,6 +1085,20 @@ def get_filter_conditions(cls, **kwargs): ], ) + @classmethod + def get_computed_fields(cls, delimiter=", ", **kwargs): + """ + Get a dict of computed fields. + """ + return { + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("senior_official__last_name"), Value("")), + output_field=CharField(), + ), + } + @classmethod def get_related_table_fields(cls): """ @@ -1611,180 +1476,24 @@ def model(cls): # Return the model class that this export handles return DomainRequest - def get_filtered_domain_requests_by_org(domain_requests_to_filter, org_to_filter_by): - """Returns a list of Domain Requests that has been filtered by the given organization value""" - annotated_queryset = domain_requests_to_filter.annotate( - converted_generic_org_type=Case( - # Recreate the logic of the converted_generic_org_type property - # here in annotations - When(portfolio__isnull=False, then=F("portfolio__organization_type")), - default=F("generic_org_type"), - output_field=CharField(), - ) - ) - return annotated_queryset.filter(converted_generic_org_type=org_to_filter_by) - - # return domain_requests_to_filter.filter( - # # Filter based on the generic org value returned by converted_generic_org_type - # id__in=[ - # domainRequest.id - # for domainRequest in domain_requests_to_filter - # if domainRequest.converted_generic_org_type - # and domainRequest.converted_generic_org_type == org_to_filter_by - # ] - # ) - - @classmethod - def get_computed_fields(cls, delimiter=", ", **kwargs): - """ - Get a dict of computed fields. - """ - # NOTE: These computed fields imitate @Property functions in the Domain model and Portfolio model where needed. - # This is for performance purposes. Since we are working with dictionary values and not - # model objects as we export data, trying to reinstate model objects in order to grab @property - # values negatively impacts performance. Therefore, we will follow best practice and use annotations - return { - "converted_generic_org_type": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__organization_type")), - # Otherwise, return the natively assigned value - default=F("generic_org_type"), - output_field=CharField(), - ), - "converted_federal_agency": Case( - # When portfolio is present, use its value instead - When( - Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), - then=F("portfolio__federal_agency__agency"), - ), - # Otherwise, return the natively assigned value - default=F("federal_agency__agency"), - output_field=CharField(), - ), - "converted_federal_type": Case( - # When portfolio is present, use its value instead - # NOTE: this is an @Property funciton in portfolio. - When( - Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), - then=F("portfolio__federal_agency__federal_type"), - ), - # Otherwise, return the natively assigned value - default=F("federal_type"), - output_field=CharField(), - ), - "converted_organization_name": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__organization_name")), - # Otherwise, return the natively assigned value - default=F("organization_name"), - output_field=CharField(), - ), - "converted_city": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__city")), - # Otherwise, return the natively assigned value - default=F("city"), - output_field=CharField(), - ), - "converted_state_territory": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__state_territory")), - # Otherwise, return the natively assigned value - default=F("state_territory"), - output_field=CharField(), - ), - "converted_so_email": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__email")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__email"), - output_field=CharField(), - ), - "converted_senior_official_last_name": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__last_name")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__last_name"), - output_field=CharField(), - ), - "converted_senior_official_first_name": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__first_name")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__first_name"), - output_field=CharField(), - ), - "converted_senior_official_title": Case( - # When portfolio is present, use its value instead - When(portfolio__isnull=False, then=F("portfolio__senior_official__title")), - # Otherwise, return the natively assigned senior official - default=F("senior_official__title"), - output_field=CharField(), - ), - "converted_so_name": Case( - # When portfolio is present, use that senior official instead - When( - Q(portfolio__isnull=False) & Q(portfolio__senior_official__isnull=False), - then=Concat( - Coalesce(F("portfolio__senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("portfolio__senior_official__last_name"), Value("")), - output_field=CharField(), - ), - ), - # Otherwise, return the natively assigned senior official - default=Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - output_field=CharField(), - ), - } - @classmethod def get_sliced_requests(cls, filter_condition): """Get filtered requests counts sliced by org type and election office.""" requests = DomainRequest.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.FEDERAL) - .distinct() - .count() - ) - interstate = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.INTERSTATE) - .distinct() - .count() - ) + federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() state_or_territory = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY) - .distinct() - .count() - ) - tribal = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.TRIBAL) - .distinct() - .count() - ) - county = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.COUNTY) - .distinct() - .count() - ) - city = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.CITY).distinct().count() + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() ) + tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT) - .distinct() - .count() + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() ) school_district = ( - cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT) - .distinct() - .count() + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() ) election_board = requests.filter(is_election_board=True).distinct().count() @@ -1808,11 +1517,11 @@ def parse_row(cls, columns, model): """ # Handle the federal_type field. Defaults to the wrong format. - federal_type = model.get("converted_federal_type") + federal_type = model.get("federal_type") human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None # Handle the org_type field - org_type = model.get("converted_generic_org_type") + org_type = model.get("generic_org_type") or model.get("organization_type") human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None # Handle the status field. Defaults to the wrong format. @@ -1860,19 +1569,19 @@ def parse_row(cls, columns, model): "Other contacts": model.get("all_other_contacts"), "Current websites": model.get("all_current_websites"), # Untouched FK fields - passed into the request dict. - "Federal agency": model.get("converted_federal_agency"), - "SO first name": model.get("converted_senior_official_first_name"), - "SO last name": model.get("converted_senior_official_last_name"), - "SO email": model.get("converted_so_email"), - "SO title/role": model.get("converted_senior_official_title"), + "Federal agency": model.get("federal_agency__agency"), + "SO first name": model.get("senior_official__first_name"), + "SO last name": model.get("senior_official__last_name"), + "SO email": model.get("senior_official__email"), + "SO title/role": model.get("senior_official__title"), "Creator first name": model.get("creator__first_name"), "Creator last name": model.get("creator__last_name"), "Creator email": model.get("creator__email"), "Investigator": model.get("investigator__email"), # Untouched fields - "Organization name": model.get("converted_organization_name"), - "City": model.get("converted_city"), - "State/territory": model.get("converted_state_territory"), + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State/territory": model.get("state_territory"), "Request purpose": model.get("purpose"), "CISA regional representative": model.get("cisa_representative_email"), "Last submitted date": model.get("last_submitted_date"), @@ -2015,34 +1724,24 @@ def get_computed_fields(cls, delimiter=", ", **kwargs): """ Get a dict of computed fields. """ - # Get computed fields from the parent class - computed_fields = super().get_computed_fields() - - # Add additional computed fields - computed_fields.update( - { - "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), - "creator_active_requests_count": cls.get_creator_active_requests_count_query(), - "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), - "all_alternative_domains": StringAgg( - "alternative_domains__website", delimiter=delimiter, distinct=True - ), - # Coerce the other contacts object to "{first_name} {last_name} {email}" - "all_other_contacts": StringAgg( - Concat( - "other_contacts__first_name", - Value(" "), - "other_contacts__last_name", - Value(" "), - "other_contacts__email", - ), - delimiter=delimiter, - distinct=True, + return { + "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), + "creator_active_requests_count": cls.get_creator_active_requests_count_query(), + "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), + "all_alternative_domains": StringAgg("alternative_domains__website", delimiter=delimiter, distinct=True), + # Coerce the other contacts object to "{first_name} {last_name} {email}" + "all_other_contacts": StringAgg( + Concat( + "other_contacts__first_name", + Value(" "), + "other_contacts__last_name", + Value(" "), + "other_contacts__email", ), - } - ) - - return computed_fields + delimiter=delimiter, + distinct=True, + ), + } @classmethod def get_related_table_fields(cls): From 57c7c6f709051d41ed392124d732d670d06257fb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:56:29 -0700 Subject: [PATCH 02/14] Revert "revert csv-export" This reverts commit b0d1bc26da85e0e6ac8bae4746d529089a6b61de. --- src/registrar/tests/test_reports.py | 138 ++++---- src/registrar/utility/csv_export.py | 519 ++++++++++++++++++++++------ 2 files changed, 476 insertions(+), 181 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index f91c5b299..cafaff7b1 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -71,8 +71,8 @@ def test_generate_federal_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), + call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), ] @@ -93,8 +93,8 @@ def test_generate_full_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), + call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("zdomain12.gov,Interstate,,,,,(blank)\r\n"), @@ -251,32 +251,35 @@ def test_domain_data_type(self): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," - "SO email,Security contact email,Domain managers,Invited domain managers\n" - "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,,(blank),,," + "Domain name,Status,First ready on,Expiration date,Domain type,Agency," + "Organization name,City,State,SO,SO email," + "Security contact email,Domain managers,Invited domain managers\n" + "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive," + "Portfolio 1 Federal Agency,,,, ,,(blank)," + "meoward@rocks.com,squeaker@rocks.com\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive," + "Portfolio 1 Federal Agency,,,, ,,(blank)," + '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive," + "World War I Centennial Commission,,,, ,,(blank)," "meoward@rocks.com,\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," - ',,,(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' - "woofwardthethird@rocks.com\n" - "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,," + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),," "squeaker@rocks.com\n" - "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,,,," - "security@mail.gov,,\n" - "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,,(blank),,," - "meoward@rocks.com,squeaker@rocks.com\n" - "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,,(blank),,,meoward@rocks.com,\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal," + "Armed Forces Retirement Home,,,, ,,security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,(blank),meoward@rocks.com,\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -312,20 +315,17 @@ def test_domain_data_type_user(self): # We expect only domains associated with the user expected_content = ( "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name," - "City,State,SO,SO email," - "Security contact email,Domain managers,Invited domain managers\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,,, ,," - '(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' - "woofwardthethird@rocks.com\n" - "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,(blank)," + "City,State,SO,SO email,Security contact email,Domain managers,Invited domain managers\n" + "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," '"info@example.com, meoward@rocks.com",squeaker@rocks.com\n' + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," + '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -493,17 +493,17 @@ def test_domain_data_full(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" + "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" + "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" "zdomain12.gov,Interstate,,,,,(blank)\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -533,16 +533,16 @@ def test_domain_data_federal(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" + "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" + "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -587,13 +587,13 @@ def test_domain_growth(self): expected_content = ( "Domain name,Domain type,Agency,Organization name,City," "State,Status,Expiration date, Deleted\n" - "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" - "zdomain12.govInterstateReady(blank)\n" + "cdomain1.gov,Federal-Executive,Portfolio1FederalAgency,Ready,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,Ready,(blank)\n" + "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank)\n" + "zdomain12.gov,Interstate,Ready,(blank)\n" "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" - "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" - "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "sdomain8.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace @@ -611,7 +611,6 @@ def test_domain_managed(self): squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). She should show twice in this report but not in test_DomainManaged.""" - self.maxDiff = None # Create a CSV file in memory csv_file = StringIO() # Call the export functions @@ -646,7 +645,6 @@ def test_domain_managed(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -683,7 +681,6 @@ def test_domain_unmanaged(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -721,10 +718,9 @@ def test_domain_request_growth(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - @less_console_noise_decorator + # @less_console_noise_decorator def test_domain_request_data_full(self): """Tests the full domain request report.""" # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data @@ -766,35 +762,34 @@ def test_domain_request_data_full(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() + expected_content = ( # Header - "Domain request,Status,Domain type,Federal type," - "Federal agency,Organization name,Election office,City,State/territory," - "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,SO first name,SO last name,SO email," - "SO title/role,Request purpose,Request additional details,Other contacts," + "Domain request,Status,Domain type,Federal type,Federal agency,Organization name,Election office," + "City,State/territory,Region,Creator first name,Creator last name,Creator email," + "Creator approved domains count,Creator active requests count,Alternative domains,SO first name," + "SO last name,SO email,SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content - "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com," + "city5.gov,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' - 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' - "CISA-last-name " - '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' - 'testy2@town.com"' - ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' - "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " - "testy2@town.com" - ",cisaRep@igorville.gov,city.com,\n" - "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " - "testy2@town.com," + "city2.gov,In review,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," + "Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + "city3.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1," + '"cheeseville.gov, city1.gov, igorville.gov",,,,,Purpose of the site,CISA-first-name CISA-last-name | ' + 'There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, ' + 'Testy Tester testy2@town.com",' + 'test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy," + "Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," + "Testy Tester testy2@town.com," + "cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," + "Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," "cisaRep@igorville.gov,city.com,\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() @@ -862,7 +857,6 @@ def test_member_export(self): # Create a request and add the user to the request request = self.factory.get("/") request.user = self.user - self.maxDiff = None # Add portfolio to session request = GenericTestHelper._mock_user_request_for_factory(request) request.session["portfolio"] = self.portfolio_1 diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index a03e51de5..97feae20c 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -414,8 +414,11 @@ def get_model_annotation_dict(cls, request=None, **kwargs): ) .values(*shared_columns) ) - - return convert_queryset_to_dict(permissions.union(invitations), is_model=False) + # Adding a order_by increases output predictability. + # Doesn't matter as much for normal use, but makes tests easier. + # We should also just be ordering by default anyway. + members = permissions.union(invitations).order_by("email_display") + return convert_queryset_to_dict(members, is_model=False) @classmethod def get_invited_by_query(cls, object_id_query): @@ -525,6 +528,115 @@ def model(cls): # Return the model class that this export handles return DomainInformation + @classmethod + def get_computed_fields(cls, **kwargs): + """ + Get a dict of computed fields. + """ + # NOTE: These computed fields imitate @Property functions in the Domain model and Portfolio model where needed. + # This is for performance purposes. Since we are working with dictionary values and not + # model objects as we export data, trying to reinstate model objects in order to grab @property + # values negatively impacts performance. Therefore, we will follow best practice and use annotations + return { + "converted_generic_org_type": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + # Otherwise, return the natively assigned value + default=F("generic_org_type"), + output_field=CharField(), + ), + "converted_federal_agency": Case( + # When portfolio is present, use its value instead + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__agency"), + ), + # Otherwise, return the natively assigned value + default=F("federal_agency__agency"), + output_field=CharField(), + ), + "converted_federal_type": Case( + # When portfolio is present, use its value instead + # NOTE: this is an @Property funciton in portfolio. + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__federal_type"), + ), + # Otherwise, return the natively assigned value + default=F("federal_type"), + output_field=CharField(), + ), + "converted_organization_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_name")), + # Otherwise, return the natively assigned value + default=F("organization_name"), + output_field=CharField(), + ), + "converted_city": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__city")), + # Otherwise, return the natively assigned value + default=F("city"), + output_field=CharField(), + ), + "converted_state_territory": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__state_territory")), + # Otherwise, return the natively assigned value + default=F("state_territory"), + output_field=CharField(), + ), + "converted_so_email": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__email")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__email"), + output_field=CharField(), + ), + "converted_senior_official_last_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__last_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__last_name"), + output_field=CharField(), + ), + "converted_senior_official_first_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__first_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__first_name"), + output_field=CharField(), + ), + "converted_senior_official_title": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__title")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__title"), + output_field=CharField(), + ), + "converted_so_name": Case( + # When portfolio is present, use that senior official instead + When( + Q(portfolio__isnull=False) & Q(portfolio__senior_official__isnull=False), + then=Concat( + Coalesce(F("portfolio__senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("portfolio__senior_official__last_name"), Value("")), + output_field=CharField(), + ), + ), + # Otherwise, return the natively assigned senior official + default=Concat( + Coalesce(F("senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("senior_official__last_name"), Value("")), + output_field=CharField(), + ), + output_field=CharField(), + ), + } + @classmethod def update_queryset(cls, queryset, **kwargs): """ @@ -614,10 +726,10 @@ def parse_row(cls, columns, model): if first_ready_on is None: first_ready_on = "(blank)" - # organization_type has generic_org_type AND is_election - domain_org_type = model.get("organization_type") + # organization_type has organization_type AND is_election + domain_org_type = model.get("converted_generic_org_type") human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) - domain_federal_type = model.get("federal_type") + domain_federal_type = model.get("converted_federal_type") human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) domain_type = human_readable_domain_org_type if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: @@ -640,12 +752,12 @@ def parse_row(cls, columns, model): "First ready on": first_ready_on, "Expiration date": expiration_date, "Domain type": domain_type, - "Agency": model.get("federal_agency__agency"), - "Organization name": model.get("organization_name"), - "City": model.get("city"), - "State": model.get("state_territory"), - "SO": model.get("so_name"), - "SO email": model.get("senior_official__email"), + "Agency": model.get("converted_federal_agency"), + "Organization name": model.get("converted_organization_name"), + "City": model.get("converted_city"), + "State": model.get("converted_state_territory"), + "SO": model.get("converted_so_name"), + "SO email": model.get("converted_so_email"), "Security contact email": security_contact_email, "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), @@ -654,8 +766,23 @@ def parse_row(cls, columns, model): } row = [FIELDS.get(column, "") for column in columns] + return row + def get_filtered_domain_infos_by_org(domain_infos_to_filter, org_to_filter_by): + """Returns a list of Domain Requests that has been filtered by the given organization value.""" + + annotated_queryset = domain_infos_to_filter.annotate( + converted_generic_org_type=Case( + # Recreate the logic of the converted_generic_org_type property + # here in annotations + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + default=F("generic_org_type"), + output_field=CharField(), + ) + ) + return annotated_queryset.filter(converted_generic_org_type=org_to_filter_by) + @classmethod def get_sliced_domains(cls, filter_condition): """Get filtered domains counts sliced by org type and election office. @@ -663,23 +790,51 @@ def get_sliced_domains(cls, filter_condition): when a domain has more that one manager. """ - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() + domain_informations = DomainInformation.objects.all().filter(**filter_condition).distinct() + domains_count = domain_informations.count() + federal = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.FEDERAL) + .distinct() + .count() + ) + interstate = cls.get_filtered_domain_infos_by_org( + domain_informations, DomainRequest.OrganizationChoices.INTERSTATE + ).count() state_or_territory = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + cls.get_filtered_domain_infos_by_org( + domain_informations, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY + ) + .distinct() + .count() + ) + tribal = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.TRIBAL) + .distinct() + .count() + ) + county = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.COUNTY) + .distinct() + .count() + ) + city = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.CITY) + .distinct() + .count() ) - tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + cls.get_filtered_domain_infos_by_org( + domain_informations, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT + ) + .distinct() + .count() ) school_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT) + .distinct() + .count() ) - election_board = domains.filter(is_election_board=True).distinct().count() + election_board = domain_informations.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -706,6 +861,7 @@ def get_columns(cls): """ Overrides the columns for CSV export specific to DomainExport. """ + return [ "Domain name", "Status", @@ -723,6 +879,13 @@ def get_columns(cls): "Invited domain managers", ] + @classmethod + def get_annotations_for_sort(cls): + """ + Get a dict of annotations to make available for sorting. + """ + return cls.get_computed_fields() + @classmethod def get_sort_fields(cls): """ @@ -730,9 +893,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", + "converted_generic_org_type", + Coalesce("converted_federal_type", Value("ZZZZZ")), + "converted_federal_agency", "domain__name", ] @@ -773,20 +936,6 @@ def get_prefetch_related(cls): """ return ["domain__permissions"] - @classmethod - def get_computed_fields(cls, delimiter=", ", **kwargs): - """ - Get a dict of computed fields. - """ - return { - "so_name": Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - } - @classmethod def get_related_table_fields(cls): """ @@ -892,7 +1041,7 @@ def exporting_dr_data_to_csv(cls, response, request=None): cls.safe_get(getattr(request, "region_field", None)), request.status, cls.safe_get(getattr(request, "election_office", None)), - request.federal_type, + request.converted_federal_type, cls.safe_get(getattr(request, "domain_type", None)), cls.safe_get(getattr(request, "additional_details", None)), cls.safe_get(getattr(request, "creator_approved_domains_count", None)), @@ -943,6 +1092,13 @@ def get_columns(cls): "Security contact email", ] + @classmethod + def get_annotations_for_sort(cls, delimiter=", "): + """ + Get a dict of annotations to make available for sorting. + """ + return cls.get_computed_fields() + @classmethod def get_sort_fields(cls): """ @@ -950,9 +1106,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", + "converted_generic_org_type", + Coalesce("converted_federal_type", Value("ZZZZZ")), + "converted_federal_agency", "domain__name", ] @@ -990,20 +1146,6 @@ def get_filter_conditions(cls, **kwargs): ], ) - @classmethod - def get_computed_fields(cls, delimiter=", ", **kwargs): - """ - Get a dict of computed fields. - """ - return { - "so_name": Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - } - @classmethod def get_related_table_fields(cls): """ @@ -1037,6 +1179,13 @@ def get_columns(cls): "Security contact email", ] + @classmethod + def get_annotations_for_sort(cls, delimiter=", "): + """ + Get a dict of annotations to make available for sorting. + """ + return cls.get_computed_fields() + @classmethod def get_sort_fields(cls): """ @@ -1044,9 +1193,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", + "converted_generic_org_type", + Coalesce("converted_federal_type", Value("ZZZZZ")), + "converted_federal_agency", "domain__name", ] @@ -1085,20 +1234,6 @@ def get_filter_conditions(cls, **kwargs): ], ) - @classmethod - def get_computed_fields(cls, delimiter=", ", **kwargs): - """ - Get a dict of computed fields. - """ - return { - "so_name": Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - } - @classmethod def get_related_table_fields(cls): """ @@ -1476,24 +1611,180 @@ def model(cls): # Return the model class that this export handles return DomainRequest + def get_filtered_domain_requests_by_org(domain_requests_to_filter, org_to_filter_by): + """Returns a list of Domain Requests that has been filtered by the given organization value""" + annotated_queryset = domain_requests_to_filter.annotate( + converted_generic_org_type=Case( + # Recreate the logic of the converted_generic_org_type property + # here in annotations + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + default=F("generic_org_type"), + output_field=CharField(), + ) + ) + return annotated_queryset.filter(converted_generic_org_type=org_to_filter_by) + + # return domain_requests_to_filter.filter( + # # Filter based on the generic org value returned by converted_generic_org_type + # id__in=[ + # domainRequest.id + # for domainRequest in domain_requests_to_filter + # if domainRequest.converted_generic_org_type + # and domainRequest.converted_generic_org_type == org_to_filter_by + # ] + # ) + + @classmethod + def get_computed_fields(cls, delimiter=", ", **kwargs): + """ + Get a dict of computed fields. + """ + # NOTE: These computed fields imitate @Property functions in the Domain model and Portfolio model where needed. + # This is for performance purposes. Since we are working with dictionary values and not + # model objects as we export data, trying to reinstate model objects in order to grab @property + # values negatively impacts performance. Therefore, we will follow best practice and use annotations + return { + "converted_generic_org_type": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + # Otherwise, return the natively assigned value + default=F("generic_org_type"), + output_field=CharField(), + ), + "converted_federal_agency": Case( + # When portfolio is present, use its value instead + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__agency"), + ), + # Otherwise, return the natively assigned value + default=F("federal_agency__agency"), + output_field=CharField(), + ), + "converted_federal_type": Case( + # When portfolio is present, use its value instead + # NOTE: this is an @Property funciton in portfolio. + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__federal_type"), + ), + # Otherwise, return the natively assigned value + default=F("federal_type"), + output_field=CharField(), + ), + "converted_organization_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_name")), + # Otherwise, return the natively assigned value + default=F("organization_name"), + output_field=CharField(), + ), + "converted_city": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__city")), + # Otherwise, return the natively assigned value + default=F("city"), + output_field=CharField(), + ), + "converted_state_territory": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__state_territory")), + # Otherwise, return the natively assigned value + default=F("state_territory"), + output_field=CharField(), + ), + "converted_so_email": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__email")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__email"), + output_field=CharField(), + ), + "converted_senior_official_last_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__last_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__last_name"), + output_field=CharField(), + ), + "converted_senior_official_first_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__first_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__first_name"), + output_field=CharField(), + ), + "converted_senior_official_title": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__title")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__title"), + output_field=CharField(), + ), + "converted_so_name": Case( + # When portfolio is present, use that senior official instead + When( + Q(portfolio__isnull=False) & Q(portfolio__senior_official__isnull=False), + then=Concat( + Coalesce(F("portfolio__senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("portfolio__senior_official__last_name"), Value("")), + output_field=CharField(), + ), + ), + # Otherwise, return the natively assigned senior official + default=Concat( + Coalesce(F("senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("senior_official__last_name"), Value("")), + output_field=CharField(), + ), + output_field=CharField(), + ), + } + @classmethod def get_sliced_requests(cls, filter_condition): """Get filtered requests counts sliced by org type and election office.""" requests = DomainRequest.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() + federal = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.FEDERAL) + .distinct() + .count() + ) + interstate = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.INTERSTATE) + .distinct() + .count() + ) state_or_territory = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY) + .distinct() + .count() + ) + tribal = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.TRIBAL) + .distinct() + .count() + ) + county = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.COUNTY) + .distinct() + .count() + ) + city = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.CITY).distinct().count() ) - tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT) + .distinct() + .count() ) school_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT) + .distinct() + .count() ) election_board = requests.filter(is_election_board=True).distinct().count() @@ -1517,11 +1808,11 @@ def parse_row(cls, columns, model): """ # Handle the federal_type field. Defaults to the wrong format. - federal_type = model.get("federal_type") + federal_type = model.get("converted_federal_type") human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None # Handle the org_type field - org_type = model.get("generic_org_type") or model.get("organization_type") + org_type = model.get("converted_generic_org_type") human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None # Handle the status field. Defaults to the wrong format. @@ -1569,19 +1860,19 @@ def parse_row(cls, columns, model): "Other contacts": model.get("all_other_contacts"), "Current websites": model.get("all_current_websites"), # Untouched FK fields - passed into the request dict. - "Federal agency": model.get("federal_agency__agency"), - "SO first name": model.get("senior_official__first_name"), - "SO last name": model.get("senior_official__last_name"), - "SO email": model.get("senior_official__email"), - "SO title/role": model.get("senior_official__title"), + "Federal agency": model.get("converted_federal_agency"), + "SO first name": model.get("converted_senior_official_first_name"), + "SO last name": model.get("converted_senior_official_last_name"), + "SO email": model.get("converted_so_email"), + "SO title/role": model.get("converted_senior_official_title"), "Creator first name": model.get("creator__first_name"), "Creator last name": model.get("creator__last_name"), "Creator email": model.get("creator__email"), "Investigator": model.get("investigator__email"), # Untouched fields - "Organization name": model.get("organization_name"), - "City": model.get("city"), - "State/territory": model.get("state_territory"), + "Organization name": model.get("converted_organization_name"), + "City": model.get("converted_city"), + "State/territory": model.get("converted_state_territory"), "Request purpose": model.get("purpose"), "CISA regional representative": model.get("cisa_representative_email"), "Last submitted date": model.get("last_submitted_date"), @@ -1724,24 +2015,34 @@ def get_computed_fields(cls, delimiter=", ", **kwargs): """ Get a dict of computed fields. """ - return { - "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), - "creator_active_requests_count": cls.get_creator_active_requests_count_query(), - "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), - "all_alternative_domains": StringAgg("alternative_domains__website", delimiter=delimiter, distinct=True), - # Coerce the other contacts object to "{first_name} {last_name} {email}" - "all_other_contacts": StringAgg( - Concat( - "other_contacts__first_name", - Value(" "), - "other_contacts__last_name", - Value(" "), - "other_contacts__email", + # Get computed fields from the parent class + computed_fields = super().get_computed_fields() + + # Add additional computed fields + computed_fields.update( + { + "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), + "creator_active_requests_count": cls.get_creator_active_requests_count_query(), + "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), + "all_alternative_domains": StringAgg( + "alternative_domains__website", delimiter=delimiter, distinct=True ), - delimiter=delimiter, - distinct=True, - ), - } + # Coerce the other contacts object to "{first_name} {last_name} {email}" + "all_other_contacts": StringAgg( + Concat( + "other_contacts__first_name", + Value(" "), + "other_contacts__last_name", + Value(" "), + "other_contacts__email", + ), + delimiter=delimiter, + distinct=True, + ), + } + ) + + return computed_fields @classmethod def get_related_table_fields(cls): From 8bfeedf6ef43eb1d8df216834a9f0da9804554d1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:09:09 -0700 Subject: [PATCH 03/14] remove converted fields --- src/registrar/utility/csv_export.py | 122 ++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 97feae20c..310bfd8c3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1077,6 +1077,67 @@ class DomainDataFull(DomainExport): Inherits from BaseExport -> DomainExport """ + # NOTE - this override is temporary. Delete this after we consolidate these @property fields. + @classmethod + def parse_row(cls, columns, model): + """ + Given a set of columns and a model dictionary, generate a new row from cleaned column data. + """ + + status = model.get("domain__state") + human_readable_status = Domain.State.get_state_label(status) + + expiration_date = model.get("domain__expiration_date") + if expiration_date is None: + expiration_date = "(blank)" + + first_ready_on = model.get("domain__first_ready") + if first_ready_on is None: + first_ready_on = "(blank)" + + # organization_type has organization_type AND is_election + domain_org_type = model.get("converted_generic_org_type") + human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) + domain_federal_type = model.get("converted_federal_type") + human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) + domain_type = human_readable_domain_org_type + if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: + domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" + + security_contact_email = model.get("security_contact_email") + invalid_emails = {DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value} + if ( + not security_contact_email + or not isinstance(security_contact_email, str) + or security_contact_email.lower().strip() in invalid_emails + ): + security_contact_email = "(blank)" + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + "Domain name": model.get("domain__name"), + "Status": human_readable_status, + "First ready on": first_ready_on, + "Expiration date": expiration_date, + "Domain type": domain_type, + "Agency": model.get("federal_agency"), + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State": model.get("state_territory"), + "SO": model.get("so_name"), + "SO email": model.get("so_email"), + "Security contact email": security_contact_email, + "Created at": model.get("domain__created_at"), + "Deleted": model.get("domain__deleted"), + "Domain managers": model.get("managers"), + "Invited domain managers": model.get("invited_users"), + } + + row = [FIELDS.get(column, "") for column in columns] + + return row + @classmethod def get_columns(cls): """ @@ -1164,6 +1225,67 @@ class DomainDataFederal(DomainExport): Inherits from BaseExport -> DomainExport """ + # NOTE - this override is temporary. Delete this after we consolidate these @property fields. + @classmethod + def parse_row(cls, columns, model): + """ + Given a set of columns and a model dictionary, generate a new row from cleaned column data. + """ + + status = model.get("domain__state") + human_readable_status = Domain.State.get_state_label(status) + + expiration_date = model.get("domain__expiration_date") + if expiration_date is None: + expiration_date = "(blank)" + + first_ready_on = model.get("domain__first_ready") + if first_ready_on is None: + first_ready_on = "(blank)" + + # organization_type has organization_type AND is_election + domain_org_type = model.get("converted_generic_org_type") + human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) + domain_federal_type = model.get("converted_federal_type") + human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) + domain_type = human_readable_domain_org_type + if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: + domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" + + security_contact_email = model.get("security_contact_email") + invalid_emails = {DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value} + if ( + not security_contact_email + or not isinstance(security_contact_email, str) + or security_contact_email.lower().strip() in invalid_emails + ): + security_contact_email = "(blank)" + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + "Domain name": model.get("domain__name"), + "Status": human_readable_status, + "First ready on": first_ready_on, + "Expiration date": expiration_date, + "Domain type": domain_type, + "Agency": model.get("federal_agency"), + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State": model.get("state_territory"), + "SO": model.get("so_name"), + "SO email": model.get("so_email"), + "Security contact email": security_contact_email, + "Created at": model.get("domain__created_at"), + "Deleted": model.get("domain__deleted"), + "Domain managers": model.get("managers"), + "Invited domain managers": model.get("invited_users"), + } + + row = [FIELDS.get(column, "") for column in columns] + + return row + @classmethod def get_columns(cls): """ From 0cd504eb781bf47160f4188b77e7e6196246431d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:16:17 -0700 Subject: [PATCH 04/14] Update csv_export.py --- src/registrar/utility/csv_export.py | 133 +++++++--------------------- 1 file changed, 34 insertions(+), 99 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 310bfd8c3..07014f185 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -744,30 +744,41 @@ def parse_row(cls, columns, model): ): security_contact_email = "(blank)" + model["status"] = human_readable_status + model["first_ready_on"] = first_ready_on + model["expiration_date"] = expiration_date + model["domain_type"] = domain_type + model["security_contact_email"] = security_contact_email # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = cls.get_fields(model) + + row = [FIELDS.get(column, "") for column in columns] + + return row + + # NOTE - this override is temporary. Delete this after we consolidate these @property fields. + @classmethod + def get_fields(cls, model): FIELDS = { "Domain name": model.get("domain__name"), - "Status": human_readable_status, - "First ready on": first_ready_on, - "Expiration date": expiration_date, - "Domain type": domain_type, + "Status": model.get("status"), + "First ready on": model.get("first_ready_on"), + "Expiration date": model.get("expiration_date"), + "Domain type": model.get("domain_type"), "Agency": model.get("converted_federal_agency"), "Organization name": model.get("converted_organization_name"), "City": model.get("converted_city"), "State": model.get("converted_state_territory"), "SO": model.get("converted_so_name"), "SO email": model.get("converted_so_email"), - "Security contact email": security_contact_email, + "Security contact email": model.get("security_contact_email"), "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), "Domain managers": model.get("managers"), "Invited domain managers": model.get("invited_users"), } - - row = [FIELDS.get(column, "") for column in columns] - - return row + return FIELDS def get_filtered_domain_infos_by_org(domain_infos_to_filter, org_to_filter_by): """Returns a list of Domain Requests that has been filtered by the given organization value.""" @@ -1079,64 +1090,26 @@ class DomainDataFull(DomainExport): # NOTE - this override is temporary. Delete this after we consolidate these @property fields. @classmethod - def parse_row(cls, columns, model): - """ - Given a set of columns and a model dictionary, generate a new row from cleaned column data. - """ - - status = model.get("domain__state") - human_readable_status = Domain.State.get_state_label(status) - - expiration_date = model.get("domain__expiration_date") - if expiration_date is None: - expiration_date = "(blank)" - - first_ready_on = model.get("domain__first_ready") - if first_ready_on is None: - first_ready_on = "(blank)" - - # organization_type has organization_type AND is_election - domain_org_type = model.get("converted_generic_org_type") - human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) - domain_federal_type = model.get("converted_federal_type") - human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) - domain_type = human_readable_domain_org_type - if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - - security_contact_email = model.get("security_contact_email") - invalid_emails = {DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value} - if ( - not security_contact_email - or not isinstance(security_contact_email, str) - or security_contact_email.lower().strip() in invalid_emails - ): - security_contact_email = "(blank)" - - # create a dictionary of fields which can be included in output. - # "extra_fields" are precomputed fields (generated in the DB or parsed). + def get_fields(cls, model): FIELDS = { "Domain name": model.get("domain__name"), - "Status": human_readable_status, - "First ready on": first_ready_on, - "Expiration date": expiration_date, - "Domain type": domain_type, + "Status": model.get("status"), + "First ready on": model.get("first_ready_on"), + "Expiration date": model.get("expiration_date"), + "Domain type": model.get("domain_type"), "Agency": model.get("federal_agency"), "Organization name": model.get("organization_name"), "City": model.get("city"), "State": model.get("state_territory"), "SO": model.get("so_name"), "SO email": model.get("so_email"), - "Security contact email": security_contact_email, + "Security contact email": model.get("security_contact_email"), "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), "Domain managers": model.get("managers"), "Invited domain managers": model.get("invited_users"), } - - row = [FIELDS.get(column, "") for column in columns] - - return row + return FIELDS @classmethod def get_columns(cls): @@ -1227,64 +1200,26 @@ class DomainDataFederal(DomainExport): # NOTE - this override is temporary. Delete this after we consolidate these @property fields. @classmethod - def parse_row(cls, columns, model): - """ - Given a set of columns and a model dictionary, generate a new row from cleaned column data. - """ - - status = model.get("domain__state") - human_readable_status = Domain.State.get_state_label(status) - - expiration_date = model.get("domain__expiration_date") - if expiration_date is None: - expiration_date = "(blank)" - - first_ready_on = model.get("domain__first_ready") - if first_ready_on is None: - first_ready_on = "(blank)" - - # organization_type has organization_type AND is_election - domain_org_type = model.get("converted_generic_org_type") - human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) - domain_federal_type = model.get("converted_federal_type") - human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) - domain_type = human_readable_domain_org_type - if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - - security_contact_email = model.get("security_contact_email") - invalid_emails = {DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value} - if ( - not security_contact_email - or not isinstance(security_contact_email, str) - or security_contact_email.lower().strip() in invalid_emails - ): - security_contact_email = "(blank)" - - # create a dictionary of fields which can be included in output. - # "extra_fields" are precomputed fields (generated in the DB or parsed). + def get_fields(cls, model): FIELDS = { "Domain name": model.get("domain__name"), - "Status": human_readable_status, - "First ready on": first_ready_on, - "Expiration date": expiration_date, - "Domain type": domain_type, + "Status": model.get("status"), + "First ready on": model.get("first_ready_on"), + "Expiration date": model.get("expiration_date"), + "Domain type": model.get("domain_type"), "Agency": model.get("federal_agency"), "Organization name": model.get("organization_name"), "City": model.get("city"), "State": model.get("state_territory"), "SO": model.get("so_name"), "SO email": model.get("so_email"), - "Security contact email": security_contact_email, + "Security contact email": model.get("security_contact_email"), "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), "Domain managers": model.get("managers"), "Invited domain managers": model.get("invited_users"), } - - row = [FIELDS.get(column, "") for column in columns] - - return row + return FIELDS @classmethod def get_columns(cls): From 26fd19ffe80e15c7381ed52802be42da02075880 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:21:06 -0700 Subject: [PATCH 05/14] Update test_reports.py --- src/registrar/tests/test_reports.py | 138 +++++++++++++++------------- 1 file changed, 72 insertions(+), 66 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index cafaff7b1..f91c5b299 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -71,8 +71,8 @@ def test_generate_federal_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), ] @@ -93,8 +93,8 @@ def test_generate_full_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("zdomain12.gov,Interstate,,,,,(blank)\r\n"), @@ -251,35 +251,32 @@ def test_domain_data_type(self): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Status,First ready on,Expiration date,Domain type,Agency," - "Organization name,City,State,SO,SO email," - "Security contact email,Domain managers,Invited domain managers\n" - "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive," - "Portfolio 1 Federal Agency,,,, ,,(blank)," - "meoward@rocks.com,squeaker@rocks.com\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive," - "Portfolio 1 Federal Agency,,,, ,,(blank)," - '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' - "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive," - "World War I Centennial Commission,,,, ,,(blank)," + "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," + "SO email,Security contact email,Domain managers,Invited domain managers\n" + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,,(blank),,," "meoward@rocks.com,\n" - "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),," + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," + ',,,(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' + "woofwardthethird@rocks.com\n" + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,," "squeaker@rocks.com\n" - "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "ddomain3.gov,On hold,(blank),2023-11-15,Federal," - "Armed Forces Retirement Home,,,, ,,security@mail.gov,,\n" - "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" - "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,(blank),meoward@rocks.com,\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,,,," + "security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,,(blank),,," + "meoward@rocks.com,squeaker@rocks.com\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,,(blank),,,meoward@rocks.com,\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -315,17 +312,20 @@ def test_domain_data_type_user(self): # We expect only domains associated with the user expected_content = ( "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name," - "City,State,SO,SO email,Security contact email,Domain managers,Invited domain managers\n" - "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," + "City,State,SO,SO email," + "Security contact email,Domain managers,Invited domain managers\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,,, ,," + '(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' + "woofwardthethird@rocks.com\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,(blank)," '"info@example.com, meoward@rocks.com",squeaker@rocks.com\n' - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," - '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -493,17 +493,17 @@ def test_domain_data_full(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" - "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" - "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" "zdomain12.gov,Interstate,,,,,(blank)\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -533,16 +533,16 @@ def test_domain_data_federal(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" - "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" - "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -587,13 +587,13 @@ def test_domain_growth(self): expected_content = ( "Domain name,Domain type,Agency,Organization name,City," "State,Status,Expiration date, Deleted\n" - "cdomain1.gov,Federal-Executive,Portfolio1FederalAgency,Ready,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,Ready,(blank)\n" - "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank)\n" - "zdomain12.gov,Interstate,Ready,(blank)\n" + "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" + "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" + "zdomain12.govInterstateReady(blank)\n" "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" - "sdomain8.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" - "xdomain7.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace @@ -611,6 +611,7 @@ def test_domain_managed(self): squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). She should show twice in this report but not in test_DomainManaged.""" + self.maxDiff = None # Create a CSV file in memory csv_file = StringIO() # Call the export functions @@ -645,6 +646,7 @@ def test_domain_managed(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -681,6 +683,7 @@ def test_domain_unmanaged(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -718,9 +721,10 @@ def test_domain_request_growth(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # @less_console_noise_decorator + @less_console_noise_decorator def test_domain_request_data_full(self): """Tests the full domain request report.""" # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data @@ -762,34 +766,35 @@ def test_domain_request_data_full(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - expected_content = ( # Header - "Domain request,Status,Domain type,Federal type,Federal agency,Organization name,Election office," - "City,State/territory,Region,Creator first name,Creator last name,Creator email," - "Creator approved domains count,Creator active requests count,Alternative domains,SO first name," - "SO last name,SO email,SO title/role,Request purpose,Request additional details,Other contacts," + "Domain request,Status,Domain type,Federal type," + "Federal agency,Organization name,Election office,City,State/territory," + "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," + "Creator active requests count,Alternative domains,SO first name,SO last name,SO email," + "SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content - "city5.gov,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," + "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city2.gov,In review,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," - "Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city3.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1," - '"cheeseville.gov, city1.gov, igorville.gov",,,,,Purpose of the site,CISA-first-name CISA-last-name | ' - 'There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, ' - 'Testy Tester testy2@town.com",' - 'test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' - "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy," - "Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com," - "cisaRep@igorville.gov,city.com,\n" - "city6.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," - "Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," + "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," + "testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' + 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' + "CISA-last-name " + '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' + 'testy2@town.com"' + ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com" + ",cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com," "cisaRep@igorville.gov,city.com,\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() @@ -857,6 +862,7 @@ def test_member_export(self): # Create a request and add the user to the request request = self.factory.get("/") request.user = self.user + self.maxDiff = None # Add portfolio to session request = GenericTestHelper._mock_user_request_for_factory(request) request.session["portfolio"] = self.portfolio_1 From 431b0e80cf316558c3bd1eea72aba142ae379f5b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:21:52 -0700 Subject: [PATCH 06/14] Revert "Update test_reports.py" This reverts commit 26fd19ffe80e15c7381ed52802be42da02075880. --- src/registrar/tests/test_reports.py | 138 +++++++++++++--------------- 1 file changed, 66 insertions(+), 72 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index f91c5b299..cafaff7b1 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -71,8 +71,8 @@ def test_generate_federal_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), + call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), ] @@ -93,8 +93,8 @@ def test_generate_full_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), + call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("zdomain12.gov,Interstate,,,,,(blank)\r\n"), @@ -251,32 +251,35 @@ def test_domain_data_type(self): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," - "SO email,Security contact email,Domain managers,Invited domain managers\n" - "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,,(blank),,," + "Domain name,Status,First ready on,Expiration date,Domain type,Agency," + "Organization name,City,State,SO,SO email," + "Security contact email,Domain managers,Invited domain managers\n" + "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive," + "Portfolio 1 Federal Agency,,,, ,,(blank)," + "meoward@rocks.com,squeaker@rocks.com\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive," + "Portfolio 1 Federal Agency,,,, ,,(blank)," + '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive," + "World War I Centennial Commission,,,, ,,(blank)," "meoward@rocks.com,\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," - ',,,(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' - "woofwardthethird@rocks.com\n" - "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,," + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),," "squeaker@rocks.com\n" - "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,,,," - "security@mail.gov,,\n" - "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,,(blank),,," - "meoward@rocks.com,squeaker@rocks.com\n" - "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,,(blank),,,meoward@rocks.com,\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal," + "Armed Forces Retirement Home,,,, ,,security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,(blank),meoward@rocks.com,\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -312,20 +315,17 @@ def test_domain_data_type_user(self): # We expect only domains associated with the user expected_content = ( "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name," - "City,State,SO,SO email," - "Security contact email,Domain managers,Invited domain managers\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,,, ,," - '(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' - "woofwardthethird@rocks.com\n" - "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,(blank)," + "City,State,SO,SO email,Security contact email,Domain managers,Invited domain managers\n" + "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," '"info@example.com, meoward@rocks.com",squeaker@rocks.com\n' + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," + '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -493,17 +493,17 @@ def test_domain_data_full(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" + "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" + "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" "zdomain12.gov,Interstate,,,,,(blank)\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -533,16 +533,16 @@ def test_domain_data_federal(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" + "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" + "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -587,13 +587,13 @@ def test_domain_growth(self): expected_content = ( "Domain name,Domain type,Agency,Organization name,City," "State,Status,Expiration date, Deleted\n" - "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" - "zdomain12.govInterstateReady(blank)\n" + "cdomain1.gov,Federal-Executive,Portfolio1FederalAgency,Ready,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,Ready,(blank)\n" + "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank)\n" + "zdomain12.gov,Interstate,Ready,(blank)\n" "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" - "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" - "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "sdomain8.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace @@ -611,7 +611,6 @@ def test_domain_managed(self): squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). She should show twice in this report but not in test_DomainManaged.""" - self.maxDiff = None # Create a CSV file in memory csv_file = StringIO() # Call the export functions @@ -646,7 +645,6 @@ def test_domain_managed(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -683,7 +681,6 @@ def test_domain_unmanaged(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -721,10 +718,9 @@ def test_domain_request_growth(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - @less_console_noise_decorator + # @less_console_noise_decorator def test_domain_request_data_full(self): """Tests the full domain request report.""" # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data @@ -766,35 +762,34 @@ def test_domain_request_data_full(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() + expected_content = ( # Header - "Domain request,Status,Domain type,Federal type," - "Federal agency,Organization name,Election office,City,State/territory," - "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,SO first name,SO last name,SO email," - "SO title/role,Request purpose,Request additional details,Other contacts," + "Domain request,Status,Domain type,Federal type,Federal agency,Organization name,Election office," + "City,State/territory,Region,Creator first name,Creator last name,Creator email," + "Creator approved domains count,Creator active requests count,Alternative domains,SO first name," + "SO last name,SO email,SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content - "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com," + "city5.gov,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' - 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' - "CISA-last-name " - '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' - 'testy2@town.com"' - ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' - "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " - "testy2@town.com" - ",cisaRep@igorville.gov,city.com,\n" - "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " - "testy2@town.com," + "city2.gov,In review,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," + "Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + "city3.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1," + '"cheeseville.gov, city1.gov, igorville.gov",,,,,Purpose of the site,CISA-first-name CISA-last-name | ' + 'There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, ' + 'Testy Tester testy2@town.com",' + 'test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy," + "Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," + "Testy Tester testy2@town.com," + "cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," + "Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," "cisaRep@igorville.gov,city.com,\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() @@ -862,7 +857,6 @@ def test_member_export(self): # Create a request and add the user to the request request = self.factory.get("/") request.user = self.user - self.maxDiff = None # Add portfolio to session request = GenericTestHelper._mock_user_request_for_factory(request) request.session["portfolio"] = self.portfolio_1 From d2d787c8eaf8562c35a791f2f22825ccaaca15a7 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:24:59 -0700 Subject: [PATCH 07/14] Revert relevant tests back to normal --- src/registrar/tests/test_reports.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index cafaff7b1..995782eea 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -71,8 +71,8 @@ def test_generate_federal_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), ] @@ -93,8 +93,8 @@ def test_generate_full_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("zdomain12.gov,Interstate,,,,,(blank)\r\n"), @@ -493,17 +493,17 @@ def test_domain_data_full(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" - "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" - "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" "zdomain12.gov,Interstate,,,,,(blank)\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -533,16 +533,16 @@ def test_domain_data_federal(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" - "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" - "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" - "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" ) - # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator From 8b72c654b8a3fb5c234b03e3a7bec334664dc8f1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:49:28 -0700 Subject: [PATCH 08/14] Update csv_export.py --- src/registrar/utility/csv_export.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 07014f185..93fcaaf84 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1097,12 +1097,12 @@ def get_fields(cls, model): "First ready on": model.get("first_ready_on"), "Expiration date": model.get("expiration_date"), "Domain type": model.get("domain_type"), - "Agency": model.get("federal_agency"), + "Agency": model.get("federal_agency__agency"), "Organization name": model.get("organization_name"), "City": model.get("city"), "State": model.get("state_territory"), "SO": model.get("so_name"), - "SO email": model.get("so_email"), + "SO email": model.get("senior_official__email"), "Security contact email": model.get("security_contact_email"), "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), @@ -1207,12 +1207,12 @@ def get_fields(cls, model): "First ready on": model.get("first_ready_on"), "Expiration date": model.get("expiration_date"), "Domain type": model.get("domain_type"), - "Agency": model.get("federal_agency"), + "Agency": model.get("federal_agency__agency"), "Organization name": model.get("organization_name"), "City": model.get("city"), "State": model.get("state_territory"), "SO": model.get("so_name"), - "SO email": model.get("so_email"), + "SO email": model.get("senior_official__email"), "Security contact email": model.get("security_contact_email"), "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), From fcdc0f0f0fc4f5d319530619260ab9ee5aaf287d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:53:17 -0700 Subject: [PATCH 09/14] Fix different ordering --- src/registrar/utility/csv_export.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 93fcaaf84..4de947594 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1140,9 +1140,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "converted_generic_org_type", - Coalesce("converted_federal_type", Value("ZZZZZ")), - "converted_federal_agency", + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", "domain__name", ] @@ -1250,9 +1250,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "converted_generic_org_type", - Coalesce("converted_federal_type", Value("ZZZZZ")), - "converted_federal_agency", + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", "domain__name", ] From b0cf5df7984800fa1283301f67d61883e58dee1f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:06:47 -0700 Subject: [PATCH 10/14] add zap --- src/zap.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/zap.conf b/src/zap.conf index 65468773a..782eaa0e4 100644 --- a/src/zap.conf +++ b/src/zap.conf @@ -75,6 +75,7 @@ 10038 OUTOFSCOPE http://app:8080/suborganization/ 10038 OUTOFSCOPE http://app:8080/transfer/ 10038 OUTOFSCOPE http://app:8080/prototype-dns +10038 OUTOFSCOPE http://app:8080/suborganization # This URL always returns 404, so include it as well. 10038 OUTOFSCOPE http://app:8080/todo # OIDC isn't configured in the test environment and DEBUG=True so this gives a 500 without CSP headers From 6b2552bdbc3f4ee093dcf9edfa5bd1d428781c30 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:08:05 -0700 Subject: [PATCH 11/14] Revert "add zap" This reverts commit b0cf5df7984800fa1283301f67d61883e58dee1f. --- src/zap.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zap.conf b/src/zap.conf index 782eaa0e4..65468773a 100644 --- a/src/zap.conf +++ b/src/zap.conf @@ -75,7 +75,6 @@ 10038 OUTOFSCOPE http://app:8080/suborganization/ 10038 OUTOFSCOPE http://app:8080/transfer/ 10038 OUTOFSCOPE http://app:8080/prototype-dns -10038 OUTOFSCOPE http://app:8080/suborganization # This URL always returns 404, so include it as well. 10038 OUTOFSCOPE http://app:8080/todo # OIDC isn't configured in the test environment and DEBUG=True so this gives a 500 without CSP headers From 0777cf1334e48ff607ddd5061d7e5b52bda0165d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:10:26 -0700 Subject: [PATCH 12/14] Comment comet --- src/registrar/utility/csv_export.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 4de947594..3b3fe350c 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -757,7 +757,10 @@ def parse_row(cls, columns, model): return row - # NOTE - this override is temporary. Delete this after we consolidate these @property fields. + # NOTE - this override is temporary. + # We are running into a problem where DomainDataFull and DomainDataFederal are + # pulling the portfolio name, rather than the suborganization name. + # This can be removed after that gets fixed. @classmethod def get_fields(cls, model): FIELDS = { @@ -1088,7 +1091,10 @@ class DomainDataFull(DomainExport): Inherits from BaseExport -> DomainExport """ - # NOTE - this override is temporary. Delete this after we consolidate these @property fields. + # NOTE - this override is temporary. + # We are running into a problem where DomainDataFull is + # pulling the portfolio name, rather than the suborganization name. + # This can be removed after that gets fixed. @classmethod def get_fields(cls, model): FIELDS = { @@ -1198,7 +1204,10 @@ class DomainDataFederal(DomainExport): Inherits from BaseExport -> DomainExport """ - # NOTE - this override is temporary. Delete this after we consolidate these @property fields. + # NOTE - this override is temporary. + # We are running into a problem where DomainDataFederal is + # pulling the portfolio name, rather than the suborganization name. + # This can be removed after that gets fixed. @classmethod def get_fields(cls, model): FIELDS = { From e9d0a5425134d020e35aa6b6b8ac32eac70b7915 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:20:25 -0700 Subject: [PATCH 13/14] Update csv_export.py --- src/registrar/utility/csv_export.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 3b3fe350c..40d84e251 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -759,7 +759,8 @@ def parse_row(cls, columns, model): # NOTE - this override is temporary. # We are running into a problem where DomainDataFull and DomainDataFederal are - # pulling the portfolio name, rather than the suborganization name. + # pulling the wrong data. + # For example, the portfolio name, rather than the suborganization name. # This can be removed after that gets fixed. @classmethod def get_fields(cls, model): @@ -1093,7 +1094,8 @@ class DomainDataFull(DomainExport): # NOTE - this override is temporary. # We are running into a problem where DomainDataFull is - # pulling the portfolio name, rather than the suborganization name. + # pulling the wrong data. + # For example, the portfolio name, rather than the suborganization name. # This can be removed after that gets fixed. @classmethod def get_fields(cls, model): @@ -1206,7 +1208,8 @@ class DomainDataFederal(DomainExport): # NOTE - this override is temporary. # We are running into a problem where DomainDataFederal is - # pulling the portfolio name, rather than the suborganization name. + # pulling the wrong data. + # For example, the portfolio name, rather than the suborganization name. # This can be removed after that gets fixed. @classmethod def get_fields(cls, model): From 5f5ca0b780d50a67ce6d76f84f9bdd01c162dcd6 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:25:04 -0700 Subject: [PATCH 14/14] Update csv_export.py --- src/registrar/utility/csv_export.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 40d84e251..66809777b 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1097,6 +1097,12 @@ class DomainDataFull(DomainExport): # pulling the wrong data. # For example, the portfolio name, rather than the suborganization name. # This can be removed after that gets fixed. + # The following fields are changed from DomainExport: + # converted_organization_name => organization_name + # converted_city => city + # converted_state_territory => state_territory + # converted_so_name => so_name + # converted_so_email => senior_official__email @classmethod def get_fields(cls, model): FIELDS = { @@ -1207,10 +1213,16 @@ class DomainDataFederal(DomainExport): """ # NOTE - this override is temporary. - # We are running into a problem where DomainDataFederal is + # We are running into a problem where DomainDataFull is # pulling the wrong data. # For example, the portfolio name, rather than the suborganization name. # This can be removed after that gets fixed. + # The following fields are changed from DomainExport: + # converted_organization_name => organization_name + # converted_city => city + # converted_state_territory => state_territory + # converted_so_name => so_name + # converted_so_email => senior_official__email @classmethod def get_fields(cls, model): FIELDS = {