Merge branch 'main' into za/2927-blocked-from-starting-requests

.github/workflows/clone-staging.yaml

@@ -1,11 +1,9 @@
 name: Clone Staging Database
 
 on:
-  # these will be uncommented after testing
-  # ----
-  # schedule:
-  #   # Run daily at 2:00 PM EST
-  #   - cron: '0 * * * *'
+  schedule:
+    # Run daily at 2:00 PM EST
+    - cron: '0 * * * *'
   # Allow manual triggering
   workflow_dispatch:
 
@@ -16,7 +14,7 @@ env:
 
 jobs:
   clone-database:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     env:
       CF_USERNAME: ${{ secrets.CF_MS_USERNAME }}
       CF_PASSWORD: ${{ secrets.CF_MS_PASSWORD }}
@@ -26,9 +24,10 @@ jobs:
             # install cf cli and other tools
             wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo gpg --dearmor -o /usr/share/keyrings/cli.cloudfoundry.org.gpg
             echo "deb [signed-by=/usr/share/keyrings/cli.cloudfoundry.org.gpg] https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
-            
+
             sudo apt-get update
-            sudo apt-get install cf8-cli postgresql-client-common
+            sudo apt-get install cf8-cli
+            
             # install cg-manage-rds tool
             pip install git+https://github.com/cloud-gov/cg-manage-rds.git
             
@@ -41,7 +40,8 @@ jobs:
             cf share-service getgov-$DESTINATION_ENVIRONMENT-database -s $SOURCE_ENVIRONMENT
             
             # clone from source to destination
-            cg-manage-rds clone getgov-$DESTINATION_ENVIRONMENT-database getgov-$SOURCE_ENVIRONMENT-database
-            
-            # unshare the service
-            cf unshare-service getgov-$DESTINATION_ENVIRONMENT-database -s $SOURCE_ENVIRONMENT
+            cf target -s $SOURCE_ENVIRONMENT
+            cg-manage-rds clone getgov-$SOURCE_ENVIRONMENT-database getgov-$DESTINATION_ENVIRONMENT-database
+        - name: Cleanup
+          if: always()
+          run: cf unshare-service getgov-$DESTINATION_ENVIRONMENT-database -s $SOURCE_ENVIRONMENT -f

docs/operations/runbooks/rotate_application_secrets.md

@@ -3,7 +3,7 @@
 
 Secrets are read from the running environment.
 
-Secrets were originally created with:
+Secrets are originally created with:
 
 ```sh
 cf cups getgov-credentials -p credentials-<ENVIRONMENT>.json
@@ -38,6 +38,49 @@ cf restage getgov-stable --strategy rolling
 
 Non-secret environment variables can be declared in `manifest-<ENVIRONMENT>.json` directly.
 
+## Rotating login.gov credentials
+The DJANGO_SECRET_KEY and DJANGO_SECRET_LOGIN_KEY are reset once a year for each sandbox, see their sections below for more information on them and how to manually generate these keys. To save time, complete the following steps to rotate these credentials using a script in non-production environments:
+
+### Step 1 login
+
+To run the script make sure you are logged on the cf cli and make sure you have access to the [Login Partner Dashboard](https://dashboard.int.identitysandbox.gov/service_providers/2640). 
+
+### Step 2 Run the script
+
+Run the following where "ENV" refers to whichever sandbox you want to reset credentials on. Note, the below assumes you are in the root directory of our app.
+
+```bash
+ops/scripts/rotate_login_certs.sh ENV
+```
+
+### Step 3 Respond to the terminal prompts
+
+Respond to the prompts from the script and, when it asks for the cert information, the below is an example of what you should enter. Note for "Common Name" you should put the name of the sandbox and for "Email Address" it should be the address of who owns that sandbox (such as the developer's email, if it's a develop sandbox, or whoever ran this action otherwise)
+
+```bash
+Country Name (2 letter code) [AU]:US
+State or Province Name (full name) [Some-State]:DC
+Locality Name (eg, city) []:DC
+Organization Name (eg, company) [Internet Widgits Pty Ltd]:DHS
+Organizational Unit Name (eg, section) []:CISA
+Common Name (e.g. server FQDN or YOUR name) []:ENV
+Email Address []: [email protected]
+```
+
+Note when this script is done it will have generated a .pem and a .crt file, as well as updated the cert info on the sandbox
+
+### Step 4 Delete the old cert
+
+Navigate to to the Login Partner Dashboard linked above and delete the old cert
+
+### Step 5 add the new cert
+
+In whichever directory you ran the script there should now be a .crt file named "public-ENV.crt", where ENV is the space name you used on Step 2. Upload this cert in the Login Partner Dashboard in the same section where you deleted the old one.
+
+### Production only
+
+This script should not be run in production. Instead, you will need to manually create the keys and then refrain from updating the sandbox. Once the cert is created you will upload it to the Login Partner Dashboard for our production system, and then open a ticket with them to update our existing Login.gov integration. Once they respond back saying it has been applied, you can then update the sandbox.
+
 ## DJANGO_SECRET_KEY
 
 This is a standard Django secret key. See Django documentation for tips on generating a new one. 
@@ -46,6 +89,7 @@ This is a standard Django secret key. See Django documentation for tips on gener
 
 This is the base64 encoded private key used in the OpenID Connect authentication flow with Login.gov. It is used to sign a token during user login; the signature is examined by Login.gov before their API grants access to user data.
 
+### Manually creating creating the Login Key
 Generate a new key using this command (or whatever is most recently [recommended by Login.gov](https://developers.login.gov/testing/#creating-a-public-certificate)):
 
 ```bash
@@ -60,6 +104,8 @@ base64 private.pem
 
 You also need to upload the `public.crt` key if recently created to the login.gov identity sandbox: https://dashboard.int.identitysandbox.gov/
 
+
+
 ## AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
 
 To access the AWS Simple Email Service, we need credentials from the CISA AWS
@@ -76,6 +122,8 @@ These are the client certificate and its private key used to identify the regist
 
 The private key is protected by a passphrase for safer transport and storage.
 
+Note this must be reset once a year.
+
 These were generated with the following steps:
 
 ### Step 1: Generate an unencrypted private key with a named curve
@@ -90,7 +138,7 @@ openssl ecparam -name prime256v1 -genkey -out client_unencrypted.key
 openssl pkcs8 -topk8 -v2 aes-256-cbc -in client_unencrypted.key -out client.key
 ```
 
-### Generate the certificate
+### Step 3: Generate the certificate
 
 ```bash
 openssl req -new -x509 -days 365 -key client.key -out client.crt -subj "/C=US/ST=DC/L=Washington/O=GSA/OU=18F/CN=GOV Prototype Registrar"
@@ -112,7 +160,7 @@ base64 -i client.key
 base64 -i client.crt
 ```
 
-You'll need to give the new certificate to the registry vendor _before_ rotating it in production. Once it has been accepted by the vendor, make sure to update the kdbx file on Google Drive.
+You'll need to give the new certificate to the registry vendor _before_ rotating it in production. Once it has been accepted by the vendor, make sure to update [the KBDX](https://docs.google.com/document/d/1_BbJmjYZNYLNh4jJPPnUEG9tFCzJrOc0nMrZrnSKKyw) file on Google Drive.
 
 ## REGISTRY_HOSTNAME
 

ops/scripts/rotate_login_certs.sh

@@ -0,0 +1,51 @@
+# This script rotates the login.gov credentials, DJANGO_SECRET_KEY and DJANGO_SECRET_LOGIN_KEY that allow for identity sandbox to work on sandboxes and local.
+# The echo prints in this script should serve for documentation for running manually.
+# Run this script once a year for each environment
+# NOTE: This script was written for MacOS and to be run at the root directory. 
+
+
+if [ -z "$1" ]; then
+    echo 'Please specify a space to update (i.e. lmm)' >&2
+    exit 1
+fi
+echo "You need access to the Login partner dashboard, otherwise you will not be able to complete the steps in this script (https://dashboard.int.identitysandbox.gov/service_providers/2640)"
+read -p " Do you have access to the partner dashboard mentioned above? (y/n)  " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    exit 1
+fi
+
+if [ ! $(command -v jq) ] || [ ! $(command -v cf) ]; then
+    echo "jq, and cf packages must be installed. Please install via your preferred manager."
+    exit 1
+fi
+
+cf target -o cisa-dotgov
+
+read -p "Are you logged in to the cisa-dotgov CF org above? (y/n) " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]
+then
+    cf login -a https://api.fr.cloud.gov --sso
+fi
+echo "Targeting space"
+cf target -o cisa-dotgov -s $1
+
+echo "Creating new login.gov credentials for $1..."
+django_key=$(python3 -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())')
+openssl req -noenc -x509 -days 365 -newkey rsa:2048 -keyout private-$1.pem -out public-$1.crt
+login_key=$(base64 -i private-$1.pem)
+
+echo "Creating the final json"
+cf env getgov-$1 | awk '/VCAP_SERVICES: /,/^$/' | sed s/VCAP_SERVICES:// | jq '."user-provided"[0].credentials' | jq --arg django_key "$django_key" --arg login_key "$login_key" '. + {"DJANGO_SECRET_KEY":$django_key, "DJANGO_SECRET_LOGIN_KEY":$login_key}' > credentials-$1.json
+
+echo "Updating creds on the sandbox" 
+cf uups getgov-credentials -p credentials-$1.json
+cf restage getgov-$1 --strategy rolling
+
+echo "\n\n\nNow you will need to update some things for Login. Please sign-in to https://dashboard.int.identitysandbox.gov/."
+echo "Navigate to our application config: https://dashboard.int.identitysandbox.gov/service_providers/2640/edit?"
+echo "There are two things to update."
+echo "1. Remove the old cert associated with the user's email (under Public Certificates)"
+echo "2. You need to upload the public-$1.crt file generated as part of the previous command. See the "choose cert file" button under Public Certificates."
+echo "Then, tell the developer to update their local .env file by retrieving their credentials from the sandbox"

src/registrar/admin.py

@@ -1640,6 +1640,70 @@ class StatusListFilter(MultipleChoiceListFilter):
         def lookups(self, request, model_admin):
             return DomainRequest.DomainRequestStatus.choices
 
+    class GenericOrgFilter(admin.SimpleListFilter):
+        """Custom Generic Organization filter that accomodates portfolio feature.
+        If we have a portfolio, use the portfolio's organization.  If not, use the
+        organization in the Domain Request object."""
+
+        title = "generic organization"
+        parameter_name = "converted_generic_orgs"
+
+        def lookups(self, request, model_admin):
+            converted_generic_orgs = set()
+
+            for domain_request in DomainRequest.objects.all():
+                converted_generic_org = domain_request.converted_generic_org_type
+                if converted_generic_org:
+                    converted_generic_orgs.add(converted_generic_org)
+
+            return sorted((org, org) for org in converted_generic_orgs)
+
+        # Filter queryset
+        def queryset(self, request, queryset):
+            if self.value():  # Check if a generic org is selected in the filter
+                return queryset.filter(
+                    # Filter based on the generic org value returned by converted_generic_org_type
+                    id__in=[
+                        domain_request.id
+                        for domain_request in queryset
+                        if domain_request.converted_generic_org_type
+                        and domain_request.converted_generic_org_type == self.value()
+                    ]
+                )
+            return queryset
+
+    class FederalTypeFilter(admin.SimpleListFilter):
+        """Custom Federal Type filter that accomodates portfolio feature.
+        If we have a portfolio, use the portfolio's federal type.  If not, use the
+        organization in the Domain Request object."""
+
+        title = "federal Type"
+        parameter_name = "converted_federal_types"
+
+        def lookups(self, request, model_admin):
+            converted_federal_types = set()
+
+            for domain_request in DomainRequest.objects.all():
+                converted_federal_type = domain_request.converted_federal_type
+                if converted_federal_type:
+                    converted_federal_types.add(converted_federal_type)
+
+            return sorted((type, type) for type in converted_federal_types)
+
+        # Filter queryset
+        def queryset(self, request, queryset):
+            if self.value():  # Check if federal Type is selected in the filter
+                return queryset.filter(
+                    # Filter based on the federal type returned by converted_federal_type
+                    id__in=[
+                        domain_request.id
+                        for domain_request in queryset
+                        if domain_request.converted_federal_type
+                        and domain_request.converted_federal_type == self.value()
+                    ]
+                )
+            return queryset
+
     class InvestigatorFilter(admin.SimpleListFilter):
         """Custom investigator filter that only displays users with the manager role"""
 
@@ -1700,20 +1764,44 @@ def queryset(self, request, queryset):
             if self.value() == "0":
                 return queryset.filter(Q(is_election_board=False) | Q(is_election_board=None))
 
+    @admin.display(description=_("Generic Org Type"))
+    def converted_generic_org_type(self, obj):
+        return obj.converted_generic_org_type
+
+    @admin.display(description=_("Organization Name"))
+    def converted_organization_name(self, obj):
+        return obj.converted_organization_name
+
+    @admin.display(description=_("Federal Agency"))
+    def converted_federal_agency(self, obj):
+        return obj.converted_federal_agency
+
+    @admin.display(description=_("Federal Type"))
+    def converted_federal_type(self, obj):
+        return obj.converted_federal_type
+
+    @admin.display(description=_("City"))
+    def converted_city(self, obj):
+        return obj.converted_city
+
+    @admin.display(description=_("State/Territory"))
+    def converted_state_territory(self, obj):
+        return obj.converted_state_territory
+
     # Columns
     list_display = [
         "requested_domain",
         "first_submitted_date",
         "last_submitted_date",
         "last_status_update",
         "status",
-        "generic_org_type",
-        "federal_type",
-        "federal_agency",
-        "organization_name",
         "custom_election_board",
-        "city",
-        "state_territory",
+        "converted_generic_org_type",
+        "converted_organization_name",
+        "converted_federal_agency",
+        "converted_federal_type",
+        "converted_city",
+        "converted_state_territory",
         "investigator",
     ]
 
@@ -1738,8 +1826,8 @@ def status_history(self, obj):
     # Filters
     list_filter = (
         StatusListFilter,
-        "generic_org_type",
-        "federal_type",
+        GenericOrgFilter,
+        FederalTypeFilter,
         ElectionOfficeFilter,
         "rejection_reason",
         InvestigatorFilter,
@@ -1869,15 +1957,16 @@ def status_history(self, obj):
         "suborganization_city",
         "suborganization_state_territory",
     ]
+
     autocomplete_fields = [
         "approved_domain",
         "requested_domain",
         "creator",
-        "senior_official",
         "investigator",
         "portfolio",
         "sub_organization",
     ]
+
     filter_horizontal = ("current_websites", "alternative_domains", "other_contacts")
 
     # Table ordering