Merge branch 'master' into master

.annotation_safe_list.yml

@@ -9,13 +9,13 @@
 
 # Via Django
 auth.Group:
-  ".. no_pii:" : "No PII"
+  ".. no_pii:": "No PII"
 auth.Permission:
-  ".. no_pii:" : "No PII"
+  ".. no_pii:": "No PII"
 auth.User:
   ".. pii:": "Contains username, password, and email address, retired in AccountRetirementView"
-  ".. pii_types:" : username, email_address, password
-  ".. pii_retirement:" : local_api
+  ".. pii_types:": username, email_address, password
+  ".. pii_retirement:": local_api
 contenttypes.ContentType:
   ".. no_pii:": "No PII"
 admin.LogEntry:
@@ -27,6 +27,66 @@ sessions.Session:
 sites.Site:
   ".. no_pii:": "No PII"
 
+# Automatically generated edx-platform models that can't be annotated
+calendar_sync.HistoricalUserCalendarSyncConfig:
+  ".. no_pii:": "No PII"
+certificates.HistoricalCertificateAllowlist:
+  ".. no_pii:": "No PII"
+certificates.HistoricalCertificateDateOverride:
+  ".. no_pii:": "No PII"
+certificates.HistoricalCertificateInvalidation:
+  ".. no_pii:": "No PII"
+certificates.HistoricalGeneratedCertificate:
+  ".. pii:": "PII can exist in the generated certificate linked to in this model. Certificate data is currently retained."
+  ".. pii_types:": "name, username"
+  ".. pii_retirement:": "retained"
+course_apps.HistoricalCourseAppStatus:
+  ".. no_pii:": "No PII"
+course_goals.HistoricalCourseGoal:
+  ".. no_pii:": "No PII"
+course_live.HistoricalCourseLiveConfiguration:
+  ".. no_pii:": "No PII"
+course_modes.HistoricalCourseMode:
+  ".. no_pii:": "No PII"
+course_overviews.HistoricalCourseOverview:
+  ".. no_pii:": "No PII"
+discussions.HistoricalDiscussionsConfiguration:
+  ".. no_pii:": "No PII"
+entitlements.HistoricalCourseEntitlement:
+  ".. no_pii:": "No PII"
+entitlements.HistoricalCourseEntitlementSupportDetail:
+  ".. no_pii:": "No PII"
+experiments.HistoricalExperimentKeyValue:
+  ".. no_pii:": "No PII"
+external_user_ids.HistoricalExternalId:
+  ".. no_pii:": "We store external_user_id here, but do not consider that PII under OEP-30."
+external_user_ids.HistoricalExternalIdType:
+  ".. no_pii:": "No PII"
+grades.HistoricalPersistentSubsectionGradeOverride:
+  ".. no_pii:": "No PII"
+instructor_task.HistoricalInstructorTaskSchedule:
+  ".. no_pii:": "No PII"
+program_enrollments.HistoricalProgramCourseEnrollment:
+  ".. no_pii:": "No PII"
+program_enrollments.HistoricalProgramEnrollment:
+  ".. pii:": "PII is found in the external key for a program enrollment"
+  ".. pii_types:": "other"
+  ".. pii_retirement:": "local_api"
+programs.HistoricalProgramDiscussionsConfiguration:
+  ".. no_pii:": "No PII"
+programs.HistoricalProgramLiveConfiguration:
+  ".. no_pii:": "No PII"
+schedules.HistoricalSchedule:
+  ".. no_pii:": "No PII"
+split_modulestore_django.HistoricalSplitModulestoreCourseIndex:
+  ".. no_pii:": "No PII"
+student.HistoricalCourseEnrollment:
+  ".. no_pii:": "No PII"
+student.HistoricalManualEnrollmentAudit:
+  ".. pii:": "Contains enrolled_email, retired in LMSAccountRetirementView"
+  ".. pii_types:": "email_address"
+  ".. pii_retirement:": "local_api"
+
 # Automatically generated models in edx-enterprise that can't be annotated there
 consent.HistoricalDataSharingConsent:
   ".. pii:": "The username field inherited from Consent contains PII."
@@ -45,7 +105,7 @@ enterprise.HistoricalEnterpriseCustomerCatalog:
 enterprise.HistoricalEnterpriseCustomerEntitlement:
   ".. no_pii:": "No PII"
 
-# Via ORA2
+# Via edx-ora2, these can be removed once the models are annotated for real
 assessment.Assessment:
   ".. no_pii:": "No PII"
 assessment.AssessmentFeedback:
@@ -127,10 +187,24 @@ djcelery.TaskState:
 djcelery.WorkerState:
   ".. no_pii:": "No PII"
 
+# Via django-celery-results
+django_celery_results.ChordCounter:
+  ".. no_pii:": "No PII"
+django_celery_results.GroupResult:
+  ".. no_pii:": "No PII"
+django_celery_results.TaskResult:
+  ".. no_pii:": "No PII"
+
 # Via edx-oauth2-provider https://github.com/edx/edx-oauth2-provider
 edx_oauth2_provider.TrustedClient:
   ".. no_pii:": "No PII"
 
+# Via edx-name-affirmation, not part of the openedx org
+edx_name_affirmation.HistoricalVerifiedName:
+  ".. pii:": "Contains name fields."
+  ".. pii_types:": "name"
+  ".. pii_retirement:": "local_api"
+
 # Via VAL
 edxval.CourseVideo:
   ".. no_pii:": "No PII"
@@ -149,6 +223,12 @@ edxval.VideoImage:
 edxval.VideoTranscript:
   ".. no_pii:": "No PII"
 
+# Via PyLTI1p3
+lti1p3_tool_config.LtiTool:
+  ".. no_pii:": "No PII"
+lti1p3_tool_config.LtiToolKey:
+  ".. no_pii:": "No PII"
+
 # Via Milestones
 milestones.CourseContentMilestone:
   ".. no_pii:": "No PII"
@@ -190,6 +270,10 @@ oauth2_provider.Grant:
   ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView."
   ".. pii_types:": password, other
   ".. pii_retirement:": local_api
+oauth2_provider.IDToken:
+  ".. pii:": "Contains 3rd party authentication secrets, currently this is retained until the token times out, but should be retired explicitly with the other models from this package."
+  ".. pii_types:": password, other
+  ".. pii_retirement:": retained
 oauth2_provider.RefreshToken:
   ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView."
   ".. pii_types:": password, other
@@ -250,6 +334,8 @@ submissions.StudentItem:
   ".. no_pii:": "No PII"
 submissions.Submission:
   ".. no_pii:": "No PII"
+submissions.TeamSubmission:
+  ".. no_pii:": "No PII"
 
 # Via sorl-thumbnail https://github.com/jazzband/sorl-thumbnail
 thumbnail.KVStore:

.github/workflows/check-consistent-dependencies.yml

@@ -15,7 +15,7 @@ defaults:
 jobs:
   check-requirements:
     name: Compile requirements
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       # Only run remaining steps if there are changes to requirements/**

.github/workflows/ci-static-analysis.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         python-version:
           - "3.11"
-        os: ["ubuntu-22.04"]
+        os: ["ubuntu-24.04"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/migrations-check.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python-version:
           - "3.11"
         # 'pinned' is used to install the latest patch version of Django
@@ -126,7 +126,7 @@ jobs:
     if: always()
     needs:
       - check_migrations
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         # uses: re-actors/[email protected]

.github/workflows/pylint-checks.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   run-pylint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -75,7 +75,7 @@ jobs:
     if: always()
     needs:
       - run-pylint
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         # uses: re-actors/[email protected]

.github/workflows/quality-checks.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python-version:
           - "3.11"
         node-version: [20]

.github/workflows/static-assets-check.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python-version:
           - "3.11"
         node-version: [18, 20]

.github/workflows/unit-tests.yml

@@ -15,7 +15,7 @@ concurrency:
 jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os-version }}
     strategy:
       matrix:
         python-version:
@@ -43,22 +43,27 @@ jobs:
           - "xmodule-with-cms"
         mongo-version:
           - "7.0"
+        os-version:
+          - ubuntu-24.04
 
-        # We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore)
-        # This code is left here as an example for future refernce in case we need to reduce the number of shards we're
-        # testing but still have good confidence with older versions of mongo.  We use Mongo 4.4 in the example.
+        # It's useful to run some subset of the tests on the older version of Ubuntu
+        # so that we don't spend too many resources on this but can find major issues quickly
+        # while we're in a situation where we support two versions.  This section may be commented
+        # out when not in use to easily add/drop future support for any given major dependency.
         #
-        # exclude:
-        #   - mongo-version: "4.4"
-        # include:
-        #   - shard_name: "xmodule-with-cms"
-        #     python-version: "3.11"
-        #     django-version: "pinned"
-        #     mongo-version: "4.4"
-        #   - shard_name: "xmodule-with-lms"
-        #     python-version: "3.11"
-        #     django-version: "pinned"
-        #     mongo-version: "4.4"
+        # We're testing the older version of Ubuntu and running the xmodule tests since those rely on many
+        # dependent complex libraries and will hopefully catch most issues quickly.
+        include:
+          - shard_name: "xmodule-with-cms"
+            python-version: "3.11"
+            django-version: "pinned"
+            mongo-version: "7.0"
+            os-version: "ubuntu-22.04"
+          - shard_name: "xmodule-with-lms"
+            python-version: "3.11"
+            django-version: "pinned"
+            mongo-version: "7.0"
+            os-version: "ubuntu-22.04"
 
     steps:
       - name: checkout repo
@@ -90,19 +95,10 @@ jobs:
           activate = 1
           EOF
 
-      - name: install mongo version
-        run: |
-          if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then
-            wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add -
-            echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list
-            sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*"
-          fi
-
-      - name: start mongod server for tests
-        run: |
-          sudo mkdir -p /data/db
-          sudo chmod -R a+rw /data/db
-          mongod &
+      - name: Start MongoDB
+        uses: supercharge/[email protected]
+        with:
+          mongodb-version: ${{ matrix.mongo-version }}
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -164,7 +160,7 @@ jobs:
           overwrite: true
 
   collect-and-verify:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python
@@ -229,7 +225,7 @@ jobs:
   # https://github.com/orgs/community/discussions/33579
   success:
     name: Unit tests successful
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: always()
     needs: [run-tests]
     steps:
@@ -240,7 +236,7 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [run-tests]
     steps:
       - uses: actions/checkout@v4
@@ -268,7 +264,7 @@ jobs:
           overwrite: true
 
   merge-artifacts:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [compile-warnings-report]
     steps:
       - name: Merge Pytest Warnings JSON Artifacts
@@ -288,7 +284,7 @@ jobs:
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [run-tests]
     strategy:
       matrix:

.github/workflows/verify-dunder-init.yml

@@ -1,4 +1,4 @@
-name: CI
+name: Verify Dunder __init__.py Files
 
 on:
   pull_request:

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.12"
+    python: "3.11"
 
 sphinx:
   configuration: docs/conf.py

cms/djangoapps/contentstore/courseware_index.py

@@ -256,7 +256,8 @@ def prepare_item_index(item, skip_index=False, groups_usage_info=None):
                 # Now index the content
                 for item in structure.get_children():
                     prepare_item_index(item, groups_usage_info=groups_usage_info)
-                searcher.index(items_index, request_timeout=timeout)
+                if items_index:
+                    searcher.index(items_index, request_timeout=timeout)
                 cls.remove_deleted_items(searcher, structure_key, indexed_items)
         except Exception as err:  # pylint: disable=broad-except
             # broad exception so that index operation does not prevent the rest of the application from working