From c1e4a055d8400dbe335d9af98a53e87f78328c21 Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Thu, 11 May 2023 08:26:22 +0200 Subject: [PATCH 01/12] fix: update shard names for GH tests Github hosted tests weren't working because shards were not up to date. --- .github/workflows/unit-tests-gh-hosted.yml | 35 +++++++++++----------- .github/workflows/unit-tests.yml | 2 ++ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index 894f61048e90..eccd5fef11f3 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -17,23 +17,24 @@ jobs: python-version: [ '3.8' ] django-version: - "pinned" - shard_name: [ - "lms-1", - "lms-2", - "lms-3", - "lms-4", - "lms-5", - "lms-6", - "openedx-1", - "openedx-2", - "openedx-3", - "openedx-4", - "cms-1", - "cms-2", - "common-1", - "common-2", - "common-3", - ] + # When updating the shards, remember to make the same changes in + # .github/workflows/unit-tests.yml + shard_name: + - "lms-1" + - "lms-2" + - "lms-3" + - "lms-4" + - "lms-5" + - "lms-6" + - "openedx-1" + - "openedx-2" + - "openedx-3" + - "openedx-4" + - "cms-1" + - "cms-2" + - "common-1" + - "common-2" + - "xmodule-1" name: gh-hosted-python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5b4b8ed94355..a8de7d1a8b11 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,6 +18,8 @@ jobs: django-version: - "pinned" #- "4.0" + # When updating the shards, remember to make the same changes in + # .github/workflows/unit-tests-gh-hosted.yml shard_name: - "lms-1" - "lms-2" From 3df4f3cda508e62a29c29c141e051e46220cbf6b Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Thu, 11 May 2023 08:36:18 +0200 Subject: [PATCH 02/12] Revert "fix: update test scripts for palm" The Github runners are working now, so switch back to using them instead of 2U self-hosted workers which are only guaranteed to work on master and will probably stop working on the Palm branch at some point. This reverts commit de62e380a8a3f6ac5359fd063cc9709d15b66fd4. --- .github/workflows/unit-tests-gh-hosted.yml | 4 ++-- .github/workflows/unit-tests.yml | 2 +- .github/workflows/verify-gha-unit-tests-count.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index eccd5fef11f3..80a6a4e53a08 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -9,7 +9,7 @@ on: jobs: run-test: - if: github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private' + if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -79,7 +79,7 @@ jobs: uses: ./.github/actions/unit-tests collect-and-verify: - if: github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private' + if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) runs-on: ubuntu-20.04 strategy: matrix: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a8de7d1a8b11..43ac221e4680 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -9,7 +9,7 @@ on: jobs: run-tests: name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} - if: github.repository == 'edx/edx-platform-private' || github.repository == 'openedx/edx-platform' + if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) runs-on: [ edx-platform-runner ] strategy: matrix: diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml index a519c62fa0a0..c68092942d70 100644 --- a/.github/workflows/verify-gha-unit-tests-count.yml +++ b/.github/workflows/verify-gha-unit-tests-count.yml @@ -8,7 +8,7 @@ on: jobs: collect-and-verify: - if: github.repository == 'edx/edx-platform-private' || github.repository == 'openedx/edx-platform' + if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) runs-on: [ edx-platform-runner ] steps: - name: sync directory owner From 0d63190bb61a9bba9ed086949509290b2646a250 Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Thu, 11 May 2023 09:06:08 +0200 Subject: [PATCH 03/12] fix: don't run test check when using GH runners --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 43ac221e4680..c66c30d7be6f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -82,7 +82,7 @@ jobs: # https://github.com/orgs/community/discussions/33579 success: name: Tests successful - if: always() + if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) needs: - run-tests runs-on: ubuntu-latest From 8eb46489db893a4dbb6a6f4a4615bbbfab79314e Mon Sep 17 00:00:00 2001 From: Eugene Dyudyunov Date: Fri, 12 May 2023 12:33:10 +0300 Subject: [PATCH 04/12] fix: upgrade reminder email image Replace a relative URL with the absolute one. --- openedx/core/djangoapps/schedules/resolvers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/schedules/resolvers.py b/openedx/core/djangoapps/schedules/resolvers.py index 1b406a6d7150..3498042415ff 100644 --- a/openedx/core/djangoapps/schedules/resolvers.py +++ b/openedx/core/djangoapps/schedules/resolvers.py @@ -3,6 +3,7 @@ import datetime import logging from itertools import groupby +from urllib.parse import urljoin import attr from django.conf import settings @@ -322,7 +323,7 @@ def get_template_context(self, user, user_schedules): context = { 'course_links': course_links, 'first_course_name': first_schedule.enrollment.course.display_name, - 'cert_image': static('course_experience/images/verified-cert.png'), + 'cert_image': urljoin(settings.LMS_ROOT_URL, static('course_experience/images/verified-cert.png')), 'course_ids': course_id_strs, } context.update(first_valid_upsell_context) From eec6b53d4fabc4288f4aab8b380db559c61a2b3a Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Thu, 25 May 2023 14:05:11 +0200 Subject: [PATCH 05/12] chore: upgrade Django to 3.2.19 (#32295) --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c29df9ff37d2..d83b55348774 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -177,7 +177,7 @@ defusedxml==0.7.1 # social-auth-core deprecated==1.2.13 # via jwcrypto -django==3.2.18 +django==3.2.19 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 145905017d4b..6feae4daa8c8 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -274,7 +274,7 @@ distlib==0.3.6 # via # -r requirements/edx/testing.txt # virtualenv -django==3.2.18 +django==3.2.19 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 5a94e1edf188..b1901213a7ac 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -258,7 +258,7 @@ dill==0.3.6 # via pylint distlib==0.3.6 # via virtualenv -django==3.2.18 +django==3.2.19 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt From 466baf69b23374cf1e36e461879c257f7aec46e9 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Wed, 19 Apr 2023 15:29:06 +0200 Subject: [PATCH 06/12] fix: missing advance_settings_access template variable Co-authored-by: Farhaan Bukhsh (cherry picked from commit 065f894d1b797476b78bc3762d4230140aead828) --- .../tests/test_course_settings.py | 59 +++++++++++++++---- cms/djangoapps/contentstore/views/course.py | 21 +++---- cms/templates/settings.html | 3 + cms/templates/settings_graders.html | 3 + cms/templates/widgets/header.html | 3 +- common/djangoapps/student/auth.py | 16 ++++- 6 files changed, 79 insertions(+), 26 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_course_settings.py b/cms/djangoapps/contentstore/tests/test_course_settings.py index a135faf06712..be7691ee7e45 100644 --- a/cms/djangoapps/contentstore/tests/test_course_settings.py +++ b/cms/djangoapps/contentstore/tests/test_course_settings.py @@ -106,7 +106,12 @@ def setUp(self): super().setUp() self.fullcourse = CourseFactory.create() self.course_setting_url = get_url(self.course.id, 'advanced_settings_handler') - self.non_staff_client, _ = self.create_non_staff_authed_user_client() + + self.non_staff_client, self.nonstaff = self.create_non_staff_authed_user_client() + # "nonstaff" means "non Django staff" here. We assign this user to course staff + # role to check that even so they won't have advanced settings access when explicitly + # restricted. + CourseStaffRole(self.course.id).add_users(self.nonstaff) @override_settings(FEATURES={'DISABLE_MOBILE_COURSE_AVAILABLE': True}) def test_mobile_field_available(self): @@ -145,16 +150,50 @@ def test_discussion_fields_available(self, is_pages_and_resources_enabled, self.assertEqual('discussion_blackouts' in response, fields_visible) self.assertEqual('discussion_topics' in response, fields_visible) - @override_settings(FEATURES={'DISABLE_ADVANCED_SETTINGS': True}) - def test_disable_advanced_settings_feature(self): - """ - If this feature is enabled, only staff should be able to access the advanced settings page. - """ - response = self.non_staff_client.get_html(self.course_setting_url) - self.assertEqual(response.status_code, 403) + @ddt.data(False, True) + def test_disable_advanced_settings_feature(self, disable_advanced_settings): + """ + If this feature is enabled, only Django Staff/Superuser should be able to access the "Advanced Settings" page. + For non-staff users the "Advanced Settings" tab link should not be visible. + """ + advanced_settings_link_html = f"Advanced Settings".encode('utf-8') + + with override_settings(FEATURES={'DISABLE_ADVANCED_SETTINGS': disable_advanced_settings}): + for handler in ( + 'import_handler', + 'export_handler', + 'course_team_handler', + 'course_info_handler', + 'assets_handler', + 'tabs_handler', + 'settings_handler', + 'grading_handler', + 'textbooks_list_handler', + ): + # Test that non-staff users don't see the "Advanced Settings" tab link. + response = self.non_staff_client.get_html( + get_url(self.course.id, handler) + ) + self.assertEqual(response.status_code, 200) + if disable_advanced_settings: + self.assertNotIn(advanced_settings_link_html, response.content) + else: + self.assertIn(advanced_settings_link_html, response.content) + + # Test that staff users see the "Advanced Settings" tab link. + response = self.client.get_html( + get_url(self.course.id, handler) + ) + self.assertEqual(response.status_code, 200) + self.assertIn(advanced_settings_link_html, response.content) - response = self.client.get_html(self.course_setting_url) - self.assertEqual(response.status_code, 200) + # Test that non-staff users can't access the "Advanced Settings" page. + response = self.non_staff_client.get_html(self.course_setting_url) + self.assertEqual(response.status_code, 403 if disable_advanced_settings else 200) + + # Test that staff users can access the "Advanced Settings" page. + response = self.client.get_html(self.course_setting_url) + self.assertEqual(response.status_code, 200) @ddt.ddt diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 3bf9ade016e4..023cb07199fe 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -43,7 +43,12 @@ from common.djangoapps.course_action_state.models import CourseRerunState, CourseRerunUIStateManager from common.djangoapps.course_modes.models import CourseMode from common.djangoapps.edxmako.shortcuts import render_to_response -from common.djangoapps.student.auth import has_course_author_access, has_studio_read_access, has_studio_write_access +from common.djangoapps.student.auth import ( + has_course_author_access, + has_studio_read_access, + has_studio_write_access, + has_studio_advanced_settings_access +) from common.djangoapps.student.roles import ( CourseInstructorRole, CourseStaffRole, @@ -147,17 +152,6 @@ class AccessListFallback(Exception): pass # lint-amnesty, pylint: disable=unnecessary-pass -def has_advanced_settings_access(user): - """ - If DISABLE_ADVANCED_SETTINGS feature is enabled, only global staff can access "Advanced Settings". - """ - return ( - not settings.FEATURES.get('DISABLE_ADVANCED_SETTINGS', False) - or user.is_staff - or user.is_superuser - ) - - def get_course_and_check_access(course_key, user, depth=0): """ Function used to calculate and return the locator and course block @@ -763,7 +757,6 @@ def course_index(request, course_key): 'frontend_app_publisher_url': frontend_app_publisher_url, 'mfe_proctored_exam_settings_url': get_proctored_exam_settings_url(course_block.id), 'advance_settings_url': reverse_course_url('advanced_settings_handler', course_block.id), - 'advance_settings_access': has_advanced_settings_access(request.user), 'proctoring_errors': proctoring_errors, }) @@ -1432,7 +1425,7 @@ def advanced_settings_handler(request, course_key_string): json: update the Course's settings. The payload is a json rep of the metadata dicts. """ - if not has_advanced_settings_access(request.user): + if not has_studio_advanced_settings_access(request.user): raise PermissionDenied() course_key = CourseKey.from_string(course_key_string) diff --git a/cms/templates/settings.html b/cms/templates/settings.html index c3ed1afb9957..b7ef368a0911 100644 --- a/cms/templates/settings.html +++ b/cms/templates/settings.html @@ -7,6 +7,7 @@ <%namespace name='static' file='static_content.html'/> <%! from django.utils.translation import gettext as _ + from common.djangoapps.student.auth import has_studio_advanced_settings_access from cms.djangoapps.contentstore import utils from lms.djangoapps.certificates.api import can_show_certificate_available_date_field from openedx.core.djangolib.js_utils import ( @@ -721,7 +722,9 @@

${_("Other Course Settings")}

  • ${_("Grading")}
  • ${_("Course Team")}
  • ${_("Group Configurations")}
    • + % if has_studio_advanced_settings_access(request.user):
  • ${_("Advanced Settings")}
    • + % endif % if mfe_proctored_exam_settings_url:
  • ${_("Proctored Exam Settings")}
    • % endif diff --git a/cms/templates/settings_graders.html b/cms/templates/settings_graders.html index bde135122155..ac7119177978 100644 --- a/cms/templates/settings_graders.html +++ b/cms/templates/settings_graders.html @@ -11,6 +11,7 @@ import json from cms.djangoapps.contentstore import utils from django.utils.translation import gettext as _ + from common.djangoapps.student.auth import has_studio_advanced_settings_access from cms.djangoapps.models.settings.encoder import CourseSettingsEncoder from openedx.core.djangolib.js_utils import ( dump_js_escaped_json, js_escaped_string @@ -165,7 +166,9 @@

    ${_("Other Course Settings")}

  • ${_("Details & Schedule")}
  • ${_("Course Team")}
  • ${_("Group Configurations")}
    • + % if has_studio_advanced_settings_access(request.user):
  • ${_("Advanced Settings")}
    • + % endif % if mfe_proctored_exam_settings_url:
  • ${_("Proctored Exam Settings")}
    • % endif diff --git a/cms/templates/widgets/header.html b/cms/templates/widgets/header.html index 23678ec395ae..48b818da95a5 100644 --- a/cms/templates/widgets/header.html +++ b/cms/templates/widgets/header.html @@ -7,6 +7,7 @@ from django.urls import reverse from django.utils.translation import gettext as _ from urllib.parse import quote_plus + from common.djangoapps.student.auth import has_studio_advanced_settings_access from cms.djangoapps.contentstore import toggles from cms.djangoapps.contentstore.utils import get_pages_and_resources_url from openedx.core.djangoapps.discussions.config.waffle import ENABLE_PAGES_AND_RESOURCES_MICROFRONTEND @@ -120,7 +121,7 @@

    ${_("Course" ${_("Proctored Exam Settings")} % endif - % if advance_settings_access: + % if has_studio_advanced_settings_access(request.user):
  • ${_("Advanced Settings")}
    • diff --git a/common/djangoapps/student/auth.py b/common/djangoapps/student/auth.py index 3091dbc45406..f8e45ce6ba6c 100644 --- a/common/djangoapps/student/auth.py +++ b/common/djangoapps/student/auth.py @@ -125,9 +125,23 @@ def has_course_author_access(user, course_key): return has_studio_write_access(user, course_key) +def has_studio_advanced_settings_access(user): + """ + If DISABLE_ADVANCED_SETTINGS feature is enabled, only Django Superuser + or Django Staff can access "Advanced Settings". + + By default, this feature is disabled. + """ + return ( + not settings.FEATURES.get('DISABLE_ADVANCED_SETTINGS', False) + or user.is_staff + or user.is_superuser + ) + + def has_studio_read_access(user, course_key): """ - Return True iff user is allowed to view this course/library in studio. + Return True if user is allowed to view this course/library in studio. Will also return True if user has write access in studio (has_course_author_access) There is currently no such thing as read-only course access in studio, but From a7f24d0b829c9deb7816ffe50a323b6dffed9eba Mon Sep 17 00:00:00 2001 From: connorhaugh <49422820+connorhaugh@users.noreply.github.com> Date: Tue, 30 May 2023 10:39:33 -0400 Subject: [PATCH 07/12] fix: remove drag and drop from V1 llibs (#32326) --- cms/djangoapps/contentstore/views/component.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/views/component.py b/cms/djangoapps/contentstore/views/component.py index 457f84793efc..b7dd09d5c7cb 100644 --- a/cms/djangoapps/contentstore/views/component.py +++ b/cms/djangoapps/contentstore/views/component.py @@ -292,8 +292,8 @@ def create_support_legend_dict(): # by the components in the order listed in COMPONENT_TYPES. component_types = COMPONENT_TYPES[:] - # Libraries do not support discussions and openassessment and other libraries - component_not_supported_by_library = ['discussion', 'library', 'openassessment'] + # Libraries do not support discussions, drag-and-drop, and openassessment and other libraries + component_not_supported_by_library = ['discussion', 'library', 'openassessment', 'drag-and-drop-v2'] if library: component_types = [component for component in component_types if component not in set(component_not_supported_by_library)] From b040d353237ab4fc47dda544b5b8594ab48c0411 Mon Sep 17 00:00:00 2001 From: Syed Sajjad Hussain Shah <52817156+syedsajjadkazmii@users.noreply.github.com> Date: Thu, 20 Apr 2023 12:33:54 +0500 Subject: [PATCH 08/12] fix: inconsistency between JWT and session authentication after password reset (#32073) VAN-1371 Co-authored-by: Syed Sajjad Hussain Shah (cherry picked from commit 416a502c9683b5aec3b82d4fd2d19fbd8e35a897) --- .../djangoapps/cache_toolbox/middleware.py | 11 ++++- .../cache_toolbox/tests/test_middleware.py | 44 ++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/cache_toolbox/middleware.py b/openedx/core/djangoapps/cache_toolbox/middleware.py index 6284613d49e6..4ba2162fe35f 100644 --- a/openedx/core/djangoapps/cache_toolbox/middleware.py +++ b/openedx/core/djangoapps/cache_toolbox/middleware.py @@ -52,6 +52,14 @@ compounded by most projects wishing to avoid expiring session data as long as possible (in addition to storing sessions in persistent stores). +Dependency with SafeSessionMiddleware +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +CacheBackedAuthenticationMiddleware middleware logs out the user if the +session hash is changed due to password change. It flushes the session +and mark cookies for deletion in request which are then deleted in the +process_response of SafeSessionMiddleware. + Usage ~~~~~ @@ -88,7 +96,7 @@ from django.utils.crypto import constant_time_compare from django.utils.deprecation import MiddlewareMixin -from openedx.core.djangoapps.safe_sessions.middleware import SafeSessionMiddleware +from openedx.core.djangoapps.safe_sessions.middleware import SafeSessionMiddleware, _mark_cookie_for_deletion from .model import cache_model @@ -138,3 +146,4 @@ def _verify_session_auth(self, request): # change. Log the user out. request.session.flush() request.user = AnonymousUser() + _mark_cookie_for_deletion(request) diff --git a/openedx/core/djangoapps/cache_toolbox/tests/test_middleware.py b/openedx/core/djangoapps/cache_toolbox/tests/test_middleware.py index 68f0577968e2..21547b69ca1c 100644 --- a/openedx/core/djangoapps/cache_toolbox/tests/test_middleware.py +++ b/openedx/core/djangoapps/cache_toolbox/tests/test_middleware.py @@ -2,11 +2,15 @@ from unittest.mock import patch from django.conf import settings -from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user +from django.contrib.auth.models import User, AnonymousUser # lint-amnesty, pylint: disable=imported-auth-user from django.urls import reverse from django.test import TestCase +from django.contrib.auth import SESSION_KEY +from django.http import HttpResponse, SimpleCookie -from openedx.core.djangolib.testing.utils import skip_unless_cms, skip_unless_lms +from openedx.core.djangoapps.cache_toolbox.middleware import CacheBackedAuthenticationMiddleware +from openedx.core.djangoapps.safe_sessions.middleware import SafeCookieData, SafeSessionMiddleware +from openedx.core.djangolib.testing.utils import skip_unless_cms, skip_unless_lms, get_mock_request from common.djangoapps.student.tests.factories import UserFactory @@ -18,6 +22,9 @@ def setUp(self): password = 'test-password' self.user = UserFactory(password=password) self.client.login(username=self.user.username, password=password) + self.request = get_mock_request(self.user) + self.client.response = HttpResponse() + self.client.response.cookies = SimpleCookie() # preparing cookies def _test_change_session_hash(self, test_url, redirect_url, target_status_code=200): """ @@ -45,3 +52,36 @@ def test_session_change_cms(self): home_url = reverse('home') # Studio login redirects to LMS login self._test_change_session_hash(home_url, settings.LOGIN_URL + '?next=' + home_url, target_status_code=302) + + def test_user_logout_on_session_hash_change(self): + """ + Verify that if a user's session auth hash and the request's hash + differ, the user is logged out: + - session is flushed + - request user is changed to Anonymous user + - logged in cookies are deleted + """ + # preparing session and setting cookies + session_id = self.client.session.session_key + safe_cookie_data = SafeCookieData.create(session_id, self.user.id) + self.request.COOKIES[settings.SESSION_COOKIE_NAME] = str(safe_cookie_data) + self.client.response.cookies[settings.SESSION_COOKIE_NAME] = session_id + self.client.response.cookies['edx-jwt-cookie-header-payload'] = 'test-jwt-payload' + SafeSessionMiddleware().process_request(self.request) + + # asserts that user, session, and JWT cookies exist + assert self.request.session.get(SESSION_KEY) is not None + assert self.request.user != AnonymousUser() + assert self.client.response.cookies.get(settings.SESSION_COOKIE_NAME).value == session_id + assert self.client.response.cookies.get('edx-jwt-cookie-header-payload').value == 'test-jwt-payload' + + with patch.object(User, 'get_session_auth_hash', return_value='abc123'): + CacheBackedAuthenticationMiddleware().process_request(self.request) + SafeSessionMiddleware().process_response(self.request, self.client.response) + + # asserts that user, session, and JWT cookies do not exist + assert self.request.session.get(SESSION_KEY) is None + assert self.request.user == AnonymousUser() + assert self.client.response.cookies.get(settings.SESSION_COOKIE_NAME).value != session_id + assert self.client.response.cookies.get(settings.SESSION_COOKIE_NAME).value == "" + assert self.client.response.cookies.get('edx-jwt-cookie-header-payload').value == "" From 2b50ccb701c41ba860f81abb6504fb263b2450a1 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 17 Apr 2023 17:21:48 +0000 Subject: [PATCH 09/12] build: Deduplicate mismatched pip.txt files that caused build failure (#32081) There was a `requirements/pip.txt` with old versions, and a newer `requirements/edx/pip.txt` managed via a `pip.in` file. The old one was used in most places, but came out of sync with pip-tools.txt, which was managed properly. Eventually this caused a `pip check` failure due to the mismatch. This should resolve at least part of https://github.com/edx/edx-arch-experiments/issues/267 This PR moves pip.in and pip-tools.in and their corresponding pin files up to the `requirements/` dir, since they should be shared between the edx and sandbox environments. This also has the effect of upgrading pip to match the version in the file we've been uselessly upgrading. Other improvements: - Remove `-q` option from pip and pip-sync calls, as it was hiding some debugging information that would have resolved this sooner. - Depend on `pre-requirements` from `compile-requirements`, rather than from `upgrade`. (The base target is the one that actually needs it.) This also lets us remove the explicit `pip install pip-tools` line. - Install the recompiled pip and pip-tools files right away, not after the loop. When we upgrade pip-tools, we want to use the upgraded version, not the previous version. This requires moving the pip-tools.txt recompilation outside of the loop and into its own explicit line. - Don't upgrade pip if we're not running `make upgrade` (respect the compile options). - Remove apparently-unneeded `--no-emit-trusted-host --no-emit-index-url` options (we don't pass trusted-host or index-url options). (cherry picked from commit b852344fcfaba49b7df47e050a691e72eff3649f) --- Makefile | 28 +++++++++++++++------------- requirements/edx/development.in | 2 +- requirements/edx/development.txt | 14 +++++++------- requirements/edx/pip.txt | 14 -------------- requirements/{edx => }/pip-tools.in | 2 +- requirements/{edx => }/pip-tools.txt | 4 ++-- requirements/{edx => }/pip.in | 2 +- requirements/pip.txt | 16 ++++++++++++++-- scripts/ci-runner.Dockerfile | 2 +- 9 files changed, 42 insertions(+), 42 deletions(-) delete mode 100644 requirements/edx/pip.txt rename requirements/{edx => }/pip-tools.in (95%) rename requirements/{edx => }/pip-tools.txt (82%) rename requirements/{edx => }/pip.in (77%) diff --git a/Makefile b/Makefile index 7aa7334fdfac..fec1c2caf2e7 100644 --- a/Makefile +++ b/Makefile @@ -64,8 +64,8 @@ pull: ## update the Docker image used by "make shell" docker pull edxops/edxapp:latest pre-requirements: ## install Python requirements for running pip-tools - pip install -qr requirements/pip.txt - pip install -qr requirements/edx/pip-tools.txt + pip install -r requirements/pip.txt + pip install -r requirements/pip-tools.txt local-requirements: # edx-platform installs some Python projects from within the edx-platform repo itself. @@ -74,7 +74,7 @@ local-requirements: dev-requirements: pre-requirements @# The "$(wildcard..)" is to include private.txt if it exists, and make no mention @# of it if it does not. Shell wildcarding can't do that with default options. - pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt) + pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt) make local-requirements base-requirements: pre-requirements @@ -96,7 +96,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend # Order is very important in this list: files must appear after everything they include! REQ_FILES = \ - requirements/edx/pip-tools \ requirements/edx/coverage \ requirements/edx/doc \ requirements/edx/paver \ @@ -117,23 +116,26 @@ $(COMMON_CONSTRAINTS_TXT): echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@) compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade -compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt - pip install -q pip-tools - pip-compile --allow-unsafe --upgrade -o requirements/edx/pip.txt requirements/edx/pip.in +compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt + @# Bootstrapping: Rebuild pip and pip-tools first, and then install them + @# so that if there are any failures we'll know now, rather than the next + @# time someone tries to use the outputs. + pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in + pip install -r requirements/pip.txt + + pip-compile -v ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in + pip install -r requirements/pip-tools.txt @ export REBUILD='--rebuild'; \ for f in $(REQ_FILES); do \ echo ; \ echo "== $$f ===============================" ; \ - echo "pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \ - pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \ + echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \ + pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \ export REBUILD=''; \ done - pip install -qr requirements/edx/pip.txt - pip install -qr requirements/edx/pip-tools.txt - -upgrade: pre-requirements ## update the pip requirements files to use the latest releases satisfying our constraints +upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" check-types: ## run static type-checking tests diff --git a/requirements/edx/development.in b/requirements/edx/development.in index 6b4cfa8317a6..1a857f291711 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -10,7 +10,7 @@ -c ../constraints.txt --r pip-tools.txt # pip-tools and its dependencies, for managing requirements files +-r ../pip-tools.txt # pip-tools and its dependencies, for managing requirements files -r testing.txt # Dependencies for running the various test suites click # Used for perf_tests utilities in modulestore diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 6feae4daa8c8..64da84e15034 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -125,7 +125,7 @@ bridgekeeper==0.9 # via -r requirements/edx/testing.txt build==0.10.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # pip-tools celery==5.2.7 # via @@ -168,8 +168,8 @@ chem==1.2.0 click==8.1.3 # via # -c requirements/edx/../constraints.txt + # -r requirements/edx/../pip-tools.txt # -r requirements/edx/development.in - # -r requirements/edx/pip-tools.txt # -r requirements/edx/testing.txt # celery # click-didyoumean @@ -1047,7 +1047,7 @@ outcome-surveys==2.4.0 # via -r requirements/edx/testing.txt packaging==23.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # -r requirements/edx/testing.txt # build # drf-yasg @@ -1093,7 +1093,7 @@ pillow==9.4.0 # edx-enterprise # edx-organizations pip-tools==6.12.3 - # via -r requirements/edx/pip-tools.txt + # via -r requirements/edx/../pip-tools.txt pkgutil-resolve-name==1.3.10 # via # -r requirements/edx/testing.txt @@ -1237,7 +1237,7 @@ pyparsing==3.0.9 # openedx-calc pyproject-hooks==1.0.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # build pyquery==2.0.0 # via -r requirements/edx/testing.txt @@ -1584,7 +1584,7 @@ toml==0.10.2 # via vulture tomli==2.0.1 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # -r requirements/edx/testing.txt # build # coverage @@ -1691,7 +1691,7 @@ webob==1.8.7 # xblock wheel==0.40.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # pip-tools wrapt==1.15.0 # via diff --git a/requirements/edx/pip.txt b/requirements/edx/pip.txt deleted file mode 100644 index f0e5e9397cae..000000000000 --- a/requirements/edx/pip.txt +++ /dev/null @@ -1,14 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# make upgrade -# -wheel==0.40.0 - # via -r requirements/edx/pip.in - -# The following packages are considered to be unsafe in a requirements file: -pip==23.0.1 - # via -r requirements/edx/pip.in -setuptools==67.6.0 - # via -r requirements/edx/pip.in diff --git a/requirements/edx/pip-tools.in b/requirements/pip-tools.in similarity index 95% rename from requirements/edx/pip-tools.in rename to requirements/pip-tools.in index 50f3de3e19d6..5d4419ea4bc0 100644 --- a/requirements/edx/pip-tools.in +++ b/requirements/pip-tools.in @@ -7,6 +7,6 @@ # * confirm that it has no system requirements beyond what we already install # * run "make upgrade" to update the detailed requirements files --c ../constraints.txt +-c constraints.txt pip-tools # Contains pip-compile, used to generate pip requirements files diff --git a/requirements/edx/pip-tools.txt b/requirements/pip-tools.txt similarity index 82% rename from requirements/edx/pip-tools.txt rename to requirements/pip-tools.txt index 15e167ee7153..f434fbb8b915 100644 --- a/requirements/edx/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,12 +8,12 @@ build==0.10.0 # via pip-tools click==8.1.3 # via - # -c requirements/edx/../constraints.txt + # -c requirements/constraints.txt # pip-tools packaging==23.0 # via build pip-tools==6.12.3 - # via -r requirements/edx/pip-tools.in + # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 # via build tomli==2.0.1 diff --git a/requirements/edx/pip.in b/requirements/pip.in similarity index 77% rename from requirements/edx/pip.in rename to requirements/pip.in index 741969aac127..cc36db5a0831 100644 --- a/requirements/edx/pip.in +++ b/requirements/pip.in @@ -1,4 +1,4 @@ --c ../constraints.txt +-c constraints.txt # Core dependencies for installing other dependencies pip diff --git a/requirements/pip.txt b/requirements/pip.txt index a811cc82f187..45fb600be5f0 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,2 +1,14 @@ -pip==22.1 -wheel==0.37.1 +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# make upgrade +# +wheel==0.40.0 + # via -r requirements/pip.in + +# The following packages are considered to be unsafe in a requirements file: +pip==23.0.1 + # via -r requirements/pip.in +setuptools==67.6.0 + # via -r requirements/pip.in diff --git a/scripts/ci-runner.Dockerfile b/scripts/ci-runner.Dockerfile index 999b89941eff..291f1d33d32f 100644 --- a/scripts/ci-runner.Dockerfile +++ b/scripts/ci-runner.Dockerfile @@ -46,7 +46,7 @@ COPY openedx/core/lib openedx/core/lib COPY lms lms COPY cms cms COPY requirements/pip.txt requirements/pip.txt -COPY requirements/edx/pip-tools.txt requirements/edx/pip-tools.txt +COPY requirements/pip-tools.txt requirements/pip-tools.txt COPY requirements/edx/testing.txt requirements/edx/testing.txt COPY Makefile Makefile RUN make test-requirements From 2817dd170d61369e10d64b1870c9d12edbf316c1 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Tue, 18 Apr 2023 18:10:01 +0500 Subject: [PATCH 10/12] fix: fix make upgrade job and _mysql issue (#32084) * fix: upgrade pip & pip-tools versions * fix: install libmysqlclient-dev system package again * fix: skip reinstalling mysqlclient package * fix: add pytz package constraint --------- Co-authored-by: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> (cherry picked from commit 0ec7b8cc30abbc17569ca1737bcb581c42cb50cd) --- .github/workflows/migrations-check.yml | 9 ++-- requirements/constraints.txt | 4 ++ requirements/edx-sandbox/py38.txt | 6 +-- requirements/edx/base.txt | 52 +++++++++--------- requirements/edx/coverage.txt | 4 +- requirements/edx/development.txt | 75 +++++++++++++------------- requirements/edx/doc.txt | 10 ++-- requirements/edx/paver.txt | 6 +-- requirements/edx/testing.txt | 68 +++++++++++------------ requirements/pip-tools.txt | 4 +- requirements/pip.txt | 4 +- 11 files changed, 126 insertions(+), 116 deletions(-) diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index bd0a07680ad7..fc4a48876a93 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -61,10 +61,6 @@ jobs: - name: Install Python dependencies run: | make dev-requirements - pip uninstall -y mysqlclient - pip install --no-binary mysqlclient mysqlclient - pip uninstall -y xmlsec - pip install --no-binary xmlsec xmlsec - name: Initiate Services run: | @@ -78,6 +74,11 @@ jobs: UPDATE mysql.user SET authentication_string = null WHERE user = 'root'; FLUSH PRIVILEGES; EOF + + - name: Install mysqlclient-dev binary + run: | + sudo apt-get update + sudo apt-get install -y libmysqlclient-dev - name: Run Tests env: diff --git a/requirements/constraints.txt b/requirements/constraints.txt index fd79c09c815d..61324782cc06 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -90,3 +90,7 @@ babel==2.11.0 social-auth-app-django==5.0.0 algoliasearch==2.6.3 django-ipware==4.0.2 + +# pytz>2022 has major changes which are causing test failures. +# Pinning this version for now so this could be fixed in a separate PR later on +pytz<2023 diff --git a/requirements/edx-sandbox/py38.txt b/requirements/edx-sandbox/py38.txt index 08f881603dad..3935028963eb 100644 --- a/requirements/edx-sandbox/py38.txt +++ b/requirements/edx-sandbox/py38.txt @@ -38,7 +38,7 @@ matplotlib==3.3.4 # -r requirements/edx-sandbox/py38.in mpmath==1.3.0 # via sympy -networkx==3.0 +networkx==3.1 # via -r requirements/edx-sandbox/py38.in nltk==3.8.1 # via @@ -52,7 +52,7 @@ numpy==1.22.4 # scipy openedx-calc==3.0.1 # via -r requirements/edx-sandbox/py38.in -pillow==9.4.0 +pillow==9.5.0 # via matplotlib pycparser==2.21 # via cffi @@ -66,7 +66,7 @@ python-dateutil==2.8.2 # via matplotlib random2==1.0.1 # via -r requirements/edx-sandbox/py38.in -regex==2022.10.31 +regex==2023.3.23 # via nltk scipy==1.7.3 # via diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d83b55348774..a3bbb45cf8ed 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -16,7 +16,7 @@ algoliasearch==2.6.3 # -r requirements/edx/base.in amqp==5.1.1 # via kombu -analytics-python==1.4.0 +analytics-python==1.4.post1 # via -r requirements/edx/base.in aniso8601==9.0.1 # via edx-tincan-py35 @@ -34,7 +34,7 @@ async-timeout==4.0.2 # via # aiohttp # redis -attrs==22.2.0 +attrs==23.1.0 # via # -r requirements/edx/base.in # aiohttp @@ -53,7 +53,7 @@ backoff==1.10.0 # via analytics-python backports-zoneinfo==0.2.1 # via icalendar -beautifulsoup4==4.11.2 +beautifulsoup4==4.12.2 # via pynliner billiard==3.6.4.0 # via celery @@ -261,7 +261,7 @@ django-config-models==2.3.0 # edx-enterprise # edx-name-affirmation # lti-consumer-xblock -django-cors-headers==3.13.0 +django-cors-headers==3.14.0 # via -r requirements/edx/base.in django-countries==7.5.1 # via @@ -282,7 +282,7 @@ django-fernet-fields==0.6 # via # -r requirements/edx/base.in # edx-enterprise -django-filter==22.1 +django-filter==23.1 # via # -r requirements/edx/base.in # edx-enterprise @@ -467,7 +467,7 @@ edx-django-utils==5.3.0 # ora2 # outcome-surveys # super-csv -edx-drf-extensions==8.4.1 +edx-drf-extensions==8.6.0 # via # -r requirements/edx/base.in # edx-completion @@ -484,7 +484,7 @@ edx-enterprise==3.61.11 # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in # learner-pathway-progress -edx-event-bus-kafka==3.9.4 +edx-event-bus-kafka==3.9.6 # via -r requirements/edx/base.in edx-i18n-tools==0.9.2 # via ora2 @@ -542,6 +542,7 @@ edx-toggles==5.0.0 # edx-completion # edx-event-bus-kafka # edx-name-affirmation + # edx-search # edxval # learner-pathway-progress # ora2 @@ -570,7 +571,7 @@ event-tracking==2.1.0 # edx-search fastavro==1.7.3 # via openedx-events -filelock==3.10.0 +filelock==3.11.0 # via snowflake-connector-python frozenlist==1.3.3 # via @@ -600,7 +601,7 @@ html5lib==1.1 # via # -r requirements/edx/base.in # ora2 -icalendar==5.0.4 +icalendar==5.0.5 # via -r requirements/edx/base.in idna==3.4 # via @@ -609,7 +610,7 @@ idna==3.4 # requests # snowflake-connector-python # yarl -importlib-metadata==6.0.0 +importlib-metadata==6.4.1 # via markdown importlib-resources==5.12.0 # via jsonschema @@ -711,7 +712,7 @@ markupsafe==2.1.2 # xblock maxminddb==2.2.0 # via geoip2 -mock==5.0.1 +mock==5.0.2 # via -r requirements/edx/paver.txt mongodbproxy @ git+https://github.com/openedx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/github.in @@ -731,7 +732,7 @@ mysqlclient==2.1.1 # via # -r requirements/edx/base.in # openedx-blockstore -newrelic==8.7.0 +newrelic==8.8.0 # via # -r requirements/edx/base.in # edx-django-utils @@ -767,7 +768,7 @@ openedx-django-require==2.0.0 # via -r requirements/edx/base.in openedx-django-wiki==2.0.0 # via -r requirements/edx/base.in -openedx-events==5.1.0 +openedx-events==7.0.0 # via # -r requirements/edx/base.in # edx-event-bus-kafka @@ -783,7 +784,7 @@ oscrypto==1.3.0 # via snowflake-connector-python outcome-surveys==2.4.0 # via -r requirements/edx/base.in -packaging==23.0 +packaging==23.1 # via # drf-yasg # py2neo @@ -811,7 +812,7 @@ pgpy==0.6.0 # via edx-enterprise piexif==1.1.3 # via -r requirements/edx/base.in -pillow==9.4.0 +pillow==9.5.0 # via # -r requirements/edx/base.in # edx-enterprise @@ -822,7 +823,7 @@ polib==1.2.0 # via edx-i18n-tools prompt-toolkit==3.0.38 # via click-repl -psutil==5.9.4 +psutil==5.9.5 # via # -r requirements/edx/paver.txt # edx-django-utils @@ -843,7 +844,7 @@ pycryptodomex==3.17 # lti-consumer-xblock # pyjwkest # snowflake-connector-python -pygments==2.14.0 +pygments==2.15.0 # via # -r requirements/edx/base.in # py2neo @@ -917,7 +918,7 @@ python-memcached==1.59 # via -r requirements/edx/paver.txt python-slugify==8.0.1 # via code-annotations -python-swiftclient==4.2.0 +python-swiftclient==4.3.0 # via ora2 python3-openid==3.2.0 ; python_version >= "3" # via @@ -929,6 +930,7 @@ python3-saml==1.9.0 # -r requirements/edx/base.in pytz==2022.7.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in # babel # celery @@ -961,13 +963,13 @@ pyyaml==6.0 # xblock random2==1.0.1 # via -r requirements/edx/base.in -rapidfuzz==2.13.7 +rapidfuzz==2.15.1 # via levenshtein recommender-xblock==2.0.1 # via -r requirements/edx/base.in -redis==4.5.1 +redis==4.5.4 # via -r requirements/edx/base.in -regex==2022.10.31 +regex==2023.3.23 # via nltk requests==2.28.2 # via @@ -1019,7 +1021,7 @@ semantic-version==2.10.0 # via edx-drf-extensions shapely==2.0.1 # via -r requirements/edx/base.in -simplejson==3.18.4 +simplejson==3.19.1 # via # -r requirements/edx/base.in # sailthru-client @@ -1062,7 +1064,7 @@ slumber==0.7.1 # edx-bulk-grades # edx-enterprise # edx-rest-api-client -snowflake-connector-python==3.0.1 +snowflake-connector-python==3.0.2 # via edx-enterprise social-auth-app-django==5.0.0 # via @@ -1081,7 +1083,7 @@ sorl-thumbnail==12.9.0 # openedx-django-wiki sortedcontainers==2.4.0 # via -r requirements/edx/base.in -soupsieve==2.4 +soupsieve==2.4.1 # via beautifulsoup4 sqlparse==0.4.3 # via @@ -1143,7 +1145,7 @@ vine==5.0.0 # kombu voluptuous==0.13.1 # via ora2 -watchdog==2.3.1 +watchdog==3.0.0 # via -r requirements/edx/paver.txt wcwidth==0.2.6 # via prompt-toolkit diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 0192aa373a2f..248e7daa1f3d 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -6,7 +6,7 @@ # chardet==5.1.0 # via diff-cover -coverage==7.2.1 +coverage==7.2.3 # via -r requirements/edx/coverage.in diff-cover==7.5.0 # via -r requirements/edx/coverage.in @@ -16,5 +16,5 @@ markupsafe==2.1.2 # via jinja2 pluggy==1.0.0 # via diff-cover -pygments==2.14.0 +pygments==2.15.0 # via diff-cover diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 64da84e15034..58dc6727674c 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -24,7 +24,7 @@ amqp==5.1.1 # via # -r requirements/edx/testing.txt # kombu -analytics-python==1.4.0 +analytics-python==1.4.post1 # via -r requirements/edx/testing.txt aniso8601==9.0.1 # via @@ -59,7 +59,7 @@ async-timeout==4.0.2 # -r requirements/edx/testing.txt # aiohttp # redis -attrs==22.2.0 +attrs==23.1.0 # via # -r requirements/edx/testing.txt # aiohttp @@ -68,7 +68,6 @@ attrs==22.2.0 # lti-consumer-xblock # openedx-blockstore # openedx-events - # pytest babel==2.11.0 # via # -c requirements/edx/../constraints.txt @@ -84,7 +83,7 @@ backports-zoneinfo==0.2.1 # via # -r requirements/edx/testing.txt # icalendar -beautifulsoup4==4.11.2 +beautifulsoup4==4.12.2 # via # -r requirements/edx/testing.txt # pynliner @@ -220,7 +219,7 @@ coreschema==0.0.4 # -r requirements/edx/testing.txt # coreapi # drf-yasg -coverage[toml]==7.2.1 +coverage[toml]==7.2.3 # via # -r requirements/edx/testing.txt # pytest-cov @@ -363,7 +362,7 @@ django-config-models==2.3.0 # edx-enterprise # edx-name-affirmation # lti-consumer-xblock -django-cors-headers==3.13.0 +django-cors-headers==3.14.0 # via -r requirements/edx/testing.txt django-countries==7.5.1 # via @@ -378,7 +377,7 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-debug-toolbar==3.8.1 +django-debug-toolbar==4.0.0 # via -r requirements/edx/development.in django-environ==0.10.0 # via @@ -388,7 +387,7 @@ django-fernet-fields==0.6 # via # -r requirements/edx/testing.txt # edx-enterprise -django-filter==22.1 +django-filter==23.1 # via # -r requirements/edx/testing.txt # edx-enterprise @@ -593,7 +592,7 @@ edx-django-utils==5.2.0 # ora2 # outcome-surveys # super-csv -edx-drf-extensions==8.4.1 +edx-drf-extensions==8.6.0 # via # -r requirements/edx/testing.txt # edx-completion @@ -610,7 +609,7 @@ edx-enterprise==3.61.11 # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt # learner-pathway-progress -edx-event-bus-kafka==3.9.4 +edx-event-bus-kafka==3.9.6 # via -r requirements/edx/testing.txt edx-i18n-tools==0.9.2 # via @@ -677,6 +676,7 @@ edx-toggles==5.0.0 # edx-completion # edx-event-bus-kafka # edx-name-affirmation + # edx-search # edxval # learner-pathway-progress # ora2 @@ -716,11 +716,11 @@ execnet==1.9.0 # pytest-xdist factory-boy==3.2.1 # via -r requirements/edx/testing.txt -faker==17.6.0 +faker==18.4.0 # via # -r requirements/edx/testing.txt # factory-boy -fastapi==0.94.1 +fastapi==0.95.1 # via # -r requirements/edx/testing.txt # pact-python @@ -728,7 +728,7 @@ fastavro==1.7.3 # via # -r requirements/edx/testing.txt # openedx-events -filelock==3.10.0 +filelock==3.11.0 # via # -r requirements/edx/testing.txt # snowflake-connector-python @@ -786,7 +786,7 @@ httpx==0.23.1 # via # -r requirements/edx/testing.txt # pact-python -icalendar==5.0.4 +icalendar==5.0.5 # via -r requirements/edx/testing.txt idna==3.4 # via @@ -801,7 +801,7 @@ imagesize==1.4.1 # via sphinx import-linter==1.8.0 # via -r requirements/edx/testing.txt -importlib-metadata==6.0.0 +importlib-metadata==6.4.1 # via # -r requirements/edx/testing.txt # markdown @@ -960,7 +960,7 @@ mccabe==0.7.0 # pylint mistune==2.0.5 # via sphinx-mdinclude -mock==5.0.1 +mock==5.0.2 # via -r requirements/edx/testing.txt mongodbproxy @ git+https://github.com/openedx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/testing.txt @@ -980,7 +980,7 @@ multidict==6.0.4 # -r requirements/edx/testing.txt # aiohttp # yarl -mypy==1.1.1 +mypy==1.2.0 # via -r requirements/edx/development.in mypy-extensions==1.0.0 # via mypy @@ -988,7 +988,7 @@ mysqlclient==2.1.1 # via # -r requirements/edx/testing.txt # openedx-blockstore -newrelic==8.7.0 +newrelic==8.8.0 # via # -r requirements/edx/testing.txt # edx-django-utils @@ -1027,7 +1027,7 @@ openedx-django-require==2.0.0 # via -r requirements/edx/testing.txt openedx-django-wiki==2.0.0 # via -r requirements/edx/testing.txt -openedx-events==5.1.0 +openedx-events==7.0.0 # via # -r requirements/edx/testing.txt # edx-event-bus-kafka @@ -1045,7 +1045,7 @@ oscrypto==1.3.0 # snowflake-connector-python outcome-surveys==2.4.0 # via -r requirements/edx/testing.txt -packaging==23.0 +packaging==23.1 # via # -r requirements/edx/../pip-tools.txt # -r requirements/edx/testing.txt @@ -1083,22 +1083,22 @@ pgpy==0.6.0 # via # -r requirements/edx/testing.txt # edx-enterprise -picobox==2.2.0 +picobox==3.0.0 # via sphinxcontrib-openapi piexif==1.1.3 # via -r requirements/edx/testing.txt -pillow==9.4.0 +pillow==9.5.0 # via # -r requirements/edx/testing.txt # edx-enterprise # edx-organizations -pip-tools==6.12.3 +pip-tools==6.13.0 # via -r requirements/edx/../pip-tools.txt pkgutil-resolve-name==1.3.10 # via # -r requirements/edx/testing.txt # jsonschema -platformdirs==3.1.1 +platformdirs==3.2.0 # via # -r requirements/edx/testing.txt # pylint @@ -1117,7 +1117,7 @@ prompt-toolkit==3.0.38 # via # -r requirements/edx/testing.txt # click-repl -psutil==5.9.4 +psutil==5.9.5 # via # -r requirements/edx/testing.txt # edx-django-utils @@ -1152,11 +1152,11 @@ pycryptodomex==3.17 # lti-consumer-xblock # pyjwkest # snowflake-connector-python -pydantic==1.10.6 +pydantic==1.10.7 # via # -r requirements/edx/testing.txt # fastapi -pygments==2.14.0 +pygments==2.15.0 # via # -r requirements/edx/testing.txt # diff-cover @@ -1250,7 +1250,7 @@ pysrt==1.1.2 # via # -r requirements/edx/testing.txt # edxval -pytest==7.2.2 +pytest==7.3.1 # via # -r requirements/edx/testing.txt # pylint-pytest @@ -1300,7 +1300,7 @@ python-slugify==8.0.1 # via # -r requirements/edx/testing.txt # code-annotations -python-swiftclient==4.2.0 +python-swiftclient==4.3.0 # via # -r requirements/edx/testing.txt # ora2 @@ -1314,6 +1314,7 @@ python3-saml==1.9.0 # -r requirements/edx/testing.txt pytz==2022.7.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt # babel # celery @@ -1349,15 +1350,15 @@ pyyaml==6.0 # xblock random2==1.0.1 # via -r requirements/edx/testing.txt -rapidfuzz==2.13.7 +rapidfuzz==2.15.1 # via # -r requirements/edx/testing.txt # levenshtein recommender-xblock==2.0.1 # via -r requirements/edx/testing.txt -redis==4.5.1 +redis==4.5.4 # via -r requirements/edx/testing.txt -regex==2022.10.31 +regex==2023.3.23 # via # -r requirements/edx/testing.txt # nltk @@ -1432,7 +1433,7 @@ semantic-version==2.10.0 # edx-drf-extensions shapely==2.0.1 # via -r requirements/edx/testing.txt -simplejson==3.18.4 +simplejson==3.19.1 # via # -r requirements/edx/testing.txt # sailthru-client @@ -1491,7 +1492,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python==3.0.1 +snowflake-connector-python==3.0.2 # via # -r requirements/edx/testing.txt # edx-enterprise @@ -1512,7 +1513,7 @@ sorl-thumbnail==12.9.0 # openedx-django-wiki sortedcontainers==2.4.0 # via -r requirements/edx/testing.txt -soupsieve==2.4 +soupsieve==2.4.1 # via # -r requirements/edx/testing.txt # beautifulsoup4 @@ -1594,7 +1595,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tomlkit==0.11.6 +tomlkit==0.11.7 # via # -r requirements/edx/testing.txt # pylint @@ -1665,7 +1666,7 @@ voluptuous==0.13.1 # ora2 vulture==2.7 # via -r requirements/edx/development.in -watchdog==2.3.1 +watchdog==3.0.0 # via -r requirements/edx/testing.txt wcwidth==0.2.6 # via diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 2c73cb516713..087cfd909599 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -34,7 +34,7 @@ idna==3.4 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.0.0 +importlib-metadata==6.4.1 # via sphinx jinja2==3.1.2 # via @@ -42,16 +42,18 @@ jinja2==3.1.2 # sphinx markupsafe==2.1.2 # via jinja2 -packaging==23.0 +packaging==23.1 # via sphinx pbr==5.11.1 # via stevedore -pygments==2.14.0 +pygments==2.15.0 # via sphinx python-slugify==8.0.1 # via code-annotations pytz==2022.7.1 - # via babel + # via + # -c requirements/edx/../constraints.txt + # babel pyyaml==6.0 # via code-annotations requests==2.28.2 diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index 28aff7e3ec5a..fe1b04e27b1c 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -20,7 +20,7 @@ libsass==0.10.0 # via -r requirements/edx/paver.in markupsafe==2.1.2 # via -r requirements/edx/paver.in -mock==5.0.1 +mock==5.0.2 # via -r requirements/edx/paver.in path==16.6.0 # via -r requirements/edx/paver.in @@ -28,7 +28,7 @@ paver==1.3.4 # via -r requirements/edx/paver.in pbr==5.11.1 # via stevedore -psutil==5.9.4 +psutil==5.9.5 # via -r requirements/edx/paver.in pymongo==3.13.0 # via @@ -50,7 +50,7 @@ stevedore==5.0.0 # edx-opaque-keys urllib3==1.26.15 # via requests -watchdog==2.3.1 +watchdog==3.0.0 # via -r requirements/edx/paver.in wrapt==1.15.0 # via -r requirements/edx/paver.in diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index b1901213a7ac..7bf33a176d50 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -22,7 +22,7 @@ amqp==5.1.1 # via # -r requirements/edx/base.txt # kombu -analytics-python==1.4.0 +analytics-python==1.4.post1 # via -r requirements/edx/base.txt aniso8601==9.0.1 # via @@ -55,7 +55,7 @@ async-timeout==4.0.2 # -r requirements/edx/base.txt # aiohttp # redis -attrs==22.2.0 +attrs==23.1.0 # via # -r requirements/edx/base.txt # aiohttp @@ -64,8 +64,6 @@ attrs==22.2.0 # lti-consumer-xblock # openedx-blockstore # openedx-events - # outcome - # pytest babel==2.11.0 # via # -c requirements/edx/../constraints.txt @@ -80,7 +78,7 @@ backports-zoneinfo==0.2.1 # via # -r requirements/edx/base.txt # icalendar -beautifulsoup4==4.11.2 +beautifulsoup4==4.12.2 # via # -r requirements/edx/base.txt # -r requirements/edx/testing.in @@ -210,7 +208,7 @@ coreschema==0.0.4 # -r requirements/edx/base.txt # coreapi # drf-yasg -coverage[toml]==7.2.1 +coverage[toml]==7.2.3 # via # -r requirements/edx/coverage.txt # pytest-cov @@ -346,7 +344,7 @@ django-config-models==2.3.0 # edx-enterprise # edx-name-affirmation # lti-consumer-xblock -django-cors-headers==3.13.0 +django-cors-headers==3.14.0 # via -r requirements/edx/base.txt django-countries==7.5.1 # via @@ -369,7 +367,7 @@ django-fernet-fields==0.6 # via # -r requirements/edx/base.txt # edx-enterprise -django-filter==22.1 +django-filter==23.1 # via # -r requirements/edx/base.txt # edx-enterprise @@ -572,7 +570,7 @@ edx-django-utils==5.2.0 # ora2 # outcome-surveys # super-csv -edx-drf-extensions==8.4.1 +edx-drf-extensions==8.6.0 # via # -r requirements/edx/base.txt # edx-completion @@ -589,7 +587,7 @@ edx-enterprise==3.61.11 # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # learner-pathway-progress -edx-event-bus-kafka==3.9.4 +edx-event-bus-kafka==3.9.6 # via -r requirements/edx/base.txt edx-i18n-tools==0.9.2 # via @@ -655,6 +653,7 @@ edx-toggles==5.0.0 # edx-completion # edx-event-bus-kafka # edx-name-affirmation + # edx-search # edxval # learner-pathway-progress # ora2 @@ -690,15 +689,15 @@ execnet==1.9.0 # via pytest-xdist factory-boy==3.2.1 # via -r requirements/edx/testing.in -faker==17.6.0 +faker==18.4.0 # via factory-boy -fastapi==0.94.1 +fastapi==0.95.1 # via pact-python fastavro==1.7.3 # via # -r requirements/edx/base.txt # openedx-events -filelock==3.10.0 +filelock==3.11.0 # via # -r requirements/edx/base.txt # snowflake-connector-python @@ -749,7 +748,7 @@ httpretty==1.1.4 # via -r requirements/edx/testing.in httpx==0.23.1 # via pact-python -icalendar==5.0.4 +icalendar==5.0.5 # via -r requirements/edx/base.txt idna==3.4 # via @@ -762,7 +761,7 @@ idna==3.4 # yarl import-linter==1.8.0 # via -r requirements/edx/testing.in -importlib-metadata==6.0.0 +importlib-metadata==6.4.1 # via # -r requirements/edx/base.txt # markdown @@ -912,7 +911,7 @@ maxminddb==2.2.0 # geoip2 mccabe==0.7.0 # via pylint -mock==5.0.1 +mock==5.0.2 # via -r requirements/edx/base.txt mongodbproxy @ git+https://github.com/openedx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/base.txt @@ -936,7 +935,7 @@ mysqlclient==2.1.1 # via # -r requirements/edx/base.txt # openedx-blockstore -newrelic==8.7.0 +newrelic==8.8.0 # via # -r requirements/edx/base.txt # edx-django-utils @@ -975,7 +974,7 @@ openedx-django-require==2.0.0 # via -r requirements/edx/base.txt openedx-django-wiki==2.0.0 # via -r requirements/edx/base.txt -openedx-events==5.1.0 +openedx-events==7.0.0 # via # -r requirements/edx/base.txt # edx-event-bus-kafka @@ -993,7 +992,7 @@ oscrypto==1.3.0 # snowflake-connector-python outcome-surveys==2.4.0 # via -r requirements/edx/base.txt -packaging==23.0 +packaging==23.1 # via # -r requirements/edx/base.txt # drf-yasg @@ -1030,7 +1029,7 @@ pgpy==0.6.0 # edx-enterprise piexif==1.1.3 # via -r requirements/edx/base.txt -pillow==9.4.0 +pillow==9.5.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1039,7 +1038,7 @@ pkgutil-resolve-name==1.3.10 # via # -r requirements/edx/base.txt # jsonschema -platformdirs==3.1.1 +platformdirs==3.2.0 # via # pylint # virtualenv @@ -1058,7 +1057,7 @@ prompt-toolkit==3.0.38 # via # -r requirements/edx/base.txt # click-repl -psutil==5.9.4 +psutil==5.9.5 # via # -r requirements/edx/base.txt # edx-django-utils @@ -1091,9 +1090,9 @@ pycryptodomex==3.17 # lti-consumer-xblock # pyjwkest # snowflake-connector-python -pydantic==1.10.6 +pydantic==1.10.7 # via fastapi -pygments==2.14.0 +pygments==2.15.0 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt @@ -1176,7 +1175,7 @@ pysrt==1.1.2 # via # -r requirements/edx/base.txt # edxval -pytest==7.2.2 +pytest==7.3.1 # via # -r requirements/edx/testing.in # pylint-pytest @@ -1226,7 +1225,7 @@ python-slugify==8.0.1 # via # -r requirements/edx/base.txt # code-annotations -python-swiftclient==4.2.0 +python-swiftclient==4.3.0 # via # -r requirements/edx/base.txt # ora2 @@ -1240,6 +1239,7 @@ python3-saml==1.9.0 # -r requirements/edx/base.txt pytz==2022.7.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # babel # celery @@ -1272,15 +1272,15 @@ pyyaml==6.0 # xblock random2==1.0.1 # via -r requirements/edx/base.txt -rapidfuzz==2.13.7 +rapidfuzz==2.15.1 # via # -r requirements/edx/base.txt # levenshtein recommender-xblock==2.0.1 # via -r requirements/edx/base.txt -redis==4.5.1 +redis==4.5.4 # via -r requirements/edx/base.txt -regex==2022.10.31 +regex==2023.3.23 # via # -r requirements/edx/base.txt # nltk @@ -1352,7 +1352,7 @@ semantic-version==2.10.0 # edx-drf-extensions shapely==2.0.1 # via -r requirements/edx/base.txt -simplejson==3.18.4 +simplejson==3.19.1 # via # -r requirements/edx/base.txt # sailthru-client @@ -1406,7 +1406,7 @@ sniffio==1.3.0 # anyio # httpcore # httpx -snowflake-connector-python==3.0.1 +snowflake-connector-python==3.0.2 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1427,7 +1427,7 @@ sorl-thumbnail==12.9.0 # openedx-django-wiki sortedcontainers==2.4.0 # via -r requirements/edx/base.txt -soupsieve==2.4 +soupsieve==2.4.1 # via # -r requirements/edx/base.txt # beautifulsoup4 @@ -1476,7 +1476,7 @@ tomli==2.0.1 # pylint # pytest # tox -tomlkit==0.11.6 +tomlkit==0.11.7 # via pylint tox==3.28.0 # via @@ -1538,7 +1538,7 @@ voluptuous==0.13.1 # via # -r requirements/edx/base.txt # ora2 -watchdog==2.3.1 +watchdog==3.0.0 # via -r requirements/edx/base.txt wcwidth==0.2.6 # via diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index f434fbb8b915..cda5abf53760 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -10,9 +10,9 @@ click==8.1.3 # via # -c requirements/constraints.txt # pip-tools -packaging==23.0 +packaging==23.1 # via build -pip-tools==6.12.3 +pip-tools==6.13.0 # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 # via build diff --git a/requirements/pip.txt b/requirements/pip.txt index 45fb600be5f0..4b902cbc3468 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -8,7 +8,7 @@ wheel==0.40.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.0.1 +pip==23.1 # via -r requirements/pip.in -setuptools==67.6.0 +setuptools==67.6.1 # via -r requirements/pip.in From 62652c742260b221c1ab39953c2504d06503cd70 Mon Sep 17 00:00:00 2001 From: Muhammad Faraz Maqsood Date: Tue, 20 Jun 2023 15:30:27 +0500 Subject: [PATCH 11/12] enhancement: add support for caching programs for one site (#32380) (#32506) * perf!: add support for caching programs on per site bases --- .../management/commands/cache_programs.py | 13 +++- .../commands/tests/test_cache_programs.py | 77 +++++++++++++++---- 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/openedx/core/djangoapps/catalog/management/commands/cache_programs.py b/openedx/core/djangoapps/catalog/management/commands/cache_programs.py index b815f8b4e4e5..9e0664f4930f 100644 --- a/openedx/core/djangoapps/catalog/management/commands/cache_programs.py +++ b/openedx/core/djangoapps/catalog/management/commands/cache_programs.py @@ -45,8 +45,17 @@ class Command(BaseCommand): """ help = "Rebuild the LMS' cache of program data." + def add_arguments(self, parser): + parser.add_argument( + '--domain', + dest='domain', + type=str, + help='Help in caching the programs for one site' + ) + # lint-amnesty, pylint: disable=bad-option-value, unicode-format-string def handle(self, *args, **options): # lint-amnesty, pylint: disable=too-many-statements + domain = options.get('domain', '') failure = False logger.info('populate-multitenant-programs switch is ON') @@ -68,7 +77,9 @@ def handle(self, *args, **options): # lint-amnesty, pylint: disable=too-many-st programs_by_type = {} programs_by_type_slug = {} organizations = {} - for site in Site.objects.all(): + + sites = Site.objects.filter(domain=domain) if domain else Site.objects.all() + for site in sites: site_config = getattr(site, 'configuration', None) if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'): logger.info(f'Skipping site {site.domain}. No configuration.') diff --git a/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py b/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py index 331d53a73042..4078b2d90d55 100644 --- a/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py +++ b/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py @@ -48,29 +48,53 @@ def setUp(self): } ) + self.site_domain2 = 'testsite2.com' + self.site2 = self.set_up_site( + self.site_domain2, + { + 'COURSE_CATALOG_API_URL': self.catalog_integration.get_internal_api_url().rstrip('/') + } + ) + self.list_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/programs/' self.detail_tpl = self.list_url.rstrip('/') + '/{uuid}/' self.pathway_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/pathways/' self.programs = ProgramFactory.create_batch(3) + self.programs2 = ProgramFactory.create_batch(3) self.pathways = PathwayFactory.create_batch(3) + self.pathways2 = PathwayFactory.create_batch(3) self.child_program = ProgramFactory.create() + self.child_program2 = ProgramFactory.create() self.programs[0]['curricula'][0]['programs'].append(self.child_program) self.programs.append(self.child_program) - self.programs[0]['authoring_organizations'] = OrganizationFactory.create_batch(2) + self.programs2[0]['curricula'][0]['programs'].append(self.child_program2) + self.programs2.append(self.child_program2) + self.programs2[0]['authoring_organizations'] = OrganizationFactory.create_batch(2) + for pathway in self.pathways: self.programs += pathway['programs'] - self.uuids = [program['uuid'] for program in self.programs] + for pathway in self.pathways2: + self.programs2 += pathway['programs'] + + self.uuids = { + f"{self.site_domain}": [program["uuid"] for program in self.programs], + f"{self.site_domain2}": [program["uuid"] for program in self.programs2], + } # add some of the previously created programs to some pathways self.pathways[0]['programs'].extend([self.programs[0], self.programs[1]]) self.pathways[1]['programs'].append(self.programs[0]) - def mock_list(self): + # add some of the previously created programs to some pathways + self.pathways2[0]['programs'].extend([self.programs2[0], self.programs2[1]]) + self.pathways2[1]['programs'].append(self.programs2[0]) + + def mock_list(self, site=""): """ Mock the data returned by the program listing API endpoint. """ # pylint: disable=unused-argument def list_callback(request, uri, headers): @@ -81,8 +105,8 @@ def list_callback(request, uri, headers): 'uuids_only': ['1'] } assert request.querystring == expected - - return (200, headers, json.dumps(self.uuids)) + uuids = self.uuids[self.site_domain2] if site else self.uuids[self.site_domain] + return (200, headers, json.dumps(uuids)) httpretty.register_uri( httpretty.GET, @@ -130,17 +154,36 @@ def pathways_callback(request, uri, headers): # pylint: disable=unused-argument return (200, headers, json.dumps(body)) - # NOTE: httpretty does not properly match against query strings here (using match_querystring arg) - # as such, it does not actually look at the query parameters (for page num), but returns items in a LIFO order. - # this means that for multiple pages, you must call this function starting from the last page. - # we do assert the page number query param above, however httpretty.register_uri( httpretty.GET, - self.pathway_url, + self.pathway_url + f'?exclude_utm=1&page={page_number}', body=pathways_callback, content_type='application/json', + match_querystring=True, ) + def test_handle_domain(self): + """ + Verify that the command argument is working correct or not. + """ + UserFactory(username=self.catalog_integration.service_username) + + programs = { + PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs2 + } + + self.mock_list(self.site2) + self.mock_pathways(self.pathways2) + + for uuid in self.uuids[self.site_domain2]: + program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)] + self.mock_detail(uuid, program) + + call_command('cache_programs', f'--domain={self.site_domain2}') + + cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain2)) + assert set(cached_uuids) == set(self.uuids[self.site_domain2]) + def test_handle_programs(self): """ Verify that the command requests and caches program UUIDs and details. @@ -158,14 +201,14 @@ def test_handle_programs(self): self.mock_list() self.mock_pathways(self.pathways) - for uuid in self.uuids: + for uuid in self.uuids[self.site_domain]: program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)] self.mock_detail(uuid, program) call_command('cache_programs') cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain)) - assert set(cached_uuids) == set(self.uuids) + assert set(cached_uuids) == set(self.uuids[self.site_domain]) program_keys = list(programs.keys()) cached_programs = cache.get_many(program_keys) @@ -228,7 +271,7 @@ def test_handle_pathways(self): self.mock_list() self.mock_pathways(self.pathways) - for uuid in self.uuids: + for uuid in self.uuids[self.site_domain]: program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)] self.mock_detail(uuid, program) @@ -267,7 +310,7 @@ def test_pathways_multiple_pages(self): } self.mock_list() - for uuid in self.uuids: + for uuid in self.uuids[self.site_domain]: program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)] self.mock_detail(uuid, program) @@ -337,7 +380,7 @@ def test_handle_missing_pathways(self): self.mock_list() - for uuid in self.uuids: + for uuid in self.uuids[self.site_domain]: program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)] self.mock_detail(uuid, program) @@ -365,7 +408,7 @@ def test_handle_missing_programs(self): self.mock_list() - for uuid in self.uuids[:2]: + for uuid in self.uuids[self.site_domain][:2]: program = partial_programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)] self.mock_detail(uuid, program) @@ -375,7 +418,7 @@ def test_handle_missing_programs(self): assert context.value.code == 1 cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain)) - assert set(cached_uuids) == set(self.uuids) + assert set(cached_uuids) == set(self.uuids[self.site_domain]) program_keys = list(all_programs.keys()) cached_programs = cache.get_many(program_keys) From 42e84e386e833f6707eb2b6d4371ea8bd6cc693a Mon Sep 17 00:00:00 2001 From: ihor-romaniuk Date: Wed, 28 Sep 2022 18:03:46 +0300 Subject: [PATCH 12/12] fix: save scroll position on exit from video xblock fullscreen mode --- xmodule/js/src/video/04_video_full_screen.js | 23 ++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/xmodule/js/src/video/04_video_full_screen.js b/xmodule/js/src/video/04_video_full_screen.js index 0730300d105e..9da85d8dfb67 100644 --- a/xmodule/js/src/video/04_video_full_screen.js +++ b/xmodule/js/src/video/04_video_full_screen.js @@ -161,7 +161,21 @@ return this.videoFullScreen.height; } - /** + function notifyParent(fullscreenOpen) { + if (window !== window.parent) { + // This is used by the Learning MFE to know about changing fullscreen mode. + // The MFE is then able to respond appropriately and scroll window to the previous position. + window.parent.postMessage({ + type: 'plugin.videoFullScreen', + payload: { + open: fullscreenOpen + } + }, document.referrer + ); + } + } + + /** * Event handler to toggle fullscreen mode. * @param {jquery Event} event */ @@ -192,6 +206,8 @@ this.resizer.delta.reset().setMode('width'); } this.el.trigger('fullscreen', [this.isFullScreen]); + + this.videoFullScreen.notifyParent(false); } function handleEnter() { @@ -202,6 +218,8 @@ return; } + this.videoFullScreen.notifyParent(true); + this.videoFullScreen.fullScreenState = this.isFullScreen = true; fullScreenClassNameEl.addClass('video-fullscreen'); this.videoFullScreen.fullScreenEl @@ -267,7 +285,8 @@ handleFullscreenChange: handleFullscreenChange, toggle: toggle, toggleHandler: toggleHandler, - updateControlsHeight: updateControlsHeight + updateControlsHeight: updateControlsHeight, + notifyParent: notifyParent }; state.bindTo(methodsDict, state.videoFullScreen, state);