From e74f608f28b8f13866e81252e54c55fc47e26052 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Sat, 18 Nov 2023 14:40:07 +0100 Subject: [PATCH 1/4] feat: add algoliasearch in base.in --- Makefile | 7 +++++-- requirements/base.in | 1 + requirements/dev.txt | 6 ++++++ requirements/doc.txt | 3 +++ requirements/edx-platform-constraints.txt | 8 ++++++-- requirements/test-master.txt | 5 +++++ requirements/test.txt | 3 +++ 7 files changed, 29 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 270de80fd3..fe0a1ff5ed 100644 --- a/Makefile +++ b/Makefile @@ -74,11 +74,14 @@ docs: ## generate Sphinx HTML documentation, including API docs # Define PIP_COMPILE_OPTS=-v to get more information during make upgrade. PIP_COMPILE = pip-compile --upgrade --rebuild $(PIP_COMPILE_OPTS) LOCAL_EDX_PINS = requirements/edx-platform-constraints.txt -PLATFORM_BASE_REQS = https://raw.githubusercontent.com/edx/edx-platform/master/requirements/edx/base.txt +# edx-enterprise can't work alone and thus doesn't have it's own requirements. It pulls base requirements +# file from edx-platform and constrains it's own requirements (mostly for the test environment) using it. +PLATFORM_BASE_REQS = https://raw.githubusercontent.com/open-craft/edx-platform/opencraft-release/nutmeg.2/requirements/edx/base.txt COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt .PHONY: $(COMMON_CONSTRAINTS_TXT) $(COMMON_CONSTRAINTS_TXT): - wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)" + # Requirements were upgraded 15 February 2022. This commit was added to `edx-lint` on 14 February 2022. + wget -O "$(@)" https://raw.githubusercontent.com/openedx/edx-lint/7dce0b15b1e8ef402bc8ad0d160a4a2516d079fe/edx_lint/files/common_constraints.txt || touch "$(@)" echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@) check_pins: $(COMMON_CONSTRAINTS_TXT) ## check that our local copy of edx-platform pins is accurate diff --git a/requirements/base.in b/requirements/base.in index 618db96b37..2927b083c3 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,6 +2,7 @@ # This file contains the dependencies explicitly needed for this library. # # Packages directly used by this library that we do not need pinned to a specific version. +algoliasearch bleach celery code-annotations diff --git a/requirements/dev.txt b/requirements/dev.txt index 6dc56c542b..384577137e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,6 +8,11 @@ alabaster==0.7.12 # via # -r requirements/doc.txt # sphinx +algoliasearch==2.6.3 + # via + # -r requirements/doc.txt + # -r requirements/test-master.txt + # -r requirements/test.txt amqp==5.0.9 # via # -r requirements/doc.txt @@ -561,6 +566,7 @@ requests==2.27.1 # -r requirements/doc.txt # -r requirements/test-master.txt # -r requirements/test.txt + # algoliasearch # edx-drf-extensions # edx-rest-api-client # pyjwkest diff --git a/requirements/doc.txt b/requirements/doc.txt index c381cc744a..950b472bf0 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -6,6 +6,8 @@ # alabaster==0.7.12 # via sphinx +algoliasearch==2.6.3 + # via -r requirements/test-master.txt amqp==5.0.9 # via # -r requirements/test-master.txt @@ -279,6 +281,7 @@ readme-renderer==32.0 requests==2.27.1 # via # -r requirements/test-master.txt + # algoliasearch # edx-drf-extensions # edx-rest-api-client # pyjwkest diff --git a/requirements/edx-platform-constraints.txt b/requirements/edx-platform-constraints.txt index 202eecc650..facc71973c 100644 --- a/requirements/edx-platform-constraints.txt +++ b/requirements/edx-platform-constraints.txt @@ -25,6 +25,8 @@ aiohttp==3.8.1 # via geoip2 aiosignal==1.2.0 # via aiohttp +algoliasearch==2.6.3 + # via -r requirements/edx/base.in # via kombu analytics-python==1.4.0 # via -r requirements/edx/base.in @@ -685,7 +687,8 @@ openedx-events==0.7.1 # via -r requirements/edx/base.in openedx-filters==0.4.3 # via -r requirements/edx/base.in -ora2==3.8.2 +# TODO: Backport https://github.com/openedx/edx-ora2/pull/1869 to Olive if it uses a lower version than 4.5.0. +ora2 @ git+https://github.com/open-craft/edx-ora2@agrendalath/bb-6151-nutmeg_backport # via -r requirements/edx/base.in packaging==21.3 # via @@ -725,7 +728,7 @@ psutil==5.9.0 # via # -r requirements/edx/paver.txt # edx-django-utils -py2neo==2021.2.3 +py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in @@ -855,6 +858,7 @@ regex==2022.1.18 requests==2.27.1 # via # -r requirements/edx/paver.txt + # algoliasearch # analytics-python # coreapi # django-oauth-toolkit diff --git a/requirements/test-master.txt b/requirements/test-master.txt index 57b0c70a83..abb686afb6 100644 --- a/requirements/test-master.txt +++ b/requirements/test-master.txt @@ -4,6 +4,10 @@ # # make upgrade # +algoliasearch==2.6.3 + # via + # -c requirements/edx-platform-constraints.txt + # -r requirements/base.in amqp==5.0.9 # via kombu aniso8601==9.0.1 @@ -281,6 +285,7 @@ requests==2.27.1 # via # -c requirements/edx-platform-constraints.txt # -r requirements/base.in + # algoliasearch # edx-drf-extensions # edx-rest-api-client # pyjwkest diff --git a/requirements/test.txt b/requirements/test.txt index 00917d8189..4a5d3cd225 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,6 +4,8 @@ # # make upgrade # +algoliasearch==2.6.3 + # via -r requirements/test-master.txt # via # -r requirements/test-master.txt # kombu @@ -287,6 +289,7 @@ pyyaml==6.0 requests==2.27.1 # via # -r requirements/test-master.txt + # algoliasearch # edx-drf-extensions # edx-rest-api-client # pyjwkest From e67e61ef39c57c226bc7099cac1085c010a20c23 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Fri, 10 Nov 2023 15:03:06 +0100 Subject: [PATCH 2/4] feat: per-user secured Algolia API keys Co-authored-by: Piotr Surowiec (cherry picked from commit 3c9c3dc2f888bfbd18c58156825789fc7ea9d7f9) --- enterprise/api/v1/views.py | 39 ++++++++++++++++++++ enterprise/settings/test.py | 2 ++ tests/test_enterprise/api/test_views.py | 48 +++++++++++++++++++++++++ 3 files changed, 89 insertions(+) diff --git a/enterprise/api/v1/views.py b/enterprise/api/v1/views.py index 8859ef60e5..5b9dadc724 100644 --- a/enterprise/api/v1/views.py +++ b/enterprise/api/v1/views.py @@ -7,6 +7,7 @@ from urllib.parse import quote_plus, unquote import requests +from algoliasearch.search_client import SearchClient from django_filters.rest_framework import DjangoFilterBackend from edx_rbac.decorators import permission_required from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication @@ -451,6 +452,44 @@ def toggle_universal_link(self, request, pk=None): headers = self.get_success_headers(response_body) return Response(response_body, status=HTTP_200_OK, headers=headers) + @action(detail=False) + def algolia_key(self, request, *args, **kwargs): + """ + Returns an Algolia API key that is secured to only allow searching for + objects associated with enterprise customers that the user is linked to. + + This endpoint is used with `frontend-app-learner-portal-enterprise` MFE + currently. + """ + + if not (api_key := getattr(settings, "ENTERPRISE_ALGOLIA_SEARCH_API_KEY", "")): + LOGGER.warning("Algolia search API key is not configured. To enable this view, " + "set `ENTERPRISE_ALGOLIA_SEARCH_API_KEY` in settings.") + raise Http404 + + queryset = self.queryset.filter( + **{ + self.USER_ID_FILTER: request.user.id, + "enterprise_customer_users__linked": True + } + ).values_list("uuid", flat=True) + + if len(queryset) == 0: + raise NotFound(_("User is not linked to any enterprise customers.")) + + secured_key = SearchClient.generate_secured_api_key( + api_key, + { + "filters": " OR ".join( + f"enterprise_customer_uuids:{enterprise_customer_uuid}" + for enterprise_customer_uuid + in queryset + ), + } + ) + + return Response({"key": secured_key}, status=HTTP_200_OK) + class EnterpriseCourseEnrollmentViewSet(EnterpriseReadWriteModelViewSet): """ diff --git a/enterprise/settings/test.py b/enterprise/settings/test.py index bd426d34ff..dc2be2c857 100644 --- a/enterprise/settings/test.py +++ b/enterprise/settings/test.py @@ -218,6 +218,8 @@ def root(*args): 'status': 'published' } +ENTERPRISE_ALGOLIA_SEARCH_API_KEY = 'test' + TABLEAU_URL = 'http://localhost' TABLEAU_ADMIN_USER = 'edx' TABLEAU_ADMIN_USER_PASSWORD = 'edx' diff --git a/tests/test_enterprise/api/test_views.py b/tests/test_enterprise/api/test_views.py index 0445a10ab8..a46057d694 100644 --- a/tests/test_enterprise/api/test_views.py +++ b/tests/test_enterprise/api/test_views.py @@ -2,6 +2,7 @@ Tests for the `edx-enterprise` api module. """ +import base64 import json import uuid from datetime import datetime, timedelta @@ -114,6 +115,7 @@ ENTERPRISE_CUSTOMER_REPORTING_ENDPOINT = reverse('enterprise-customer-reporting-list') ENTERPRISE_LEARNER_LIST_ENDPOINT = reverse('enterprise-learner-list') ENTERPRISE_CUSTOMER_WITH_ACCESS_TO_ENDPOINT = reverse('enterprise-customer-with-access-to') +ENTERPRISE_CUSTOMER_ALGOLIA_KEY_ENDPOINT = reverse('enterprise-customer-algolia-key') PENDING_ENTERPRISE_LEARNER_LIST_ENDPOINT = reverse('pending-enterprise-learner-list') LICENSED_ENTERPISE_COURSE_ENROLLMENTS_REVOKE_ENDPOINT = reverse( 'licensed-enterprise-course-enrollment-license-revoke' @@ -1608,6 +1610,52 @@ def test_partial_update(self, enterprise_role, enterprise_uuid_for_role, expecte enterprise_customer.refresh_from_db() assert enterprise_customer.slug == 'new-slug' + def test_algolia_key(self): + """ + Tests that the endpoint algolia_key endpoint returns the correct secured key. + """ + + # Test that the endpoint returns 401 if the user is not logged in. + self.client.logout() + response = self.client.get(ENTERPRISE_CUSTOMER_ALGOLIA_KEY_ENDPOINT) + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + username = 'test_learner_portal_user' + self.create_user(username=username, is_staff=False) + self.client.login(username=username, password=TEST_PASSWORD) + + # Test that the endpoint returns 404 if the Algolia Search API key is not set. + with override_settings(ENTERPRISE_ALGOLIA_SEARCH_API_KEY=None): + response = self.client.get(ENTERPRISE_CUSTOMER_ALGOLIA_KEY_ENDPOINT) + assert response.status_code == status.HTTP_404_NOT_FOUND + + # Test that the endpoint returns 404 if the user is not linked to any enterprise. + response = self.client.get(ENTERPRISE_CUSTOMER_ALGOLIA_KEY_ENDPOINT) + assert response.status_code == status.HTTP_404_NOT_FOUND + + # Test that the endpoint returns 200 if the user is linked to at least one enterprise. + enterprise_customer_1 = factories.EnterpriseCustomerFactory(uuid=FAKE_UUIDS[0]) + enterprise_customer_2 = factories.EnterpriseCustomerFactory(uuid=FAKE_UUIDS[1]) + factories.EnterpriseCustomerFactory(uuid=FAKE_UUIDS[2]) # extra unlinked enterprise + + factories.EnterpriseCustomerUserFactory( + user_id=self.user.id, + enterprise_customer=enterprise_customer_1 + ) + factories.EnterpriseCustomerUserFactory( + user_id=self.user.id, + enterprise_customer=enterprise_customer_2 + ) + + response = self.client.get(ENTERPRISE_CUSTOMER_ALGOLIA_KEY_ENDPOINT) + assert response.status_code == status.HTTP_200_OK + + # Test that the endpoint returns the key encoding correct filters. + decoded_key = base64.b64decode(response.json()["key"]).decode("utf-8") + assert decoded_key.endswith( + f"filters=enterprise_customer_uuids%3A{FAKE_UUIDS[0]}+OR+enterprise_customer_uuids%3A{FAKE_UUIDS[1]}" + ) + @ddt.ddt @mark.django_db From d6872b58ed60a88741dab15d6db38944c2611096 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Sat, 18 Nov 2023 15:00:41 +0100 Subject: [PATCH 3/4] build: https://github.com/mitsuhiko/rye/issues/368 fix --- .github/workflows/mysql8-migrations.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/mysql8-migrations.yml b/.github/workflows/mysql8-migrations.yml index 37c7798f15..5ab98a2fd1 100644 --- a/.github/workflows/mysql8-migrations.yml +++ b/.github/workflows/mysql8-migrations.yml @@ -41,6 +41,10 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('requirements/pip_tools.txt') }} restore-keys: ${{ runner.os }}-pip- + - name: Downgrade pip to work around https://github.com/mitsuhiko/rye/issues/368 + run: | + pip install --upgrade pip==23.1 + - name: Ubuntu and sql Versions run: | lsb_release -a From b0664c2fcbfaf7544d4f676f5e08f16f227fabe6 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Tue, 21 Nov 2023 14:51:53 +0100 Subject: [PATCH 4/4] fix: pin algoliasearch dependency --- requirements/base.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.in b/requirements/base.in index 2927b083c3..f7696c7c6c 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,7 +2,7 @@ # This file contains the dependencies explicitly needed for this library. # # Packages directly used by this library that we do not need pinned to a specific version. -algoliasearch +algoliasearch<3.0.0 bleach celery code-annotations