feat: adds xApiTransforms for completion aggregator events (#205)

* chore: adds edx-event-routing-backends requirement @ 9.3.0
* chore: adds 'factory' as a test requirement because ERB tests needs it.
* feat: adds xApiTransforms for completion aggregator events
* feat: adds the completion_aggregator events to the event tracking whitelist in plugin settings
* test: adds transformer and plugin_settings tests
* test: make COMPLETION_AGGREGATOR_ASYNC_AGGREGATION consistent between test settings and plugin settings.
* chore: bumps version to 4.2.0
* docs: adds note about xAPI to README

---------

Co-authored-by: andrey-canon <[email protected]>

.gitignore

@@ -33,6 +33,7 @@ pip-log.txt
 .tox
 coverage.xml
 htmlcov/
+test_output/*.json
 
 # Translations
 *.mo

CHANGELOG.rst

@@ -14,6 +14,12 @@ Change Log
 Unreleased
 ~~~~~~~~~~
 
+[4.2.0] - 2024-06-21
+~~~~~~~~~~~~~~~~~~~~
+
+* Transform `openedx.completion_aggregator.progress.*` tracking log events into xAPI using edx-event-routing-backends so
+  they can be included in Aspects analytics data.
+
 [4.1.0] - 2024-06-18
 ~~~~~~~~~~~~~~~~~~~~
 

README.rst

@@ -50,7 +50,7 @@ For details about how the completion aggregator's REST APIs can be used, please
 Event tracking
 --------------
 
-Like other parts of Open edX, the completion aggregator emits "tracking logs" events whenever completion aggregator records are created or updated by this plugin. These events can be used for analytics, for example to track learner progress in a course.
+Like other parts of Open edX, the completion aggregator emits "tracking logs" events whenever completion aggregator records are created or updated by this plugin. These events are transformed into xAPI and routed using `edx-event-routing-backends` so they can be used for analytics, for example to track learner progress in a course.
 
 Event tracking is enabled by default for edx-platform, and so event tracking is also enabled by default in the completion aggregator. This can result in a lot of events being generated — for example when a user completes the final block in a course, aggregator completion events will be generated for the containing unit, subsection, section, and course.
 

completion_aggregator/__init__.py

@@ -5,4 +5,4 @@
 
 from __future__ import absolute_import, unicode_literals
 
-__version__ = '4.1.0'
+__version__ = '4.2.0'

completion_aggregator/apps.py

@@ -38,6 +38,10 @@ def ready(self):
         """
         Load signal handlers when the app is ready.
         """
+        # pylint: disable=import-outside-toplevel
         from . import signals
         signals.register()
-        from .tasks import aggregation_tasks, handler_tasks  # pylint: disable=unused-import
+
+        # pylint: disable=unused-import
+        from . import xapi
+        from .tasks import aggregation_tasks, handler_tasks

completion_aggregator/settings/common.py

@@ -4,6 +4,8 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from event_routing_backends.utils.settings import event_tracking_backends_config
+
 
 def plugin_settings(settings):
     """
@@ -42,3 +44,22 @@ def plugin_settings(settings):
     # 1. All courses should be reaggregated for the changes to take effect.
     # 2. It's not possible to revert this change by reaggregation without manually removing existing Aggregators.
     settings.COMPLETION_AGGREGATOR_AGGREGATE_UNRELEASED_BLOCKS = False
+
+    # Whitelist the aggregator events for use with event routing backends xAPI backend.
+    # If these settings don't already exist, then ERB hasn't been loaded yet, so we need to set them to empty lists.
+    # But once ERB does load it will append its events to our list, preserving what we added here.
+    if not hasattr(settings, 'EVENT_TRACKING_BACKENDS_ALLOWED_XAPI_EVENTS'):
+        settings.EVENT_TRACKING_BACKENDS_ALLOWED_XAPI_EVENTS = []
+    if not hasattr(settings, 'EVENT_TRACKING_BACKENDS_ALLOWED_CALIPER_EVENTS'):
+        settings.EVENT_TRACKING_BACKENDS_ALLOWED_CALIPER_EVENTS = []
+    enabled_aggregator_events = [
+        f'openedx.completion_aggregator.progress.{block_type}'
+
+        for block_type in settings.COMPLETION_AGGREGATOR_TRACKING_EVENT_TYPES
+    ]
+    settings.EVENT_TRACKING_BACKENDS_ALLOWED_XAPI_EVENTS += enabled_aggregator_events
+    settings.EVENT_TRACKING_BACKENDS.update(event_tracking_backends_config(
+        settings,
+        settings.EVENT_TRACKING_BACKENDS_ALLOWED_XAPI_EVENTS,
+        settings.EVENT_TRACKING_BACKENDS_ALLOWED_CALIPER_EVENTS,
+    ))

completion_aggregator/xapi.py

@@ -0,0 +1,71 @@
+"""
+Transformers for completion aggregation.
+"""
+
+from event_routing_backends.processors.openedx_filters.decorators import openedx_filter
+from event_routing_backends.processors.xapi import constants
+from event_routing_backends.processors.xapi.registry import XApiTransformersRegistry
+from event_routing_backends.processors.xapi.transformer import XApiTransformer
+from tincan import Activity, ActivityDefinition, Extensions, LanguageMap, Result, Verb
+
+
+class BaseProgressTransformer(XApiTransformer):
+    """
+    Base transformer for completion aggregator progress events.
+    """
+
+    _verb = Verb(
+        id=constants.XAPI_VERB_PROGRESSED,
+        display=LanguageMap({constants.EN: constants.PROGRESSED}),
+    )
+    object_type = None
+    additional_fields = ('result', )
+
+    @openedx_filter(
+        filter_type="completion_aggregator.xapi.progress.get_object",
+    )
+    def get_object(self) -> Activity:
+        """
+        Get object for xAPI transformed event.
+        """
+        if not self.object_type:
+            raise NotImplementedError()  # pragma: no cover
+
+        return Activity(
+            id=self.get_object_iri("xblock", self.get_data("data.block_id")),
+            definition=ActivityDefinition(
+                type=self.object_type,
+            ),
+        )
+
+    def get_result(self) -> Result:
+        """
+        Get result for xAPI transformed event.
+        """
+        progress = self.get_data("data.percent") or 0
+        return Result(
+            completion=progress == 1.0,
+            extensions=Extensions({
+                constants.XAPI_ACTIVITY_PROGRESS: (progress * 100),
+            }),
+        )
+
+
+@XApiTransformersRegistry.register("openedx.completion_aggregator.progress.chapter")
+@XApiTransformersRegistry.register("openedx.completion_aggregator.progress.sequential")
+@XApiTransformersRegistry.register("openedx.completion_aggregator.progress.vertical")
+class ModuleProgressTransformer(BaseProgressTransformer):
+    """
+    Transformer for event generated when a user makes progress in a section, subsection or unit.
+    """
+
+    object_type = constants.XAPI_ACTIVITY_MODULE
+
+
+@XApiTransformersRegistry.register("openedx.completion_aggregator.progress.course")
+class CourseProgressTransformer(BaseProgressTransformer):
+    """
+    Transformer for event generated when a user makes progress in a course.
+    """
+
+    object_type = constants.XAPI_ACTIVITY_COURSE

requirements/base.in

@@ -12,3 +12,4 @@ edx-toggles
 event-tracking     # Allows the aggregator to emit tracking events
 six
 XBlock[django]
+edx-event-routing-backends  # Provides xAPI transforms for aggregator events

requirements/base.txt

@@ -6,10 +6,16 @@
 #
 amqp==5.2.0
     # via kombu
+aniso8601==9.0.1
+    # via tincan
+apache-libcloud==3.8.0
+    # via edx-event-routing-backends
 appdirs==1.4.4
     # via fs
 asgiref==3.7.2
     # via django
+async-timeout==4.0.3
+    # via redis
 attrs==23.2.0
     # via openedx-events
 backports-zoneinfo[tzdata]==0.2.1
@@ -54,34 +60,47 @@ click-repl==0.3.0
 code-annotations==1.6.0
     # via edx-toggles
 cryptography==42.0.5
-    # via pyjwt
+    # via
+    #   django-fernet-fields-v2
+    #   pyjwt
 django==3.2.24
     # via
     #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
     #   -r requirements/base.in
+    #   django-config-models
     #   django-crum
+    #   django-fernet-fields-v2
     #   django-model-utils
+    #   django-redis
     #   django-waffle
     #   djangorestframework
     #   drf-jwt
     #   edx-celeryutils
     #   edx-completion
     #   edx-django-utils
     #   edx-drf-extensions
+    #   edx-event-routing-backends
     #   edx-toggles
     #   event-tracking
     #   jsonfield
     #   openedx-django-pyfs
     #   openedx-events
+    #   openedx-filters
+django-config-models==2.7.0
+    # via edx-event-routing-backends
 django-crum==0.7.9
     # via
     #   edx-django-utils
     #   edx-toggles
+django-fernet-fields-v2==0.9
+    # via edx-event-routing-backends
 django-model-utils==4.4.0
     # via
     #   -r requirements/base.in
     #   edx-celeryutils
     #   edx-completion
+django-redis==5.4.0
+    # via edx-event-routing-backends
 django-waffle==4.1.0
     # via
     #   edx-django-utils
@@ -90,23 +109,31 @@ django-waffle==4.1.0
 djangorestframework==3.14.0
     # via
     #   -r requirements/base.in
+    #   django-config-models
     #   drf-jwt
     #   edx-completion
     #   edx-drf-extensions
 drf-jwt==1.19.2
     # via edx-drf-extensions
 edx-celeryutils==1.2.5
-    # via -r requirements/base.in
+    # via
+    #   -r requirements/base.in
+    #   edx-event-routing-backends
 edx-completion==4.6.0
     # via -r requirements/base.in
 edx-django-utils==5.10.1
     # via
+    #   django-config-models
     #   edx-drf-extensions
     #   edx-toggles
     #   event-tracking
     #   openedx-events
 edx-drf-extensions==10.2.0
     # via edx-completion
+edx-event-routing-backends==9.3.0
+    # via
+    #   -c requirements/constraints.txt
+    #   -r requirements/base.in
 edx-opaque-keys[django]==2.5.1
     # via
     #   -r requirements/base.in
@@ -117,13 +144,17 @@ edx-toggles==5.1.1
     # via
     #   -r requirements/base.in
     #   edx-completion
+    #   edx-event-routing-backends
     #   event-tracking
-event-tracking==2.3.0
+event-tracking==2.4.0
     # via
     #   -r requirements/base.in
-    # edx-completion
+    #   edx-completion
+    #   edx-event-routing-backends
 fastavro==1.9.4
     # via openedx-events
+fasteners==0.19
+    # via edx-event-routing-backends
 fs==2.4.16
     # via
     #   fs-s3fs
@@ -133,14 +164,18 @@ fs-s3fs==1.1.1
     # via openedx-django-pyfs
 idna==3.6
     # via requests
+isodate==0.6.1
+    # via edx-event-routing-backends
 jinja2==3.1.3
     # via code-annotations
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
 jsonfield==3.1.0
-    # via edx-celeryutils
+    # via
+    #   edx-celeryutils
+    #   edx-event-routing-backends
 kombu==5.3.5
     # via celery
 lazy==1.6
@@ -160,6 +195,8 @@ openedx-django-pyfs==3.5.0
     # via xblock
 openedx-events==9.5.2
     # via event-tracking
+openedx-filters==1.8.1
+    # via edx-event-routing-backends
 pbr==6.0.0
     # via stevedore
 prompt-toolkit==3.0.43
@@ -182,6 +219,7 @@ python-dateutil==2.8.2
     # via
     #   botocore
     #   celery
+    #   edx-event-routing-backends
     #   xblock
 python-slugify==8.0.4
     # via code-annotations
@@ -190,14 +228,21 @@ pytz==2024.1
     #   django
     #   djangorestframework
     #   edx-completion
+    #   edx-event-routing-backends
     #   event-tracking
+    #   tincan
     #   xblock
 pyyaml==6.0.1
     # via
     #   code-annotations
     #   xblock
+redis==5.0.5
+    # via django-redis
 requests==2.31.0
-    # via edx-drf-extensions
+    # via
+    #   apache-libcloud
+    #   edx-drf-extensions
+    #   edx-event-routing-backends
 s3transfer==0.10.0
     # via boto3
 semantic-version==2.10.0
@@ -210,6 +255,7 @@ six==1.16.0
     #   event-tracking
     #   fs
     #   fs-s3fs
+    #   isodate
     #   python-dateutil
 sqlparse==0.4.4
     # via django
@@ -220,6 +266,8 @@ stevedore==5.2.0
     #   edx-opaque-keys
 text-unidecode==1.3
     # via python-slugify
+tincan==1.0.0
+    # via edx-event-routing-backends
 typing-extensions==4.10.0
     # via
     #   asgiref

requirements/ci.txt

@@ -13,9 +13,7 @@ filelock==3.13.1
 packaging==23.2
     # via tox
 platformdirs==4.2.0
-    # via
-    #   tox
-    #   virtualenv
+    # via virtualenv
 pluggy==0.13.1
     # via
     #   -c requirements/constraints.txt