Skip to content

Commit

Permalink
feat(h5p-server): added content user state (closes #1014) (#1886)
Browse files Browse the repository at this point in the history
* test(jest): run jest tests in watch mode via test:watch

* chore(scripts): add start:rest script

* feat(contentUserData): add parameters to contentUserDataUrl config

* feat(contentUserData): read saveFreq from config

* feat(contentUserData): add the contentUserData into the H5PIntegration

* test(contentUserDataGET): add example for GET issue

#1014 (comment)

* test(contentUserData): revert example

* chore(scripts): add h5p-server build script

* test(launch): add DEBUG to vscode launch

* refactor(script): rename start:rest to start:rest:server

#1886 (comment)

* refactor(saveFreq): rename saveFreq to contentUserStateSaveInterval

#1886 (comment)

* feat(contentUserData): delete contentUserData when content is deleted

* test(contentUserData): use mock for H5PPlayer.render test

* refactor(h5p-examples): add mock implementation to pass build

* feat(contentUserData): build contentUserDataIntegration in Manager

* feat(contentUserData): update interfaces

* test(contentUserDataManager): add tests

* refactor(h5p-examples): remove test/example conentUserDataStorage

* refactor(saveContentUserData): add invalidate and preload parameters

* feat(h5p-express): add ContentUserDataController

* fix(contentManager): delete contentUserDataStorage after content

#1886 (comment)

* refactor(ContentUserDataManager): remove unnecessary code

* refactor(ContentUserDataManager): use boolean instead of number

#1886 (comment)

* refactor(ContentManager): remove unnecessary code

* refactor(ContentUserDataManager): use boolean instead of number

#1886 (comment)

* fix(ContentUserDataManager): sanitize userState before saving

* feat(h5pexpress): add ContentUserDataRouter

* refactor(h5p-server): add deleteAllContentUserDataforContentId method

* refactor(h5p-examples): add reference implementation to h5p-example

* docs(ContentUserDataStorage): add contentUserDataStorage docs

* feat(contentUserData): add setFinished

* fix(ContentUserDataManager): throw H5pError instead of regular error

* fix(contentUserStateSaveInterval): change from seconds to milliseconds

#1886 (comment)

* refactor(code): remove redundant return statements

* refactor(code): reorder import statements

* refactor(log): change to debug from info

* refactor(code): remove typos, reorder imports, remove redundant code

* refactor(h5p-examples): don't use singleton

* fix(H5PPlayer): use contentUserStateSaveInterval in milliseconds

* refactor(code): fix typos

* fix(contentUserDataManager): saveContentUserData: check arguments

* fix(h5p-examples): correct json number declaration (#1991)

* feat(listContentUserDataByUserId): add listContentUserDataByUserId

* style(jsdocs): add divergence

* refactor: formatting and minor issues

* refactor: fixed imports

* test: corrected test

* refactor: decreased content user state save interval

* test: fix test

* fix(contentUserDataManager): remove userData when invalidate is true

#1886 (comment)

* fix(contentUserData): generate integration only when preload is true

#1886 (comment)

* feat(FileContentUserDataStorage): add FileContentUserDataStorage

* feat(contentUserData): include contentUserData in rest-example

* feat(contentUserData): include in rest example

* test(contentUserDataRouter): fix test

* chore(contentUserDataStorage): add json to gitignore

* build(h5p-shared-state-server): build h5p-shared-state-server on install

* fix(contentUserData): delete invalid contentUserData if content changes

#1886 (comment)

* refactor(contentUserData): rename saveContentUserData

saveContentUserData-method is renamed to createOrUpdateContentUserData

* feat(mongos3): add MongoContentUserDataStorage

* refactor: made namings more consistent

* refactor: minor cleanup

* fix: examples work

* test: use snapshot instead of inline HTML

* feat: corrected and extended MongoContentUserDataStorage

* fix: corrected FileContentUserDataStorage signature

* test: added more tests

* test: more tests

* feat: delete finished data when content is deleted

* feat: reimplemented FileContentUserDataStorage to work with directories

* test: added tests for FileContentUserDataStorage

* test: renamed generalized test file

* fix: missing user data doesn't return 404

* fix: corrected CSFR token generation + own route for setFinished

* fix: protected FileContentUserDataStorage against attacks

* test: corrected tests

* fix: corrected config to load falsy settings

* fix: improved filename validation

* feat: routes are closed when feature disabled in config

* feat: added CSRF tokens to rest example & fixed bugs

* fix: fileContentUserDataStorage saves more than one entry

* test: corrected test

* feat: improved  CSRF

* refactor: correct shared state example

* refactor: cleanup

* docs: added docs

Co-authored-by: Oliver Tacke <[email protected]>
Co-authored-by: Sebastian Rettig <[email protected]>
  • Loading branch information
3 people authored Jun 10, 2022
1 parent 5d9439c commit bdf66da
Show file tree
Hide file tree
Showing 53 changed files with 4,071 additions and 744 deletions.
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ test/data/hub-content/*
test/data/content-type-cache/real-content-types.json
test/data/hub-content-extracted/*
packages/h5p-html-exporter/test/data/content/*

google_translate.json
.json-autotranslate-cache

Expand Down
26 changes: 19 additions & 7 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,22 @@
{
"configurations": [
{
"command": "npm start",
"name": "Run npm start",
"request": "launch",
"type": "node-terminal"
}
{
"command": "DEBUG=h5p:* npm start",
"name": "Run npm start",
"request": "launch",
"type": "node-terminal"
},
{
"type": "node",
"request": "launch",
"name": "Launch Program",
"program": "${workspaceFolder}/packages/h5p-examples/build/express.js",
"runtimeArgs": ["-r", "source-map-support/register"],
"env": {
"DEBUG": "h5p*"
},
"outputCapture": "std",
"console": "integratedTerminal"
}
]
}
}
2 changes: 2 additions & 0 deletions docs/SUMMARY.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
* [Constructing H5PEditor](usage/h5p-editor-constructor.md)
* [REST Example](examples/rest/README.md)
- Advanced usage
* [User content state](advanced/user-content-state.md)
* [Basic completion tracking](advanced/completion-tracking.md)
* [Localization](advanced/localization.md)
* [Cluster](advanced/cluster.md)
* [Addons](advanced/addons.md)
Expand Down
68 changes: 68 additions & 0 deletions docs/advanced/completion-tracking.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# Completion tracking

The H5P client is capable of sending a message to the server when the user has
completed a content object. This includes the time when this occured, how long
the content was open, the achieved score and the maximum score.

While technically this message is derived from a xAPI statement generated by the
content types, it **is not the same as xAPI** and is a completely separate
system that co-exists with xAPI and a potential LRS. You can enable completion
tracking and xAPI tracking indepedently.

If you want to capture all xAPI statements, which allows you to have very
detailed tracking, you either have to inject your own xAPI capturing JavaScript
or use the xAPI capabilities of the `H5PPlayerComponent` in the [webcomponent
package](../packages/h5p-webcomponents.md)) (or the corresponding functionality
in the [React package](../packages/h5p-react.md)).

## How it works

- When the user presses "check", a xAPI statement indicating completion is
generated by the content type (that supports it). The H5P client captures it
and calls an AJAX route on the H5P server with basic information.
- The H5P server saves the completion data in a special storage system. The
server automatically deletes the data when the content object is deleted.

## Limitations

- While the storage classes support retrieving and deleting the completion data,
there are no endpoints on the `h5p-express` package that implement this
functionality. You have to implement these endpoints yourself.
- Not all content types emit xAPI statements indicating completion and thus
the tracking isn't fired.

## Enabling completion tracking

- Create an instance of `IContentUserDataStorage`. The recommended storage class
for production is `MongoContentUserDataStorage` in the
`@lumieducation/h5p-mongos3` package. There's also a
`FileContentUserDataStorageClass` in the `@lumieducation/h5p-server` package
that you can use for development or testing purposes.
- Pass the implementation of IContentUserDataStorage into the `H5PEditor` and
`H5PPlayer` constructor.
- Set `setFinishedEnabled` in `IH5PConfig` to `true`.
- If you use `h5pAjaxExpressRouter` from the `@lumieducation/h5p-express`
package, then the routes for the AJAX endpoint are automatically created. You
can manually turn them on by setting `routeFinishedData` in the options when
creating the route.
- If you don't use `h5pAjaxExpressRouter`, you have to route everything
manually. First get `ContentUserDataManager` from `H5PEditor` or
`H5PPlayer`. Route this endpoint and return HTTP status code 200 with a JSON
object that is based on `AjaxSuccessResponse` with empty payload:
- POST {{setFinishedUrl}}/ -> `ContentUserDataManager.setFinished`

## Configuration options

- You can customize the URL to which the AJAX calls are made by setting
`setFinishedUrl` in `IH5PConfig`.
- You can enable or disable the feature by setting `setFinishedEnabled` in
IH5PConfig.

## Security considerations

You should implement CSRF tokens when using completion tracking as the POST
endpoint would otherwise by vulnerable to CSRF attacks when using cookie
authentication. The tokens are added to the endpoint URL in the IUrlGenerator
implementation and thus sent to the server whenever a POST call is made. Check
out the REST example on how to pass the CSRF token to the H5P server components
and how to check its validity.
80 changes: 80 additions & 0 deletions docs/advanced/user-content-state.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# User content state

The H5P client is capable of saving the current state of the user so that the
user can resume where they left off. This means that e.g. their attempts entered
into textboxes are the same as when they last left off.

## How it works

- If state saving is enabled, a timer interval is set in the H5PIntegration
object by the server
- The H5P core client reads the interval and tells the content type that is
currently being displayed to persist it's state into a JSON object
- The H5P core client sends the state to an AJAX route on the server (specified
in H5PIntegration).
- The server stores the state in a special storage system. When content is
deleted, the user state is deleted as well. When content is updated, the user
state is deleted if the content type requests this (the case for (nearly?) all
content types).
- When the user later re-opens the content, the server checks if there is a user
state for the user that should be "preloaded". This means that the initial
information about the content object also includes the state and the client
doesn't have to make a second request to get it. If the state is marked as
"preloaded" (this is done by the content type), the content type uses it
during it's initialization routine.
- If "preloaded" is set to `false` the H5P client can also request the user
state through an AJAX call from the server.
- There can also be a user state in the editor. For instance, it saves whether
the user has dismissed the tours of Interactive Video or has closed one of the
yellow "information boxes" that explain functions of the editor. The editor
always gets the state through a second AJAX call.

## Limitations of the user state

- Not all content types implement it.
- Not all content types fully restore the state (e.g. they don't restore if the
user has already pressed "checked").

## Enabling user state

- Create an instance of `IContentUserDataStorage`. The recommended storage class
for production is `MongoContentUserDataStorage` in the
`@lumieducation/h5p-mongos3` package. There's also a
`FileContentUserDataStorageClass` in the `@lumieducation/h5p-server` package
that you can use for development or testing purposes.
- Pass the implementation of IContentUserDataStorage into the `H5PEditor` and
`H5PPlayer` constructor.
- Set `contentUserStateSaveInterval` in `IH5PConfig` to the interval at which
the client should save the state (in milliseconds). The recommended number is
`10000`. (To disable the feature, set `contentUserStateSaveInterval` to
`false`)
- If you use `h5pAjaxExpressRouter` from the `@lumieducation/h5p-express`
package, then the routes for the AJAX endpoint are automatically created. You
can manually turn them on by setting `routeContentUserData` in the options
when creating the route.
- If you don't use `h5pAjaxExpressRouter`, you have to route everything
manually. First get `ContentUserDataManager` from `H5PEditor` or `H5PPlayer`.
Route these endpoints to the functions and return HTTP status code 200 with a
JSON object that is based on `AjaxSuccessResponse` with empty payload (Check
out the Express Router for details):
- GET {{contentUserDataUrl}}/:contentId/:dataType/:subContentId ->
`ContentUserDataManager.getContentUserData`
- POST {{contentUserDataUrl}}/:contentId/:dataType/:subContentId ->
`ContentUserDataManager.createOrUpdateContentUserData`

## Configuration options

- You can customize the URL at which the AJAX calls are available by setting
`contentUserDataUrl` in `IH5PConfig`.
- You can customize the interval at which content states are saved by setting
`contentUserStateSaveInterval` in IH5PConfig. If you set it to false, you can
disable the feature.

## Security considerations

You should implement CSRF tokens when using the content user state as the POST
endpoint would otherwise by vulnerable to CSRF attacks when using cookie
authentication. The tokens are added to the endpoint URL in the IUrlGenerator
implementation and thus sent to the server whenever a POST call is made. Check
out the REST example on how to pass the CSRF token to the H5P server components
and how to check its validity.
26 changes: 16 additions & 10 deletions docs/usage/h5p-editor-constructor.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@

There are two ways of creating a H5PEditor object:

* You can use the convenience function
[`H5P.fs(...)`](/packages/h5p-server/src/implementation/fs/index.ts) that uses
basic file system implementations for all data storage services. You can use
the function if you're just getting started. Later on, you'll want to
construct the editor with custom implementations of the data storage services.
Check out the JSDoc of the function for details how to use it.
* You can construct it manually by calling `new H5P.H5PEditor(...)`. The
constructor arguments are used to provide data storage services and settings.
You can find the interfaces referenced in
[`src/types.ts`](/packages/h5p-server/src/types.ts).
- You can use the convenience function
[`H5P.fs(...)`](/packages/h5p-server/src/implementation/fs/index.ts) that uses
basic file system implementations for all data storage services. You can use
the function if you're just getting started. Later on, you'll want to
construct the editor with custom implementations of the data storage services.
Check out the JSDoc of the function for details how to use it.
- You can construct it manually by calling `new H5P.H5PEditor(...)`. The
constructor arguments are used to provide data storage services and settings.
You can find the interfaces referenced in
[`src/types.ts`](/packages/h5p-server/src/types.ts).

Explanation of the arguments of the constructor:

Expand Down Expand Up @@ -114,3 +114,9 @@ for an implementation sample using the urlGenerator.

Allows you to customize styles and scripts of the client. Also allows passing in
a lock implementation (needed for multi-process or clustered setups).

## contentUserDataStorage (optional)

The `contentUserDataStorage` handles saving and loading user states, so users
can continue where they left off when they reload the page or come back later.
It must implement the `IContentUserDataStorage` interface.
5 changes: 4 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
"build:h5p-server": "npm run build --prefix packages/h5p-server",
"build:h5p-shared-state-server": "npm run build --prefix packages/h5p-shared-state-server",
"build:h5p-webcomponents": "npm run build --prefix packages/h5p-webcomponents",
"build": "npm run build --prefix packages/h5p-server && concurrently \"npm run build --prefix packages/h5p-express\" \"npm run build --prefix packages/h5p-html-exporter\" \"npm run build --prefix packages/h5p-redis-lock\" \"npm run build --prefix packages/h5p-mongos3\" && concurrently \"npm run build --prefix packages/h5p-shared-state-server\" \"npm run build --prefix packages/h5p-examples\" \"npm run build:h5p-webcomponents\" && npm run build:h5p-react",
"build:h5p-server": "npm run build --prefix packages/h5p-server",
"build:h5p-redis-lock": "npm run build --prefix packages/h5p-redis-lock",
"build": "npm run build --prefix packages/h5p-server && concurrently \"npm run build --prefix packages/h5p-express\" \"npm run build --prefix packages/h5p-html-exporter\" \"npm run build --prefix packages/h5p-redis-lock\" \"npm run build --prefix packages/h5p-mongos3\" && concurrently \"npm run build --prefix packages/h5p-examples\" \"npm run build:h5p-webcomponents\" && npm run build:h5p-react && npm run build:h5p-shared-state-server",
"download:content-type-cache": "ts-node scripts/update-real-content-type-cache.ts",
"download:content": "node scripts/download-examples.js test/data/content-type-cache/real-content-types.json test/data/hub-content",
"download:h5p": "sh scripts/install.sh",
Expand All @@ -40,6 +42,7 @@
"semantic-release": "semantic-release",
"start:dbs": "docker-compose -f scripts/mongo-s3-docker-compose.yml up -d",
"start": "npm run start --prefix packages/h5p-examples",
"start:rest:server": "npm run start --prefix packages/h5p-rest-example-server",
"stop:dbs": "docker-compose -f scripts/mongo-s3-docker-compose.yml down",
"test:db": "npx jest --config jest.db.config.js --maxWorkers=${BUILD_WORKERS-`nproc`}",
"test:e2e:tests": "npm run test:e2e:tests --prefix packages/h5p-examples",
Expand Down
7 changes: 2 additions & 5 deletions packages/h5p-examples/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,15 @@
"contentHubEnabled": true,
"hubRegistrationEndpoint": "https://api.h5p.org/v1/sites",
"hubContentTypesEndpoint": "https://api.h5p.org/v1/content-types/",
"contentUserDataUrl": "/contentUserData",
"contentTypeCacheRefreshInterval": 86400000,
"enableLrsContentTypes": true,
"maxFileSize": 1048576000,
"contentUserStateSaveInterval": 5000,
"maxTotalSize": 1048576000,
"editorAddons": {
"H5P.CoursePresentation": ["H5P.MathDisplay"],
"H5P.InteractiveVideo": ["H5P.MathDisplay"],
"H5P.DragQuestion": ["H5P.MathDisplay"]
},
"libraryConfig": {
"H5P.ShareDBTest": {
"serverUrl": "ws://localhost:5001"
}
}
}
9 changes: 8 additions & 1 deletion packages/h5p-examples/src/createH5PEditor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export default async function createH5PEditor(
localLibraryPath: string,
localContentPath?: string,
localTemporaryPath?: string,
localContentUserDataPath?: string,
translationCallback?: H5P.ITranslationFunction
): Promise<H5P.H5PEditor> {
let cache: Cache;
Expand Down Expand Up @@ -124,6 +125,11 @@ export default async function createH5PEditor(
);
}

const contentUserDataStorage =
new H5P.fsImplementations.FileContentUserDataStorage(
localContentUserDataPath
);

const h5pEditor = new H5P.H5PEditor(
new H5P.cacheImplementations.CachedKeyValueStorage('kvcache', cache), // this is a general-purpose cache
config,
Expand Down Expand Up @@ -178,7 +184,8 @@ export default async function createH5PEditor(
enableHubLocalization: true,
enableLibraryNameLocalization: true,
lockProvider: lock
}
},
contentUserDataStorage
);

// Set bucket lifecycle configuration for S3 temporary storage to make
Expand Down
51 changes: 5 additions & 46 deletions packages/h5p-examples/src/express.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,8 @@ const start = async (): Promise<void> => {
// content storage class.
path.join(__dirname, '../h5p/temporary-storage'), // the path on the local disc
// where temporary files (uploads) should be stored. Only used /
// necessary if you use the local filesystem temporary storage class.
// necessary if you use the local filesystem temporary storage class.,
path.join(__dirname, '../h5p/user-data'),
(key, language) => translationFunction(key, { lng: language })
);

Expand All @@ -103,7 +104,9 @@ const start = async (): Promise<void> => {
config,
undefined,
undefined,
(key, language) => translationFunction(key, { lng: language })
(key, language) => translationFunction(key, { lng: language }),
undefined,
h5pEditor.contentUserDataStorage
);

// We now set up the Express server in the usual fashion.
Expand Down Expand Up @@ -234,50 +237,6 @@ const start = async (): Promise<void> => {
express.static(path.join(__dirname, '../node_modules'))
);

// STUB, not implemented yet. You have to get the user id through a session
// cookie as h5P does not add it to the request. Alternatively you could add
// it to the URL generator.
server.post(
'/h5p/contentUserData/:contentId/:dataType/:subContentId',
(
req: express.Request<
{ contentId: string; dataType: string; subContentId: string },
{},
H5P.IPostContentUserData
>,
res
) => {
res.status(200).send();
}
);

// STUB, not implemented yet. You have to get the user id through a session
// cookie as h5P does not add it to the request. Alternatively you could add
// it to the URL generator.
server.get(
'/h5p/contentUserData/:contentId/:dataType/:subContentId',
(
req: express.Request<{
contentId: string;
dataType: string;
subContentId: string;
}>,
res: express.Response<H5P.IGetContentUserData | {}>
) => {
res.status(200).json({});
}
);

// STUB, not implemented yet. You have to get the user id through a session
// cookie as h5P does not add it to the request. Alternatively you could add
// it to the URL generator.
server.post(
'/h5p/setFinished',
(req: express.Request<{}, {}, H5P.IPostContentUserData>, res) => {
res.status(200).send();
}
);

// Remove temporary directory on shutdown
if (useTempUploads) {
[
Expand Down
Loading

0 comments on commit bdf66da

Please sign in to comment.