Skip to content

Commit

Permalink
Server fixes and xapi caching
Browse files Browse the repository at this point in the history
  • Loading branch information
@Lomilar
Lomilar committed Sep 17, 2024
1 parent 8322d0a commit 58e35e7
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 80 deletions.
4 changes: 4 additions & 0 deletions .eslintrc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,9 @@ module.exports = {
'prefer-const': 'off',
'require-jsdoc': 'off',
'camelcase': 'off',
'no-var': 'off',
'curly': 'off',
'quotes': 'off',
'semi': 'off',
},
};
186 changes: 115 additions & 71 deletions src/main/server/adapter/xapi/xapi.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ var xapiConfig = function () {
xapiHostname: "",
xapiAuth: "",
enabled: false
}),xapiConfigFilePath);
}), xapiConfigFilePath);
this.xapiConfig = JSON.parse(fileToString(fileLoad(xapiConfigFilePath)));
return this.xapiConfig;
}
Expand All @@ -19,7 +19,7 @@ var xapiConfigAutoExecute = xapiConfig;
var xapiEndpoint = async function (more, since, config) {
var endpoint;
if (config) {
endpoint = config.xapiEndpoint + "statements?format=exact&limit=0";
endpoint = config.xapiEndpoint + "statements?format=exact&limit=10";
} else {
endpoint = xapiConfig.call(this).xapiEndpoint + "statements?format=exact&limit=0";
}
Expand Down Expand Up @@ -51,32 +51,32 @@ var getMbox = function (agentObject) {
return null;
}

let personCache = {};
setInterval(function () {
personCache = {};
}, 1000 * 60 * 60);
var personFromEmail = async function (mbox) {
if (mbox == null) return null;
if (personCache[mbox] != null) return personCache[mbox];
var person = null;
mbox = mbox.replace("mailto:", "");
if (mbox.indexOf("@") != -1)
{
if (mbox.indexOf("@") != -1) {
let people = null;
people = await loopback.repositorySearch(global.repo,"@type:Person AND email:\"" + mbox + "\"",{});
people = await loopback.repositorySearch(global.repo, "@type:Person AND email:\"" + mbox + "\"", {});
if (people != null) {
if (people.length == 1)
if (people.length >= 1)
person = people[0];
else if (people.length > 1)
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.ERROR, "XapiPersonFromEmail", `Cannot generate xAPI statements for ${mbox} -- too many people with that email.`);
}
}
else
{
else {
let people = null;
people = await loopback.repositorySearch(global.repo,"@type:Person AND (identifier:\"" + mbox + "\" OR username:\"" + mbox + "\")",{});
people = await loopback.repositorySearch(global.repo, "@type:Person AND (identifier:\"" + mbox + "\" OR username:\"" + mbox + "\")", {});
if (people != null) {
if (people.length == 1)
if (people.length >= 1)
person = people[0];
else if (people.length > 1)
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.ERROR, "XapiPersonFromEmail", `Cannot generate xAPI statements for ${mbox} -- too many people with that identifier.`);
}
}
if (person != null) personCache[mbox] = person;
return person;
}

Expand All @@ -94,25 +94,32 @@ var pkFromMbox = async function (xapiPerson) {
return pk;
}

let alignedCompetenciesCache = {};
setInterval(function () {
alignedCompetenciesCache = {};
}, 1000 * 60 * 60);
var getAlignedCompetencies = async function (objectId) {
var results = [];
let creativeWorks = await loopback.repositorySearch(global.repo,"@type:CreativeWork AND url:\"" + objectId + "\"",{});
for (let creativeWork of creativeWorks)
{
if (alignedCompetenciesCache[objectId] != null)
return alignedCompetenciesCache[objectId];
let creativeWorks = await loopback.repositorySearch(global.repo, "@type:CreativeWork AND url:\"" + objectId + "\"", {});
for (let creativeWork of creativeWorks) {
if (creativeWork.educationalAlignment == null) continue;
if (!EcArray.isArray(creativeWork.educationalAlignment))
creativeWork.educationalAlignment = [creativeWork.educationalAlignment];
for (var i = 0; i < creativeWork.educationalAlignment.length; i++)
results.push(creativeWork.educationalAlignment[i]);
}
alignedCompetenciesCache[objectId] = results;
return results;
}

var xapiStatement = async function (s) {
let defaultAuthority = null;
var xapiStatement = async function (s, accm) {
if (s == null) return;
if (EcArray.isArray(s))
for (var i = 0;i < s.length;i++)
await xapiStatement(s[i]);
for (let st of s)
await xapiStatement(st, accm);
if (!EcObject.isObject(s)) return;
if (s.result == null) return;
var negative = false;
Expand All @@ -122,33 +129,32 @@ var xapiStatement = async function (s) {
negative = false;
else
negative = true;
} else if (s.result.score != null) {
var scaled = s.result.score.scaled;
if (scaled > 0.7)
negative = false;
else
negative = true;
} else if (s.result.response == "Pass") {
var scaled = 1.0;
negative = false;
} else if (s.result.response == "Fail") {
var scaled = 1.0;
negative = true;
} if (s.result.score != null) {
var scaled = s.result.score.scaled;
if (scaled > 0.7)
negative = false;
else
negative = true;
} else
return;

var actorPk = await pkFromMbox.call(this, s.actor);
if (actorPk == null)
{
if (actorPk == null) {
var ppk = await EcPpk.generateKey();
var person = new schema.Person();
person.assignId(global.repo.selectedServer,ppk.toPk().fingerprint());
person.assignId(global.repo.selectedServer, ppk.toPk().fingerprint());
person.addOwner(ppk.toPk());
var mb = getMbox.call(this, s.actor).replace("mailto:","");
if (mb.indexOf("@") == -1)
person.username = mb;
else
person.email = mb;
var mb = getMbox.call(this, s.actor).replace("mailto:", "");
if (mb.indexOf("@") == -1)
person.username = mb;
else
person.email = mb;
person.name = s.actor.name;
await EcRepository.save(person, (msg) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.INFO, "XapiSavePerson", msg);
Expand All @@ -159,38 +165,55 @@ var xapiStatement = async function (s) {
}
if (actorPk == null) return;
var authorityPk = await pkFromMbox.call(this, s.authority);
if (authorityPk == null)
{
if (authorityPk == null) {
let ppk = EcPpk.fromPem(xapiMePpk);
var person = new schema.Person();
person.assignId(global.repo.selectedServer,ppk.toPk().fingerprint());
person.assignId(global.repo.selectedServer, ppk.toPk().fingerprint());
person.addOwner(ppk.toPk());
var mb = getMbox.call(this, s.authority);
if (mb != null) mb = mb.replace("mailto:","");
if (mb == null) mb = "Some Authority";
if (mb.indexOf("@") == -1)
person.username = mb;
else
person.email = mb;
person.name = (s.authority != null ? s.authority.name : null) || mb;
await EcRepository.save(person, (msg) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.INFO, "XapiSavePerson", msg);
}, (error) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.ERROR, "XapiSavePerson", error);
});
var mb = getMbox.call(this, s.authority);
if (mb != null) mb = mb.replace("mailto:", "");
if (mb == null) {
mb = "Some Authority";
if (defaultAuthority == null) {
defaultAuthority = person;
if (mb.indexOf("@") == -1)
person.username = mb;
else
person.email = mb;
person.name = (s.authority != null ? s.authority.name : null) || mb;
await EcRepository.save(person, (msg) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.INFO, "XapiSavePerson", msg);
}, (error) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.ERROR, "XapiSavePerson", error);
});
}
}
else {
if (mb.indexOf("@") == -1)
person.username = mb;
else
person.email = mb;
person.name = (s.authority != null ? s.authority.name : null) || mb;
await EcRepository.save(person, (msg) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.INFO, "XapiSavePerson", msg);
}, (error) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.ERROR, "XapiSavePerson", error);
});
}
authorityPk = ppk.toPk();
}
if (authorityPk == null) return;

if (s.object == null) return;

var alignedCompetencies = await getAlignedCompetencies.call(this, s.object.id);
var alreadyAligned = {};
for (var i = 0; i < alignedCompetencies.length; i++) {
var a = new EcAssertion();
a.assignId(global.repo.selectedServer, EcCrypto.md5(s.id+alignedCompetencies[i].targetUrl));
a.assignId(global.repo.selectedServer, EcCrypto.md5(s.id + alignedCompetencies[i].targetUrl));
a.addOwner(EcPpk.fromPem(xapiMePpk).toPk());
a.addOwner(authorityPk);
a.addOwner(EcPk.fromPem(skyrepoAdminPk()));
a.addReader(actorPk);
await a.setSubject(actorPk);
await a.setAgent(authorityPk);
Expand All @@ -203,27 +226,22 @@ var xapiStatement = async function (s) {
await a.setAssertionDate(new Date(s.timestamp).getTime());
await a.setNegative(negative);
a.confidence = scaled;
EcRepository.save(a, (msg) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.INFO, "XapiSaveAssertion", msg);
}, (error) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.ERROR, "XapiSaveAssertion", error);
});
if (accm != null)
accm.push(a);
else
EcRepository.save(a, (msg) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.INFO, "XapiSaveAssertion", msg);
}, (error) => {
global.auditLogger.report(global.auditLogger.LogCategory.ADAPTER, global.auditLogger.Severity.ERROR, "XapiSaveAssertion", error);
});
}
}

var xapiStatementListener = async function () {
let accm = [];
for (let val in this.dataStreams)
await xapiStatement(this.dataStreams[val]);
// var data = fileFromDatastream.call(this);
// if (data == null)
// {
// }
// else
// {
// data = fileToString(data.get(0));
// data = JSON.parse(data);
// await xapiStatement(data);
// }
await xapiStatement(this.dataStreams[val], accm);
await global.repo.multiput(accm);
}

var ident = new EcIdentity();
Expand All @@ -235,8 +253,10 @@ xapiKey = function () {
}


var xapiLoopEach = async function(since, config, sinceFilePath) {
var results = await xapiEndpoint.call(this, null, since, config);
var xapiLoopEach = async function (since, config, sinceFilePath) {
try {
var results = await xapiEndpoint.call(this, null, since, config);
} catch (ex) { console.log(ex); return; }
while (results != null && results.statements != null && results.statements.length > 0) {
for (var i = 0; i < results.statements.length; i++) {
await xapiStatement.call(this, results.statements[i]);
Expand All @@ -249,7 +269,28 @@ var xapiLoopEach = async function(since, config, sinceFilePath) {
}
}

let openid = require('openid-client');
var xapiLoop = async function () {
let tokenSet = null;
if (process.env.OIDC_CLIENT_ENDPOINT != null) {
const oidcIssuer = await openid.Issuer.discover(process.env.OIDC_CLIENT_ENDPOINT);
console.log('Discovered issuer %s %O', oidcIssuer.issuer, oidcIssuer.metadata);
const client = new oidcIssuer.Client({
client_id: process.env.OIDC_CLIENT_CLIENT_ID,
client_secret: process.env.OIDC_CLIENT_CLIENT_SECRET,
redirect_uris: [process.env.OIDC_CLIENT_REDIRECT_URI],
response_types: [process.env.OIDC_CLIENT_RESPONSE_TYPE || "code"],
// id_token_signed_response_alg (default "RS256")
// token_endpoint_auth_method (default "client_secret_basic")
}); // => Client

tokenSet = await client.grant({
resource: 'urn:example:third-party-api',
grant_type: 'client_credentials'
});
console.log(tokenSet);
}

var ident = new EcIdentity();
ident.displayName = "xAPI Adapter";
ident.ppk = EcPpk.fromPem(xapiMePpk);
Expand All @@ -260,6 +301,9 @@ var xapiLoop = async function () {
for (let key in process.env) {
if (key.startsWith("XAPI_CONFIG_")) {
config.push(JSON.parse(process.env[key]));
if (process.env.OIDC_CLIENT_ENDPOINT != null) {
config[config.length - 1].xapiAuth = tokenSet.token_type + " " + tokenSet.access_token;
}
}
}
if (fileExists(sinceFilePath)) {
Expand All @@ -272,7 +316,7 @@ var xapiLoop = async function () {
if (!config || config.length === 0) {
await xapiLoopEach.call(this, since, null, sinceFilePath);
}

}

if (!global.disabledAdapters['xapi']) {
Expand Down
27 changes: 18 additions & 9 deletions src/main/server/skyRepo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,14 +207,16 @@ const filterResults = async function (o, dontDecryptInSso) {
if (EcArray.isArray(o)) {
let me = this;
return (await Promise.all(o.map(x => {
try {
return filterResults.call(me, x, dontDecryptInSso);
} catch (ex) {
if (ex != null && ex.toString().indexOf('Object not found or you did not supply sufficient permissions to access the object.') == -1) {
throw ex;
return new Promise((resolve,reject)=>{
try {
resolve(filterResults.call(me, x, dontDecryptInSso));
} catch (ex) {
if (ex != null && ex.toString().indexOf('Object not found or you did not supply sufficient permissions to access the object.') == -1) {
reject(ex);
}
resolve(null);
}
return null;
}
});
}))).filter(x => x);
} else if (EcObject.isObject(o)) {
delete o.decryptedSecret;
Expand Down Expand Up @@ -253,7 +255,14 @@ const filterResults = async function (o, dontDecryptInSso) {
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
let result = null;
result = await (filterResults).call(this, (o)[key], dontDecryptInSso);
try{
result = await (filterResults).call(this, (o)[key], dontDecryptInSso);
} catch (ex) {
if (ex != null && ex.toString().indexOf('Object not found or you did not supply sufficient permissions to access the object.') == -1) {
throw ex;
}
result = null;
}
if (result != null) {
(o)[key] = result;
} else {
Expand Down Expand Up @@ -8785,7 +8794,7 @@ let skyrepoPutInternal = global.skyrepoPutInternal = async function (o, id, vers
let chosenVersion = version;
// If we are doing a manual put with a CASS_LOOPBACK that has an associated CASS_LOOPBACK_PROXY from localhost,
// we have to pull the version from the object not the url (because it wasn't sent with the url because it's using the md5)
if (chosenVersion == null && (erld.id.startsWith(repo.selectedServer) || erld.id.startsWith(repo.selectedServerProxy))) {
if (chosenVersion == null && erld.id && (erld.id.startsWith(repo.selectedServer) || erld.id.startsWith(repo.selectedServerProxy))) {
chosenVersion = erld.getTimestamp();
}
if (chosenVersion == null) {
Expand Down

0 comments on commit 58e35e7

Please sign in to comment.