From 4c13157ae63312cba96a913ff09b4adcacc3bdbe Mon Sep 17 00:00:00 2001 From: Andrea Ascari Date: Mon, 7 Nov 2022 17:18:06 +0100 Subject: [PATCH] fix(guardoni): set experiment in settings.json correctly (#758) Co-authored-by: Claudio Agosti --- packages/shared/src/extension/app.ts | 4 +-- .../guardoni/__tests__/cli/cli-yt.spec.ts | 4 +-- platforms/guardoni/__tests__/guardoni.spec.ts | 28 ++++++++--------- .../scripts/cli-test-from-fixtures.mjs | 30 +++++++++++++------ platforms/guardoni/src/guardoni/cli.ts | 16 +++++++--- platforms/guardoni/src/guardoni/guardoni.ts | 26 +++++++++++++--- platforms/tktrex/backend/routes/metadata.ts | 4 +-- platforms/yttrex/backend/routes/metadata.ts | 12 ++++---- .../extension/src/background/chromeConfig.ts | 2 +- 9 files changed, 78 insertions(+), 48 deletions(-) diff --git a/packages/shared/src/extension/app.ts b/packages/shared/src/extension/app.ts index 5122fa311..ddd07201d 100644 --- a/packages/shared/src/extension/app.ts +++ b/packages/shared/src/extension/app.ts @@ -287,10 +287,10 @@ export async function boot(opts: BootOpts): Promise { throw localSettings.error; } - // merge settings taken from json with ones stored in db, giving the precedence to the latter + // merge settings taken from db with ones defined in settings.json, giving the precedence to the latter const settings: UserSettings = { - ...jsonSettings.result, ...localSettings.result, + ...jsonSettings.result, } as any; if (!settings.publicKey || !settings.secretKey) { diff --git a/platforms/guardoni/__tests__/cli/cli-yt.spec.ts b/platforms/guardoni/__tests__/cli/cli-yt.spec.ts index 2ff67abdf..825696511 100644 --- a/platforms/guardoni/__tests__/cli/cli-yt.spec.ts +++ b/platforms/guardoni/__tests__/cli/cli-yt.spec.ts @@ -260,9 +260,7 @@ describe('CLI', () => { ); expect(settingJson).toMatchObject({ - publicKey: tests.publicKey, - secretKey: tests.secretKey, - experimentId, + active: true, }); expect(result).toMatchObject({ diff --git a/platforms/guardoni/__tests__/guardoni.spec.ts b/platforms/guardoni/__tests__/guardoni.spec.ts index cbed4ffb7..99eeeca60 100644 --- a/platforms/guardoni/__tests__/guardoni.spec.ts +++ b/platforms/guardoni/__tests__/guardoni.spec.ts @@ -11,6 +11,7 @@ import { getDefaultProfile, getProfileDataDir } from '../src/guardoni/profile'; import { csvStringifyTE } from '@shared/utils/csv.utils'; import { guardoniLogger } from '../src/logger'; import { fc } from '@shared/test'; +import { throwTE } from '@shared/utils/task.utils'; const directiveLinks = [ { @@ -53,6 +54,7 @@ describe('Guardoni', () => { const profile = 'profile-test-99'; const emptyCSVTestFileName = 'yt-videos-test-empty.csv'; const csvTestFileName = 'trex-yt-videos.csv'; + const csvTestPath = path.resolve(basePath, 'experiments', csvTestFileName); const keys = { publicKey: process.env.PUBLIC_KEY, secretKey: process.env.SECRET_KEY, @@ -86,20 +88,16 @@ describe('Guardoni', () => { }; beforeAll(async () => { - const csvContent = await csvStringifyTE(directiveLinks, { - header: true, - encoding: 'utf-8', - })(); - - if (csvContent._tag === 'Left') { - throw csvContent.left as any; - } - fs.writeFileSync( - path.resolve(basePath, 'experiments', csvTestFileName), - csvContent.right, - 'utf-8' + const csvContent = await pipe( + csvStringifyTE(directiveLinks, { + header: true, + encoding: 'utf-8', + }), + throwTE ); + fs.writeFileSync(csvTestPath, csvContent, 'utf-8'); + const profileUDD = getProfileDataDir(basePath, profile); const profileExists = fs.existsSync(profileUDD); @@ -123,7 +121,7 @@ describe('Guardoni', () => { }); } - fs.rmSync(path.resolve(basePath, 'experiments', csvTestFileName)); + fs.rmSync(csvTestPath); }); describe('config', () => { @@ -278,9 +276,7 @@ describe('Guardoni', () => { guardoni, TE.chain((g) => pipe( - g.registerExperimentFromCSV( - path.resolve(basePath, 'experiments', csvTestFileName) as any - ), + g.registerExperimentFromCSV(csvTestPath as any), TE.chain((output) => g.runExperiment(output.values[0].experimentId)) ) ) diff --git a/platforms/guardoni/scripts/cli-test-from-fixtures.mjs b/platforms/guardoni/scripts/cli-test-from-fixtures.mjs index d52ff99eb..073cf5b05 100755 --- a/platforms/guardoni/scripts/cli-test-from-fixtures.mjs +++ b/platforms/guardoni/scripts/cli-test-from-fixtures.mjs @@ -103,28 +103,40 @@ void (async function () { const yt_home_experiment_register_out = await $`${cli} ${flags} ${p}-register ${experimentFile}`; - const yt_home_experiment_id = yt_home_experiment_register_out.stdout + const experiment_id = yt_home_experiment_register_out.stdout .split('\n') .find((s) => s.startsWith('experimentId:')) .replace('experimentId: \t', '') .trim(); const yt_home_experiment_run_out = - await $`${cli} ${experimentFlags} ${p}-experiment ${yt_home_experiment_id} | grep 'publicKey: ' `; + await $`${cli} ${experimentFlags} ${p}-experiment ${experiment_id} | grep 'publicKey: ' `; - const yt_home_experiment_public_key = yt_home_experiment_run_out.stdout + const supporter_public_key = yt_home_experiment_run_out.stdout .replace('publicKey: \t ', '') .replace('\n', ''); - assert.strictEqual(yt_home_experiment_public_key, process.env.PUBLIC_KEY); + assert.strictEqual(supporter_public_key, process.env.PUBLIC_KEY); const backend = p === 'tk' ? process.env.TK_BACKEND : process.env.YT_BACKEND; - const personalURL = `${backend}/v2/personal/${yt_home_experiment_public_key}/${nature}/json`; - await $`echo ${personalURL}`; + const personalURL = `${backend}/v2/personal/${supporter_public_key}/${nature}/json`; + const metadataURL = `${backend}/v2/metadata?publicKey=${supporter_public_key}&experimentId=${experiment_id}&nature=${nature}`; - // const metadata = await fetch( - // `http://localhost:9000/api/v2/metadata?publicKey=${yt_home_experiment_public_key}&experimentId=${yt_home_experiment_id}&nature=home` - // ).then((r) => r.json()); + await $`echo ${personalURL}`; + await $`echo ${metadataURL}`; + const metadata = await fetch(metadataURL).then((r) => r.json()); + + assert.strictEqual(metadata.length, sources.length); + assert.strictEqual( + metadata.map((m) => ({ + experimentId: m.experimentId, + type: m.type, + })), + Array.from({ length: sources.length }).map(() => ({ + experimentId: experiment_id, + type: nature, + })) + ); fs.removeSync(experimentFile); })(); diff --git a/platforms/guardoni/src/guardoni/cli.ts b/platforms/guardoni/src/guardoni/cli.ts index dd0fd4d46..eab3998fa 100644 --- a/platforms/guardoni/src/guardoni/cli.ts +++ b/platforms/guardoni/src/guardoni/cli.ts @@ -29,6 +29,7 @@ export const cliLogger = guardoniLogger.extend('cli'); export interface GuardoniCommandOpts { headless?: boolean; researchTag?: string; + experimentId?: NonEmptyString; publicKey: string; secretKey: string; } @@ -178,7 +179,7 @@ export const GetGuardoniCLI: GetGuardoniCLI = ( return g.runExperiment(command.experiment, command.opts); case 'navigate': { return pipe( - g.runNavigate({ ...config, ...command.opts }), + g.runNavigate(command.opts), TE.map(() => ({ type: 'success', values: [], @@ -255,6 +256,7 @@ const runGuardoni = ({ 'secret-key': _secretKey, 'cookie-modal': _cookiModal, 'research-tag': _researchTag, + 'experiment-id': _experimentId, ...guardoniConf }: any): Promise => { const basePath = guardoniConf.basePath ?? DEFAULT_BASE_PATH; @@ -264,7 +266,7 @@ const runGuardoni = ({ } cliLogger.debug( - 'Running guardoni from base path %s (%s), %O', + 'Running guardoni from base path %s%s: %O', basePath, config, guardoniConf @@ -322,6 +324,11 @@ const program = yargs(hideBin(process.argv)) desc: 'The secretKey to use to sign the evidences', default: undefined, }) + .option('experimentId', { + type: 'string', + desc: 'Setup the browser for the given experiment', + default: undefined, + }) .option('cookie-modal', { type: 'string', choices: ['accept', 'reject'], @@ -329,13 +336,14 @@ const program = yargs(hideBin(process.argv)) .option('exit', { type: 'boolean', }), - (args) => { + ({ publicKey, secretKey, experimentId, researchTag, ...args }) => { void runGuardoni({ ...args, + headless: false, platform: 'youtube', command: { run: 'navigate', - opts: args, + opts: { publicKey, secretKey, experimentId, researchTag }, }, }); } diff --git a/platforms/guardoni/src/guardoni/guardoni.ts b/platforms/guardoni/src/guardoni/guardoni.ts index 04c733570..ae52f20a2 100755 --- a/platforms/guardoni/src/guardoni/guardoni.ts +++ b/platforms/guardoni/src/guardoni/guardoni.ts @@ -77,10 +77,17 @@ const runNavigate = return pipe( downloadExtension(ctx), - TE.map(() => setLocalSettings(ctx)(opts)), + TE.map(() => + setLocalSettings(ctx)({ + publicKey: opts?.publicKey, + secretKey: opts?.secretKey, + researchTag: opts?.researchTag, + experimentId: opts?.experimentId, + }) + ), TE.chain(() => dispatchBrowser(ctx)({ - headless: opts?.headless !== undefined ? opts.headless : true, + headless: false, }) ), TE.chain((b) => { @@ -197,7 +204,7 @@ export const runExperiment = ), expId: TE.right(expId), localSettings: TE.right( - setLocalSettings(ctx)({ ...opts, experimentId }) + setLocalSettings(ctx)({ ...opts, experimentId, active: true }) ), }) ), @@ -207,7 +214,18 @@ export const runExperiment = TE.chain((data) => { return pipe( saveExperiment(ctx)(expId, profile), - TE.chain((exp) => runBrowser(ctx)(exp, data, opts)) + TE.chain((exp) => runBrowser(ctx)(exp, data, opts)), + TE.chainFirst(() => + TE.right( + setLocalSettings(ctx)({ + active: true, + publicKey: undefined, + secretKey: undefined, + experimentId: undefined, + researchTag: undefined, + }) + ) + ) ); }) ) diff --git a/platforms/tktrex/backend/routes/metadata.ts b/platforms/tktrex/backend/routes/metadata.ts index 13f7eb7ee..2c727f56f 100644 --- a/platforms/tktrex/backend/routes/metadata.ts +++ b/platforms/tktrex/backend/routes/metadata.ts @@ -33,11 +33,9 @@ const listMetadata = async ( if (experimentId) { filter.experimentId = experimentId; } - if (researchTag) { filter.researchTag = researchTag; } - if (nature) { filter['nature.type'] = nature; } @@ -58,7 +56,7 @@ const listMetadata = async ( })), })); - debug('Metadata by %O, %d evidences', filter, _.size(metadata)); + debug('Fetched %d evidences of %d', _.size(metadata.data), metadata.totals); return { json: metadata }; }; diff --git a/platforms/yttrex/backend/routes/metadata.ts b/platforms/yttrex/backend/routes/metadata.ts index 663209584..029b9f529 100644 --- a/platforms/yttrex/backend/routes/metadata.ts +++ b/platforms/yttrex/backend/routes/metadata.ts @@ -36,15 +36,15 @@ const listMetadata = async ( if (nature) { filter.type = nature; } - if (experimentId) { filter.experimentId = experimentId; } - if (researchTag) { filter.researchTag = researchTag; } + debug('Filtering metadata with %O (%d, %d)', filter, amount, skip); + const metadata = await automo .getMetadataByFilter(filter, { amount, @@ -65,13 +65,13 @@ const listMetadata = async ( })); debug( - 'Returning metadata by experimentId %s, %d evidences', - experimentId, - _.size(metadata) + 'Returning %d evidences of %d available', + _.size(metadata.data), + metadata.totals ); if (format === 'csv') { - const csv = CSV.produceCSVv1(metadata); + const csv = CSV.produceCSVv1(metadata.data); let filename = `metadata`; filename += experimentId ? `-experiment-${experimentId}` : ''; filename += researchTag ? `-research_tag-${researchTag}` : ''; diff --git a/platforms/yttrex/extension/src/background/chromeConfig.ts b/platforms/yttrex/extension/src/background/chromeConfig.ts index a68f91ecf..9fd4529e1 100644 --- a/platforms/yttrex/extension/src/background/chromeConfig.ts +++ b/platforms/yttrex/extension/src/background/chromeConfig.ts @@ -3,7 +3,7 @@ import { bo } from '@shared/extension/utils/browser.utils'; bo.runtime.onMessage.addListener((request, sender, sendResponse) => { if (request.type === 'chromeConfig') { - void db.get('/settings').then((_settings: any) => { + void db.get('settings').then((_settings: any) => { const settings = _settings ? { ..._settings,