fix(guardoni): set experiment in settings.json correctly (#758)

Co-authored-by: Claudio Agosti <[email protected]>
tracking-exposed · Nov 14, 2022 · 8ebd5a4 · 8ebd5a4
1 parent f76e336
commit 8ebd5a4
Show file tree

Hide file tree

Showing 9 changed files with 78 additions and 48 deletions.
diff --git a/packages/shared/src/extension/app.ts b/packages/shared/src/extension/app.ts
@@ -287,10 +287,10 @@ export async function boot(opts: BootOpts): Promise<App> {
     throw localSettings.error;
   }
 
-  // merge settings taken from json with ones stored in db, giving the precedence to the latter
+  // merge settings taken from db with ones defined in settings.json, giving the precedence to the latter
   const settings: UserSettings = {
-    ...jsonSettings.result,
     ...localSettings.result,
+    ...jsonSettings.result,
   } as any;
 
   if (!settings.publicKey || !settings.secretKey) {

diff --git a/platforms/guardoni/__tests__/cli/cli-yt.spec.ts b/platforms/guardoni/__tests__/cli/cli-yt.spec.ts
@@ -260,9 +260,7 @@ describe('CLI', () => {
       );
 
       expect(settingJson).toMatchObject({
-        publicKey: tests.publicKey,
-        secretKey: tests.secretKey,
-        experimentId,
+        active: true,
       });
 
       expect(result).toMatchObject({

diff --git a/platforms/guardoni/__tests__/guardoni.spec.ts b/platforms/guardoni/__tests__/guardoni.spec.ts
@@ -11,6 +11,7 @@ import { getDefaultProfile, getProfileDataDir } from '../src/guardoni/profile';
 import { csvStringifyTE } from '@shared/utils/csv.utils';
 import { guardoniLogger } from '../src/logger';
 import { fc } from '@shared/test';
+import { throwTE } from '@shared/utils/task.utils';
 
 const directiveLinks = [
   {
@@ -53,6 +54,7 @@ describe('Guardoni', () => {
   const profile = 'profile-test-99';
   const emptyCSVTestFileName = 'yt-videos-test-empty.csv';
   const csvTestFileName = 'trex-yt-videos.csv';
+  const csvTestPath = path.resolve(basePath, 'experiments', csvTestFileName);
   const keys = {
     publicKey: process.env.PUBLIC_KEY,
     secretKey: process.env.SECRET_KEY,
@@ -86,20 +88,16 @@ describe('Guardoni', () => {
   };
 
   beforeAll(async () => {
-    const csvContent = await csvStringifyTE(directiveLinks, {
-      header: true,
-      encoding: 'utf-8',
-    })();
-
-    if (csvContent._tag === 'Left') {
-      throw csvContent.left as any;
-    }
-    fs.writeFileSync(
-      path.resolve(basePath, 'experiments', csvTestFileName),
-      csvContent.right,
-      'utf-8'
+    const csvContent = await pipe(
+      csvStringifyTE(directiveLinks, {
+        header: true,
+        encoding: 'utf-8',
+      }),
+      throwTE
     );
 
+    fs.writeFileSync(csvTestPath, csvContent, 'utf-8');
+
     const profileUDD = getProfileDataDir(basePath, profile);
     const profileExists = fs.existsSync(profileUDD);
 
@@ -123,7 +121,7 @@ describe('Guardoni', () => {
       });
     }
 
-    fs.rmSync(path.resolve(basePath, 'experiments', csvTestFileName));
+    fs.rmSync(csvTestPath);
   });
 
   describe('config', () => {
@@ -278,9 +276,7 @@ describe('Guardoni', () => {
         guardoni,
         TE.chain((g) =>
           pipe(
-            g.registerExperimentFromCSV(
-              path.resolve(basePath, 'experiments', csvTestFileName) as any
-            ),
+            g.registerExperimentFromCSV(csvTestPath as any),
             TE.chain((output) => g.runExperiment(output.values[0].experimentId))
           )
         )

diff --git a/platforms/guardoni/scripts/cli-test-from-fixtures.mjs b/platforms/guardoni/scripts/cli-test-from-fixtures.mjs
@@ -103,28 +103,40 @@ void (async function () {
   const yt_home_experiment_register_out =
     await $`${cli} ${flags} ${p}-register ${experimentFile}`;
 
-  const yt_home_experiment_id = yt_home_experiment_register_out.stdout
+  const experiment_id = yt_home_experiment_register_out.stdout
     .split('\n')
     .find((s) => s.startsWith('experimentId:'))
     .replace('experimentId: \t', '')
     .trim();
 
   const yt_home_experiment_run_out =
-    await $`${cli} ${experimentFlags} ${p}-experiment ${yt_home_experiment_id} | grep 'publicKey: ' `;
+    await $`${cli} ${experimentFlags} ${p}-experiment ${experiment_id} | grep 'publicKey: ' `;
 
-  const yt_home_experiment_public_key = yt_home_experiment_run_out.stdout
+  const supporter_public_key = yt_home_experiment_run_out.stdout
     .replace('publicKey: \t ', '')
     .replace('\n', '');
 
-  assert.strictEqual(yt_home_experiment_public_key, process.env.PUBLIC_KEY);
+  assert.strictEqual(supporter_public_key, process.env.PUBLIC_KEY);
 
   const backend = p === 'tk' ? process.env.TK_BACKEND : process.env.YT_BACKEND;
-  const personalURL = `${backend}/v2/personal/${yt_home_experiment_public_key}/${nature}/json`;
-  await $`echo ${personalURL}`;
+  const personalURL = `${backend}/v2/personal/${supporter_public_key}/${nature}/json`;
+  const metadataURL = `${backend}/v2/metadata?publicKey=${supporter_public_key}&experimentId=${experiment_id}&nature=${nature}`;
 
-  // const metadata = await fetch(
-  //   `http://localhost:9000/api/v2/metadata?publicKey=${yt_home_experiment_public_key}&experimentId=${yt_home_experiment_id}&nature=home`
-  // ).then((r) => r.json());
+  await $`echo ${personalURL}`;
+  await $`echo ${metadataURL}`;
+  const metadata = await fetch(metadataURL).then((r) => r.json());
+
+  assert.strictEqual(metadata.length, sources.length);
+  assert.strictEqual(
+    metadata.map((m) => ({
+      experimentId: m.experimentId,
+      type: m.type,
+    })),
+    Array.from({ length: sources.length }).map(() => ({
+      experimentId: experiment_id,
+      type: nature,
+    }))
+  );
 
   fs.removeSync(experimentFile);
 })();
diff --git a/platforms/guardoni/src/guardoni/cli.ts b/platforms/guardoni/src/guardoni/cli.ts
@@ -29,6 +29,7 @@ export const cliLogger = guardoniLogger.extend('cli');
 export interface GuardoniCommandOpts {
   headless?: boolean;
   researchTag?: string;
+  experimentId?: NonEmptyString;
   publicKey: string;
   secretKey: string;
 }
@@ -178,7 +179,7 @@ export const GetGuardoniCLI: GetGuardoniCLI = (
               return g.runExperiment(command.experiment, command.opts);
             case 'navigate': {
               return pipe(
-                g.runNavigate({ ...config, ...command.opts }),
+                g.runNavigate(command.opts),
                 TE.map(() => ({
                   type: 'success',
                   values: [],
@@ -255,6 +256,7 @@ const runGuardoni = ({
   'secret-key': _secretKey,
   'cookie-modal': _cookiModal,
   'research-tag': _researchTag,
+  'experiment-id': _experimentId,
   ...guardoniConf
 }: any): Promise<void> => {
   const basePath = guardoniConf.basePath ?? DEFAULT_BASE_PATH;
@@ -264,7 +266,7 @@ const runGuardoni = ({
   }
 
   cliLogger.debug(
-    'Running guardoni from base path %s (%s), %O',
+    'Running guardoni from base path %s%s: %O',
     basePath,
     config,
     guardoniConf
@@ -322,20 +324,26 @@ const program = yargs(hideBin(process.argv))
           desc: 'The secretKey to use to sign the evidences',
           default: undefined,
         })
+        .option('experimentId', {
+          type: 'string',
+          desc: 'Setup the browser for the given experiment',
+          default: undefined,
+        })
         .option('cookie-modal', {
           type: 'string',
           choices: ['accept', 'reject'],
         })
         .option('exit', {
           type: 'boolean',
         }),
-    (args) => {
+    ({ publicKey, secretKey, experimentId, researchTag, ...args }) => {
       void runGuardoni({
         ...args,
+        headless: false,
         platform: 'youtube',
         command: {
           run: 'navigate',
-          opts: args,
+          opts: { publicKey, secretKey, experimentId, researchTag },
         },
       });
     }

diff --git a/platforms/guardoni/src/guardoni/guardoni.ts b/platforms/guardoni/src/guardoni/guardoni.ts
@@ -77,10 +77,17 @@ const runNavigate =
 
     return pipe(
       downloadExtension(ctx),
-      TE.map(() => setLocalSettings(ctx)(opts)),
+      TE.map(() =>
+        setLocalSettings(ctx)({
+          publicKey: opts?.publicKey,
+          secretKey: opts?.secretKey,
+          researchTag: opts?.researchTag,
+          experimentId: opts?.experimentId,
+        })
+      ),
       TE.chain(() =>
         dispatchBrowser(ctx)({
-          headless: opts?.headless !== undefined ? opts.headless : true,
+          headless: false,
         })
       ),
       TE.chain((b) => {
@@ -197,7 +204,7 @@ export const runExperiment =
           ),
           expId: TE.right(expId),
           localSettings: TE.right(
-            setLocalSettings(ctx)({ ...opts, experimentId })
+            setLocalSettings(ctx)({ ...opts, experimentId, active: true })
           ),
         })
       ),
@@ -207,7 +214,18 @@ export const runExperiment =
           TE.chain((data) => {
             return pipe(
               saveExperiment(ctx)(expId, profile),
-              TE.chain((exp) => runBrowser(ctx)(exp, data, opts))
+              TE.chain((exp) => runBrowser(ctx)(exp, data, opts)),
+              TE.chainFirst(() =>
+                TE.right(
+                  setLocalSettings(ctx)({
+                    active: true,
+                    publicKey: undefined,
+                    secretKey: undefined,
+                    experimentId: undefined,
+                    researchTag: undefined,
+                  })
+                )
+              )
             );
           })
         )

diff --git a/platforms/tktrex/backend/routes/metadata.ts b/platforms/tktrex/backend/routes/metadata.ts
@@ -33,11 +33,9 @@ const listMetadata = async (
   if (experimentId) {
     filter.experimentId = experimentId;
   }
-
   if (researchTag) {
     filter.researchTag = researchTag;
   }
-
   if (nature) {
     filter['nature.type'] = nature;
   }
@@ -58,7 +56,7 @@ const listMetadata = async (
       })),
     }));
 
-  debug('Metadata by %O, %d evidences', filter, _.size(metadata));
+  debug('Fetched %d evidences of %d', _.size(metadata.data), metadata.totals);
 
   return { json: metadata };
 };

diff --git a/platforms/yttrex/backend/routes/metadata.ts b/platforms/yttrex/backend/routes/metadata.ts
@@ -36,15 +36,15 @@ const listMetadata = async (
   if (nature) {
     filter.type = nature;
   }
-
   if (experimentId) {
     filter.experimentId = experimentId;
   }
-
   if (researchTag) {
     filter.researchTag = researchTag;
   }
 
+  debug('Filtering metadata with %O (%d, %d)', filter, amount, skip);
+
   const metadata = await automo
     .getMetadataByFilter(filter, {
       amount,
@@ -65,13 +65,13 @@ const listMetadata = async (
     }));
 
   debug(
-    'Returning metadata by experimentId %s, %d evidences',
-    experimentId,
-    _.size(metadata)
+    'Returning %d evidences of %d available',
+    _.size(metadata.data),
+    metadata.totals
   );
 
   if (format === 'csv') {
-    const csv = CSV.produceCSVv1(metadata);
+    const csv = CSV.produceCSVv1(metadata.data);
     let filename = `metadata`;
     filename += experimentId ? `-experiment-${experimentId}` : '';
     filename += researchTag ? `-research_tag-${researchTag}` : '';

diff --git a/platforms/yttrex/extension/src/background/chromeConfig.ts b/platforms/yttrex/extension/src/background/chromeConfig.ts
@@ -3,7 +3,7 @@ import { bo } from '@shared/extension/utils/browser.utils';
 
 bo.runtime.onMessage.addListener((request, sender, sendResponse) => {
   if (request.type === 'chromeConfig') {
-    void db.get('/settings').then((_settings: any) => {
+    void db.get('settings').then((_settings: any) => {
       const settings = _settings
         ? {
             ..._settings,