test(tk:backend): defined search parser e2e suite

tracking-exposed · Oct 4, 2022 · d2e7b1b · d2e7b1b
1 parent ff65a76
commit d2e7b1b
Show file tree

Hide file tree

Showing 4 changed files with 1,325 additions and 25 deletions.
diff --git a/...ms/tktrex/backend/__tests__/fixtures/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json b/...ms/tktrex/backend/__tests__/fixtures/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json
diff --git a/platforms/tktrex/backend/__tests__/search.e2e.ts b/platforms/tktrex/backend/__tests__/search.e2e.ts
@@ -0,0 +1,142 @@
+import {
+  readFixtureJSON,
+  readFixtureJSONPaths,
+  runParserTest,
+} from '@shared/test/utils/parser.utils';
+import { TKMetadata } from '@tktrex/shared/models';
+import { toMetadata } from '@tktrex/shared/parser/metadata';
+import { parsers } from '@tktrex/shared/parser/parsers';
+import { HTMLSource } from '@tktrex/shared/parser/source';
+import base58 from 'bs58';
+import { parseISO, subMinutes } from 'date-fns';
+import path from 'path';
+import nacl from 'tweetnacl';
+import { v4 as uuid } from 'uuid';
+import {
+  addDom,
+  getLastHTMLs,
+  getMetadata,
+  getMetadataSchema,
+  getSourceSchema,
+  parserConfig,
+  updateMetadataAndMarkHTML,
+} from '../lib/parser';
+import { GetTest, Test } from '../test/Test';
+
+describe('Parser: "search"', () => {
+  let appTest: Test;
+  const newKeypair = nacl.sign.keyPair();
+  const publicKey = base58.encode(newKeypair.publicKey);
+
+  let db: any;
+  beforeAll(async () => {
+    appTest = await GetTest();
+
+    db = {
+      api: appTest.mongo3,
+      read: appTest.mongo,
+      write: appTest.mongo,
+    };
+  });
+
+  afterEach(async () => {
+    await appTest.mongo3.deleteMany(
+      appTest.mongo,
+      appTest.config.get('schema').htmls,
+      {
+        publicKey: {
+          $eq: publicKey,
+        },
+      }
+    );
+  });
+
+  describe('Nature Search', () => {
+    jest.setTimeout(20 * 1000);
+
+    const history = readFixtureJSONPaths(
+      path.resolve(__dirname, 'fixtures/search')
+    );
+
+    test.each(history)(
+      'Should correctly parse "search" contribution from path %s',
+      async (fixturePath) => {
+        const { sources: _sources, metadata } = readFixtureJSON(
+          fixturePath,
+          publicKey
+        );
+        const sources = _sources.map((s: any) => ({
+          html: {
+            ...s,
+            id: uuid(),
+            clientTime: parseISO(s.clientTime ?? new Date().toISOString()),
+            savingTime: subMinutes(new Date(), 2),
+          },
+          supporter: { version: process.env.VERSION },
+        }));
+
+        await runParserTest({
+          name: 'native-parser',
+          log: appTest.logger,
+          parsers: parsers,
+          db,
+          codecs: {
+            contribution: HTMLSource,
+            metadata: TKMetadata.TKMetadata,
+          },
+          addDom,
+          sourceSchema: getSourceSchema(),
+          metadataSchema: getMetadataSchema(),
+          getEntryId: (e) => e.html.id,
+          getEntryDate: (e) => e.html.savingTime,
+          getEntryNatureType: (e) => e.html.type,
+          getContributions: getLastHTMLs(db),
+          getMetadata: getMetadata(db),
+          saveResults: updateMetadataAndMarkHTML(db),
+          buildMetadata: toMetadata,
+          config: parserConfig,
+          expectSources: (receivedSources) => {
+            receivedSources.forEach((s) => {
+              expect((s as any).processed).toBe(true);
+            });
+          },
+          expectMetadata: (
+            receivedMetadata: TKMetadata.SearchMetadata,
+            expectedMetadata: TKMetadata.SearchMetadata
+          ) => {
+            const {
+              _id: received_Id,
+              id: receivedId,
+              clientTime: clientTimeR,
+              savingTime: savingTimeR,
+              results: resultsR,
+              ...receivedM
+            } = receivedMetadata;
+
+            const {
+              _id: _received_Id,
+              id: _receivedId,
+              clientTime: clientTimeExp,
+              savingTime: savingTimeExp,
+              results: resultsExp,
+              ...expectedM
+            } = expectedMetadata;
+
+            expect(receivedM).toMatchObject(expectedM);
+            expect(
+              resultsR.map((r) => ({
+                ...r,
+                publishingDate: parseISO(r.publishingDate),
+              }))
+            ).toMatchObject(
+              resultsExp.map((r) => ({
+                ...r,
+                publishingDate: expect.any(Date),
+              }))
+            );
+          },
+        })({ sources, metadata });
+      }
+    );
+  });
+});
diff --git a/platforms/tktrex/backend/routes/__tests__/metadata.e2e.ts b/platforms/tktrex/backend/routes/__tests__/metadata.e2e.ts
@@ -37,17 +37,21 @@ describe('Metadata API', () => {
       const experimentId = fc.sample(fc.uuid(), 1)[0];
       const amount = 10;
 
-      const metadataWithExperimentId = fc.sample(MetadataArb, 100).map((m) => ({
-        ...m,
-        savingTime: new Date(),
-        experimentId,
-      }));
-
-      const metadataWithResearchTag = fc.sample(MetadataArb, 100).map((m) => ({
-        ...m,
-        savingTime: new Date(),
-        researchTag,
-      }));
+      const metadataWithExperimentId = fc
+        .sample(MetadataArb, 100)
+        .map(({ _id, ...m }) => ({
+          ...m,
+          savingTime: new Date(),
+          experimentId,
+        }));
+
+      const metadataWithResearchTag = fc
+        .sample(MetadataArb, 100)
+        .map(({ _id, ...m }) => ({
+          ...m,
+          savingTime: new Date(),
+          researchTag,
+        }));
 
       await test.mongo3.insertMany(
         test.mongo,
@@ -58,7 +62,7 @@ describe('Metadata API', () => {
       const expectedMetadata = metadataWithResearchTag
         .sort((a, b) => b.savingTime.getTime() - a.savingTime.getTime())
         .slice(0, amount)
-        .map((m) => {
+        .map(({ _id, ...m }) => {
           return {
             ...m,
             clientTime: m.clientTime.toISOString(),
@@ -95,17 +99,21 @@ describe('Metadata API', () => {
       const experimentId = fc.sample(fc.uuid(), 1)[0];
       const amount = 10;
 
-      const metadataWithExperimentId = fc.sample(MetadataArb, 100).map((m) => ({
-        ...m,
-        savingTime: new Date(),
-        experimentId,
-      }));
-
-      const metadataWithResearchTag = fc.sample(MetadataArb, 100).map((m) => ({
-        ...m,
-        savingTime: new Date(),
-        researchTag,
-      }));
+      const metadataWithExperimentId = fc
+        .sample(MetadataArb, 100)
+        .map(({ _id, ...m }) => ({
+          ...m,
+          savingTime: new Date(),
+          experimentId,
+        }));
+
+      const metadataWithResearchTag = fc
+        .sample(MetadataArb, 100)
+        .map(({ _id, ...m }) => ({
+          ...m,
+          savingTime: new Date(),
+          researchTag,
+        }));
 
       await test.mongo3.insertMany(
         test.mongo,

diff --git a/platforms/tktrex/shared/src/models/Metadata.ts b/platforms/tktrex/shared/src/models/Metadata.ts
@@ -49,7 +49,9 @@ export type Metrics = t.TypeOf<typeof Metrics>;
 
 export const MetadataBase = t.type(
   {
+    _id: t.unknown,
     id: t.string,
+    clientTime: t.string,
     savingTime: t.string,
     publicKey: t.string,
   },
@@ -107,15 +109,25 @@ export const FollowingVideoMetadata = t.intersection(
 
 export type FollowingVideoMetadata = t.TypeOf<typeof FollowingVideoMetadata>;
 
+const SearchMetadataResult = t.type(
+  {
+    order: t.number,
+    video: t.any,
+    textdesc: t.string,
+    linked: t.array(t.any),
+    thumbnail: t.string,
+    publishingDate: t.string,
+  },
+  'SearchMetadataResult',
+);
 export const SearchMetadata = t.intersection(
   [
     MetadataBase,
     SearchN,
     t.type({ nature: SearchN }),
     t.type(
       {
-        results: t.array(t.any),
-        query: t.string,
+        results: t.array(SearchMetadataResult),
         thumbnails: t.array(
           t.type({
             downloaded: t.boolean,