Skip to content
This repository has been archived by the owner on Aug 15, 2023. It is now read-only.

Commit

Permalink
test(tk:backend): defined profile parser e2e suite
Browse files Browse the repository at this point in the history
  • Loading branch information
ascariandrea committed Oct 5, 2022
1 parent f0db578 commit b0a0d05
Show file tree
Hide file tree
Showing 7 changed files with 183 additions and 25 deletions.

Large diffs are not rendered by default.

125 changes: 125 additions & 0 deletions platforms/tktrex/backend/__tests__/profile.e2e.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
import {
readFixtureJSON,
readFixtureJSONPaths,
runParserTest,
} from '@shared/test/utils/parser.utils';
import { v4 as uuid } from 'uuid';
import { TKMetadata } from '@tktrex/shared/models';
import { parsers } from '@tktrex/shared/parser/parsers';
import base58 from 'bs58';
import { parseISO, subMinutes } from 'date-fns';
import path from 'path';
import nacl from 'tweetnacl';
import { GetTest, Test } from '../test/Test';
import {
addDom,
getLastHTMLs,
getMetadata,
getMetadataSchema,
getSourceSchema,
parserConfig,
updateMetadataAndMarkHTML,
} from '../lib/parser';
import { HTMLSource } from '@tktrex/shared/parser/source';
import { toMetadata } from '@tktrex/shared/parser/metadata';

describe('Parser: "profile"', () => {
let appTest: Test;
const newKeypair = nacl.sign.keyPair();
const publicKey = base58.encode(newKeypair.publicKey);

let db: any;
beforeAll(async () => {
appTest = await GetTest();

db = {
api: appTest.mongo3,
read: appTest.mongo,
write: appTest.mongo,
};
});

afterEach(async () => {
await appTest.mongo3.deleteMany(
appTest.mongo,
appTest.config.get('schema').htmls,
{
publicKey: {
$eq: publicKey,
},
}
);
});

describe('Nature Profile', () => {
jest.setTimeout(20 * 1000);

const history = readFixtureJSONPaths(
path.resolve(__dirname, 'fixtures/profile')
);

test.each(history)(
'Should correctly parse "profile" contribution from path %s',
async (fixturePath) => {
const { sources: _sources, metadata } = readFixtureJSON(
fixturePath,
publicKey
);
const sources = _sources.map((s: any) => ({
html: {
...s,
id: uuid(),
clientTime: parseISO(s.clientTime ?? new Date().toISOString()),
savingTime: subMinutes(new Date(), 2),
},
supporter: { version: process.env.VERSION },
}));

await runParserTest({
name: 'native-parser',
log: appTest.logger,
parsers: parsers,
db,
codecs: {
contribution: HTMLSource,
metadata: TKMetadata.TKMetadata,
},
addDom,
sourceSchema: getSourceSchema(),
metadataSchema: getMetadataSchema(),
getEntryId: (e) => e.html.id,
getEntryDate: (e) => e.html.savingTime,
getEntryNatureType: (e) => e.html.type,
getContributions: getLastHTMLs(db),
getMetadata: getMetadata(db),
saveResults: updateMetadataAndMarkHTML(db),
buildMetadata: toMetadata,
config: parserConfig,
expectSources: (receivedSources) => {
receivedSources.forEach((s) => {
expect((s as any).processed).toBe(true);
});
},
expectMetadata: (receivedMetadata, expectedMetadata) => {
const {
_id: received_Id,
id: receivedId,
savingTime: savingTimeR,
...receivedM
} = receivedMetadata as any;

const {
_id: _received_Id,
id: _receivedId,
clientTime: clientTimeExp,
savingTime: savingTimeExp,
...expectedM
} = expectedMetadata as any;

expect(receivedM).toMatchObject(expectedM);
},
})({ sources, metadata });
}
);
});
});
2 changes: 1 addition & 1 deletion platforms/tktrex/backend/routes/__tests__/metadata.e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ describe('Metadata API', () => {
const expectedMetadata = metadataWithResearchTag
.sort((a, b) => b.savingTime.getTime() - a.savingTime.getTime())
.slice(0, amount)
.map(({ _id, ...m }) => {
.map((m) => {
return {
...m,
clientTime: m.clientTime.toISOString(),
Expand Down
4 changes: 2 additions & 2 deletions platforms/tktrex/backend/routes/__tests__/personal.e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ import { ContributionEventArb } from '@tktrex/shared/arbitraries/ContributionEve
import { TKMetadata } from '@tktrex/shared/models/Metadata';
import { parsers } from '@tktrex/shared/parser/parsers';
import { HTMLSource } from '@tktrex/shared/parser/source';
import { toMetadata } from '@tktrex/shared/parser/metadata';
import {
addDom,
buildMetadata,
getLastHTMLs,
getMetadata,
updateMetadataAndMarkHTML,
Expand Down Expand Up @@ -60,7 +60,7 @@ describe('/v2/personal', () => {
getMetadata: getMetadata(db),
getEntryDate: (e) => e.html.savingTime,
getEntryNatureType: (e) => e.html.type,
buildMetadata: buildMetadata,
buildMetadata: toMetadata,
saveResults: updateMetadataAndMarkHTML(db),
});
});
Expand Down
23 changes: 21 additions & 2 deletions platforms/tktrex/shared/src/models/Metadata.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as t from 'io-ts';
import { FollowingN, ForYouN, SearchN, NativeVideoN } from './Nature';
import { FollowingN, ForYouN, SearchN, NativeVideoN, ProfileN } from './Nature';

const Music = t.type(
{
Expand Down Expand Up @@ -54,6 +54,7 @@ export const MetadataBase = t.type(
clientTime: t.string,
savingTime: t.string,
publicKey: t.string,
timelineId: t.string,
},
'MetadataBase',
);
Expand Down Expand Up @@ -165,8 +166,26 @@ export const NativeMetadata = t.intersection(

export type NativeMetadata = t.TypeOf<typeof NativeMetadata>;

export const ProfileMetadata = t.intersection(
[
MetadataBase,
ProfileN,
t.type({
videos: t.array(t.any),
}),
],
'ProfileMetadata',
);
export type ProfileMetadata = t.TypeOf<typeof ProfileMetadata>;

export const TKMetadata = t.union(
[ForYouVideoMetadata, FollowingVideoMetadata, SearchMetadata, NativeMetadata],
[
ForYouVideoMetadata,
FollowingVideoMetadata,
SearchMetadata,
NativeMetadata,
ProfileMetadata,
],
'TKMetadata',
);

Expand Down
21 changes: 2 additions & 19 deletions platforms/tktrex/shared/src/models/Nature.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,6 @@ export const FollowingN = t.type(

export type FollowingN = t.TypeOf<typeof FollowingN>;

export const CreatorN = t.type(
{
type: CreatorType,
},
'CreatorN',
);
export type CreatorN = t.TypeOf<typeof CreatorN>;

export const VideoN = t.type(
{
type: VideoType,
Expand Down Expand Up @@ -65,7 +57,7 @@ export type NativeVideoN = t.TypeOf<typeof NativeVideoN>;

export const ProfileN = t.type(
{
type: t.literal('profile'),
type: t.literal('creator'),
creatorName: t.string,
},
'ProfileN',
Expand All @@ -83,16 +75,7 @@ export const HashtagsN = t.type(
export type HashtagsN = t.TypeOf<typeof HashtagsN>;

export const Nature = t.union(
[
ForYouN,
FollowingN,
CreatorN,
VideoN,
SearchN,
ProfileN,
HashtagsN,
NativeVideoN,
],
[ForYouN, FollowingN, VideoN, SearchN, ProfileN, HashtagsN, NativeVideoN],
'Nature',
);

Expand Down
3 changes: 2 additions & 1 deletion platforms/tktrex/shared/src/parser/metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,10 @@ export const toMetadata: BuildMetadataFn<HTMLSource, TKMetadata, TKParsers> = (
metadata.nature.query = metadata.query;
break;
}
case 'profile': {
case 'creator': {
const { nature, profile, downloader } = entry.findings;
metadata = {
videos: [],
...metadata,
nature,
...nature,
Expand Down

0 comments on commit b0a0d05

Please sign in to comment.