diff --git a/platforms/tktrex/backend/__tests__/fixtures/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json b/platforms/tktrex/backend/__tests__/fixtures/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json new file mode 100644 index 000000000..342b318e7 --- /dev/null +++ b/platforms/tktrex/backend/__tests__/fixtures/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json @@ -0,0 +1,72 @@ +{ + "sources": [ + { + "id": "09324c95bd1d9cb7c8f804f1bd68b9018d8b93b5", + "type": "profile", + "rect": null, + "href": "https://www.tiktok.com/@bramty", + "timelineId": "scone-142deae00b", + "clientTime": "2022-10-04T13:28:03.000Z", + "savingTime": "2022-10-04T13:28:07.874Z", + "html": "

Notifications

All activityLikesCommentsMentions and tagsFollowers

All activity

Notifications about your account will appear here.

bramty

Bramty

95Following
3.5MFollowers
199.1MLikes

✨THE BRAMFAM ✨\n📩 bramfam@therolengroup.com\n⬇️watch our new YouTube video!⬇️

Videos

Liked

Whether you have kids or simply love a full makeup look, @Farmacy Beauty Green Clean Cleansing Balm is here to save the day! You’ll also be shocked at how soft your face feels afterwards!!! #FarmacyPartner
Replying to @hannah_hakawati questions? Comments? Concerns?
Replying to @sarasewa the girlies didn’t like my last outfit 😔💔 I cried all night long and booked a plane ticket back home 😔😔😔
Replying to @aloogm nothing more important than having your kids see a happy and healthy relationship 🥰
Replying to @2345678900jj every man loves a little bit of crazy 😂
Get TikTok App
M 1Decode 0Parser 0
  • foryou: 80df9a4a-3719-462f-bae3-cad870ff2a21

    \"root\":{
    ...
    }
    14 items
If you click this button, Facebook will be able to track your visit to this site.
", + "n": [0, 0, 16, 0, 733386], + "geoip": null, + "experimentId": null, + "processed": true + }, + { + "id": "e74983ae61265b0579cb524b34fded20efe785a7", + "type": "profile", + "rect": null, + "href": "https://www.tiktok.com/@bramty", + "timelineId": "scone-142deae00b", + "clientTime": "2022-10-04T13:28:04.000Z", + "savingTime": "2022-10-04T13:28:07.903Z", + "html": "

Notifications

All activityLikesCommentsMentions and tagsFollowers

All activity

Notifications about your account will appear here.

bramty

Bramty

95Following
3.5MFollowers
199.1MLikes

✨THE BRAMFAM ✨\n📩 bramfam@therolengroup.com\n⬇️watch our new YouTube video!⬇️

Videos

Liked

Whether you have kids or simply love a full makeup look, @Farmacy Beauty Green Clean Cleansing Balm is here to save the day! You’ll also be shocked at how soft your face feels afterwards!!! #FarmacyPartner
Replying to @hannah_hakawati questions? Comments? Concerns?
Replying to @sarasewa the girlies didn’t like my last outfit 😔💔 I cried all night long and booked a plane ticket back home 😔😔😔
Replying to @aloogm nothing more important than having your kids see a happy and healthy relationship 🥰
Replying to @2345678900jj every man loves a little bit of crazy 😂
Get TikTok App
M 1Decode 0Parser 0
  • foryou: 80df9a4a-3719-462f-bae3-cad870ff2a21

    \"root\":{
    ...
    }
    14 items
If you click this button, Facebook will be able to track your visit to this site.
", + "n": [0, 1, 17, 0, 733386], + "geoip": null, + "experimentId": null, + "processed": true + }, + { + "id": "42235ef189ed4c6915f2c78906d492719d795079", + "type": "profile", + "rect": null, + "href": "https://www.tiktok.com/@bramty", + "timelineId": "scone-142deae00b", + "clientTime": "2022-10-04T13:28:07.000Z", + "savingTime": "2022-10-04T13:28:12.355Z", + "html": "

Notifications

All activityLikesCommentsMentions and tagsFollowers

All activity

Notifications about your account will appear here.

bramty

Bramty

95Following
3.5MFollowers
199.1MLikes

✨THE BRAMFAM ✨\n📩 bramfam@therolengroup.com\n⬇️watch our new YouTube video!⬇️

Videos

Liked

Whether you have kids or simply love a full makeup look, @Farmacy Beauty Green Clean Cleansing Balm is here to save the day! You’ll also be shocked at how soft your face feels afterwards!!! #FarmacyPartner
Replying to @hannah_hakawati questions? Comments? Concerns?
Replying to @sarasewa the girlies didn’t like my last outfit 😔💔 I cried all night long and booked a plane ticket back home 😔😔😔
Replying to @aloogm nothing more important than having your kids see a happy and healthy relationship 🥰
Replying to @2345678900jj every man loves a little bit of crazy 😂
#greenscreenvideo who remembers this 😩😩😩😂😂😂
Replying to @thatswhatithoughtlol the fact it had 500+ likes on that comment 😩😩😩😩 gurl I’m 27 🥲
Replying to @aalejandra.aavila I saw so many comments of girls telling ME I don’t how to speak or understand Spanish 😂 girl bye
Replying to @jgm929 any other tests? 😌
#greenscreen what’s wrong with her 😂 also, she has pink eyeshadow on for anyone wondering 😂
Get TikTok App
M 1Decode 0Parser 0
  • foryou: 80df9a4a-3719-462f-bae3-cad870ff2a21

    \"root\":{
    ...
    }
    14 items
If you click this button, Facebook will be able to track your visit to this site.
", + "n": [0, 0, 18, 0, 792708], + "geoip": null, + "experimentId": null, + "processed": true + }, + { + "id": "0676a9f79bfb835f075735a19fce1a8c92e48bda", + "type": "profile", + "rect": null, + "href": "https://www.tiktok.com/@bramty", + "timelineId": "scone-142deae00b", + "clientTime": "2022-10-04T13:28:09.000Z", + "savingTime": "2022-10-04T13:28:12.408Z", + "html": "

Notifications

All activityLikesCommentsMentions and tagsFollowers

All activity

Notifications about your account will appear here.

bramty

Bramty

95Following
3.5MFollowers
199.1MLikes

✨THE BRAMFAM ✨\n📩 bramfam@therolengroup.com\n⬇️watch our new YouTube video!⬇️

Videos

Liked

Whether you have kids or simply love a full makeup look, @Farmacy Beauty Green Clean Cleansing Balm is here to save the day! You’ll also be shocked at how soft your face feels afterwards!!! #FarmacyPartner
Replying to @hannah_hakawati questions? Comments? Concerns?
Replying to @sarasewa the girlies didn’t like my last outfit 😔💔 I cried all night long and booked a plane ticket back home 😔😔😔
Replying to @aloogm nothing more important than having your kids see a happy and healthy relationship 🥰
Replying to @2345678900jj every man loves a little bit of crazy 😂
#greenscreenvideo who remembers this 😩😩😩😂😂😂
Replying to @thatswhatithoughtlol the fact it had 500+ likes on that comment 😩😩😩😩 gurl I’m 27 🥲
Replying to @aalejandra.aavila I saw so many comments of girls telling ME I don’t how to speak or understand Spanish 😂 girl bye
Replying to @jgm929 any other tests? 😌
#greenscreen what’s wrong with her 😂 also, she has pink eyeshadow on for anyone wondering 😂
Get TikTok App
M 1Decode 0Parser 0
  • foryou: 80df9a4a-3719-462f-bae3-cad870ff2a21

    \"root\":{
    ...
    }
    14 items
If you click this button, Facebook will be able to track your visit to this site.
", + "n": [0, 2, 20, 0, 792690], + "geoip": null, + "experimentId": null, + "processed": true + } + ], + "metadata": { + "timelineId": "scone-142deae00b", + "clientTime": "2022-10-04T13:28:19.000Z", + "creatorName": "@bramty", + "id": "876a43574215629696bccb876a6f7660d3dc2dc9", + "nature": { "type": "creator", "creatorName": "@bramty" }, + "order": 0, + "publicKey": "H7AsuUszehN4qKTj2GYYwNNzkJVqUQBRo2wgKevzeUwx", + "savingTime": "2022-10-04T13:28:23.792Z", + "thumbnails": [], + "type": "creator" + } +} diff --git a/platforms/tktrex/backend/__tests__/profile.e2e.ts b/platforms/tktrex/backend/__tests__/profile.e2e.ts new file mode 100644 index 000000000..18df4acce --- /dev/null +++ b/platforms/tktrex/backend/__tests__/profile.e2e.ts @@ -0,0 +1,125 @@ +import { + readFixtureJSON, + readFixtureJSONPaths, + runParserTest, +} from '@shared/test/utils/parser.utils'; +import { v4 as uuid } from 'uuid'; +import { TKMetadata } from '@tktrex/shared/models'; +import { parsers } from '@tktrex/shared/parser/parsers'; +import base58 from 'bs58'; +import { parseISO, subMinutes } from 'date-fns'; +import path from 'path'; +import nacl from 'tweetnacl'; +import { GetTest, Test } from '../test/Test'; +import { + addDom, + getLastHTMLs, + getMetadata, + getMetadataSchema, + getSourceSchema, + parserConfig, + updateMetadataAndMarkHTML, +} from '../lib/parser'; +import { HTMLSource } from '@tktrex/shared/parser/source'; +import { toMetadata } from '@tktrex/shared/parser/metadata'; + +describe('Parser: "profile"', () => { + let appTest: Test; + const newKeypair = nacl.sign.keyPair(); + const publicKey = base58.encode(newKeypair.publicKey); + + let db: any; + beforeAll(async () => { + appTest = await GetTest(); + + db = { + api: appTest.mongo3, + read: appTest.mongo, + write: appTest.mongo, + }; + }); + + afterEach(async () => { + await appTest.mongo3.deleteMany( + appTest.mongo, + appTest.config.get('schema').htmls, + { + publicKey: { + $eq: publicKey, + }, + } + ); + }); + + describe('Nature Profile', () => { + jest.setTimeout(20 * 1000); + + const history = readFixtureJSONPaths( + path.resolve(__dirname, 'fixtures/profile') + ); + + test.each(history)( + 'Should correctly parse "profile" contribution from path %s', + async (fixturePath) => { + const { sources: _sources, metadata } = readFixtureJSON( + fixturePath, + publicKey + ); + const sources = _sources.map((s: any) => ({ + html: { + ...s, + id: uuid(), + clientTime: parseISO(s.clientTime ?? new Date().toISOString()), + savingTime: subMinutes(new Date(), 2), + }, + supporter: { version: process.env.VERSION }, + })); + + await runParserTest({ + name: 'native-parser', + log: appTest.logger, + parsers: parsers, + db, + codecs: { + contribution: HTMLSource, + metadata: TKMetadata.TKMetadata, + }, + addDom, + sourceSchema: getSourceSchema(), + metadataSchema: getMetadataSchema(), + getEntryId: (e) => e.html.id, + getEntryDate: (e) => e.html.savingTime, + getEntryNatureType: (e) => e.html.type, + getContributions: getLastHTMLs(db), + getMetadata: getMetadata(db), + saveResults: updateMetadataAndMarkHTML(db), + buildMetadata: toMetadata, + config: parserConfig, + expectSources: (receivedSources) => { + receivedSources.forEach((s) => { + expect((s as any).processed).toBe(true); + }); + }, + expectMetadata: (receivedMetadata, expectedMetadata) => { + const { + _id: received_Id, + id: receivedId, + savingTime: savingTimeR, + ...receivedM + } = receivedMetadata as any; + + const { + _id: _received_Id, + id: _receivedId, + clientTime: clientTimeExp, + savingTime: savingTimeExp, + ...expectedM + } = expectedMetadata as any; + + expect(receivedM).toMatchObject(expectedM); + }, + })({ sources, metadata }); + } + ); + }); +}); diff --git a/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts b/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts index 001d5dcf0..c7fd25465 100644 --- a/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts +++ b/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts @@ -12,9 +12,9 @@ import { ContributionEventArb } from '@tktrex/shared/arbitraries/ContributionEve import { TKMetadata } from '@tktrex/shared/models/Metadata'; import { parsers } from '@tktrex/shared/parser/parsers'; import { HTMLSource } from '@tktrex/shared/parser/source'; +import { toMetadata } from '@tktrex/shared/parser/metadata'; import { addDom, - buildMetadata, getLastHTMLs, getMetadata, updateMetadataAndMarkHTML, @@ -60,7 +60,7 @@ describe('/v2/personal', () => { getMetadata: getMetadata(db), getEntryDate: (e) => e.html.savingTime, getEntryNatureType: (e) => e.html.type, - buildMetadata: buildMetadata, + buildMetadata: toMetadata, saveResults: updateMetadataAndMarkHTML(db), }); }); diff --git a/platforms/tktrex/shared/src/models/Metadata.ts b/platforms/tktrex/shared/src/models/Metadata.ts index 5a713bee8..fe81599b2 100644 --- a/platforms/tktrex/shared/src/models/Metadata.ts +++ b/platforms/tktrex/shared/src/models/Metadata.ts @@ -1,5 +1,5 @@ import * as t from 'io-ts'; -import { FollowingN, ForYouN, SearchN, NativeVideoN } from './Nature'; +import { FollowingN, ForYouN, SearchN, NativeVideoN, ProfileN } from './Nature'; const Music = t.type( { @@ -165,8 +165,26 @@ export const NativeMetadata = t.intersection( export type NativeMetadata = t.TypeOf; +export const ProfileMetadata = t.intersection( + [ + MetadataBase, + ProfileN, + t.type({ + videos: t.array(t.any), + }), + ], + 'ProfileMetadata', +); +export type ProfileMetadata = t.TypeOf; + export const TKMetadata = t.union( - [ForYouVideoMetadata, FollowingVideoMetadata, SearchMetadata, NativeMetadata], + [ + ForYouVideoMetadata, + FollowingVideoMetadata, + SearchMetadata, + NativeMetadata, + ProfileMetadata, + ], 'TKMetadata', ); diff --git a/platforms/tktrex/shared/src/models/Nature.ts b/platforms/tktrex/shared/src/models/Nature.ts index 5a0021dab..d8bc8caf2 100644 --- a/platforms/tktrex/shared/src/models/Nature.ts +++ b/platforms/tktrex/shared/src/models/Nature.ts @@ -25,14 +25,6 @@ export const FollowingN = t.type( export type FollowingN = t.TypeOf; -export const CreatorN = t.type( - { - type: CreatorType, - }, - 'CreatorN', -); -export type CreatorN = t.TypeOf; - export const VideoN = t.type( { type: VideoType, @@ -65,7 +57,7 @@ export type NativeVideoN = t.TypeOf; export const ProfileN = t.type( { - type: t.literal('profile'), + type: t.literal('creator'), creatorName: t.string, }, 'ProfileN', @@ -83,16 +75,7 @@ export const HashtagsN = t.type( export type HashtagsN = t.TypeOf; export const Nature = t.union( - [ - ForYouN, - FollowingN, - CreatorN, - VideoN, - SearchN, - ProfileN, - HashtagsN, - NativeVideoN, - ], + [ForYouN, FollowingN, VideoN, SearchN, ProfileN, HashtagsN, NativeVideoN], 'Nature', ); diff --git a/platforms/tktrex/shared/src/parser/metadata.ts b/platforms/tktrex/shared/src/parser/metadata.ts index f0df8227b..ad4206489 100644 --- a/platforms/tktrex/shared/src/parser/metadata.ts +++ b/platforms/tktrex/shared/src/parser/metadata.ts @@ -54,9 +54,10 @@ export const toMetadata: BuildMetadataFn = ( metadata.nature.query = metadata.query; break; } - case 'profile': { + case 'creator': { const { nature, profile, downloader } = entry.findings; metadata = { + videos: [], ...metadata, nature, ...nature,