From bec0a283acd7862cf38d77f73eb2248655524fe6 Mon Sep 17 00:00:00 2001 From: ascariandrea Date: Fri, 16 Sep 2022 18:37:23 +0200 Subject: [PATCH] fix(shared): added metadata logger box to extension --- packages/shared/package.json | 1 + .../contributions/ContributionEvent.arb.ts | 32 ++ packages/shared/src/extension/app.ts | 27 +- .../shared/src/extension/tooltip/index.tsx | 23 - .../ui/components/MetadataLogger.tsx | 477 ++++++++++++++++++ .../{tooltip => ui}/components/tooltip.tsx | 0 packages/shared/src/extension/ui/index.tsx | 69 +++ platforms/storybook/.storybook/main.js | 1 + .../src/stories/MetadataLogger.stories.tsx | 136 +++++ platforms/storybook/tsconfig.json | 12 +- platforms/tktrex/extension/src/app/index.ts | 9 +- .../shared/src/parser/metadata-logger.ts | 61 +++ .../shared/src/parser/parsers/nature.ts | 7 +- platforms/yttrex/extension/src/app/index.ts | 13 +- platforms/yttrex/extension/tsconfig.json | 2 +- .../shared/src/parser/metadata-logger.ts | 66 +++ platforms/yttrex/shared/tsconfig.json | 1 + yarn.lock | 1 + 18 files changed, 895 insertions(+), 43 deletions(-) create mode 100644 packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts delete mode 100644 packages/shared/src/extension/tooltip/index.tsx create mode 100644 packages/shared/src/extension/ui/components/MetadataLogger.tsx rename packages/shared/src/extension/{tooltip => ui}/components/tooltip.tsx (100%) create mode 100644 packages/shared/src/extension/ui/index.tsx create mode 100644 platforms/storybook/src/stories/MetadataLogger.stories.tsx create mode 100644 platforms/tktrex/shared/src/parser/metadata-logger.ts create mode 100644 platforms/yttrex/shared/src/parser/metadata-logger.ts diff --git a/packages/shared/package.json b/packages/shared/package.json index c86fa57b2..6830f1978 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -35,6 +35,7 @@ "react": "^17.0.2", "react-dom": "^17.0.2", "react-i18next": "^11.18.0", + "react-json-view": "^1.21.3", "ts-endpoint": "^2.0.0", "ts-endpoint-express": "^2.0.0", "ts-io-error": "^2.0.0", diff --git a/packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts b/packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts new file mode 100644 index 000000000..e0ae46744 --- /dev/null +++ b/packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts @@ -0,0 +1,32 @@ +import { getArbitrary } from 'fast-check-io-ts'; +import { + ADVContributionEvent, + ContributionEvent, + VideoContributionEvent, +} from '../../models/ContributionEvent'; +import * as t from 'io-ts'; +import fc from 'fast-check'; + +const { ...videoContributionEventProps } = VideoContributionEvent.type.props; +export const VideoContributionEventArb = getArbitrary( + t.strict({ + ...videoContributionEventProps, + }) +).map((cc): any => ({ + ...cc, + clientTime: new Date(), +})); + +const { ...advContributionEventProps } = ADVContributionEvent.type.props; +export const ADVContributionEventArb = getArbitrary( + t.strict({ + ...advContributionEventProps, + }) +).map((cc) => ({ + ...cc, +})); + +export const ContributionEventArb: fc.Arbitrary = fc.oneof( + VideoContributionEventArb, + ADVContributionEventArb +); diff --git a/packages/shared/src/extension/app.ts b/packages/shared/src/extension/app.ts index 5122fa311..17a0db520 100644 --- a/packages/shared/src/extension/app.ts +++ b/packages/shared/src/extension/app.ts @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { ParserConfiguration, ParserFn } from '../providers/parser.provider'; import { HandshakeResponse } from '../models/HandshakeBody'; import { clearCache } from '../providers/dataDonation.provider'; import { FIXED_USER_NAME, initializeKey } from './background/account'; @@ -16,6 +17,7 @@ import log from './logger'; import HubEvent from './models/HubEvent'; import { ServerLookup } from './models/Message'; import UserSettings from './models/UserSettings'; +import { renderUI, RenderUIProps } from './ui'; import { bo } from './utils/browser.utils'; // instantiate a proper logger @@ -67,7 +69,12 @@ interface SetupObserverOpts { onLocationChange: (oldLocation: string, newLocation: string) => void; } -export interface BootOpts { +export interface BootOpts< + S = any, + M = any, + C extends ParserConfiguration = ParserConfiguration, + PP extends Record> = any +> { payload: ServerLookup['payload']; mapLocalConfig: ( c: UserSettings, @@ -79,6 +86,7 @@ export interface BootOpts { onRegister: (h: Hub, config: UserSettings) => void; }; onAuthenticated: (res: any) => void; + ui?: Omit, 'hub'>; } /** @@ -232,7 +240,12 @@ const serverHandshakeP = ( let loading = false; let app: App | undefined; -export async function boot(opts: BootOpts): Promise { +export async function boot< + S = any, + M = any, + C extends ParserConfiguration = ParserConfiguration, + PP extends Record> = any +>(opts: BootOpts): Promise { if (app) { appLog.debug('App already booted!'); return app; @@ -337,12 +350,12 @@ export async function boot(opts: BootOpts): Promise { // register platform specific event handlers opts.hub.onRegister(opts.hub.hub, config); - // emergency button should be used when a supported with - // UX hack in place didn't see any UX change, so they - // can report the problem and we can handle it. - // initializeEmergencyButton(); + // render shared ui if configuration is given + if (opts.ui) { + renderUI({ hub: opts.hub.hub, ...opts.ui }); + } - // because the URL has been for sure reloaded, be sure to also + // because the URL has been for sure reloaded, be sure to also clear cache clearCache(); // send the configuration to the server to register the extension diff --git a/packages/shared/src/extension/tooltip/index.tsx b/packages/shared/src/extension/tooltip/index.tsx deleted file mode 100644 index a460f19de..000000000 --- a/packages/shared/src/extension/tooltip/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import React from 'react'; -import ReactDOM from 'react-dom'; - -import { createTheme, ThemeProvider } from '@material-ui/core/styles'; - -import Tooltip from './components/tooltip'; - -const theme = createTheme({ - typography: { - fontFamily: 'Trex-Regular', - }, -}); - -function main(): void { - ReactDOM.render( - - - , - document.getElementById('yttrex--tooltip') - ); -} - -main(); diff --git a/packages/shared/src/extension/ui/components/MetadataLogger.tsx b/packages/shared/src/extension/ui/components/MetadataLogger.tsx new file mode 100644 index 000000000..cc0242acf --- /dev/null +++ b/packages/shared/src/extension/ui/components/MetadataLogger.tsx @@ -0,0 +1,477 @@ +import { + Accordion, + AccordionDetails, + AccordionSummary, + Box, + Typography, + TypographyProps, +} from '@material-ui/core'; +import * as E from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/function'; +import * as M from 'fp-ts/lib/Map'; +import * as O from 'fp-ts/lib/Option'; +import * as S from 'fp-ts/lib/string'; +import * as t from 'io-ts'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import * as React from 'react'; +import ReactJSON from 'react-json-view'; +import { v4 as uuid } from 'uuid'; +import { + executionLoop, + ParserConfiguration, + ParserContext, + ParserFn, +} from '../../../providers/parser.provider'; +import { Hub } from '../../hub'; +import trexLogger from '../../logger'; + +const log = trexLogger.extend('metadata-logger'); + +export interface MetadataLoggerProps< + S, + M, + C extends ParserConfiguration, + PP extends Record> +> { + hub: Hub; + mapEvent: (id: string, e: unknown) => S | null; + decode: t.Decode; + parser: Omit< + ParserContext, + | 'db' + | 'codecs' + | 'getLastContributions' + | 'getMetadata' + | 'saveResults' + | 'getContributions' + >; +} + +interface MetadataLoggerEntry { + source: S; + metadata?: M; + errors: { + decode: string[]; + parser: string[]; + }; +} + +type MetadataLoggerState = Map>; + +const ParserErrorsHeader: React.FC<{ count: number } & TypographyProps> = ({ + count, + ...props +}) => ( + + Parser {count} + +); + +const MetadataHeader: React.FC<{ count: number } & TypographyProps> = ({ + count, + ...props +}) => ( + + M {count} + +); + +const DecodeErrorsHeader: React.FC<{ count: number } & TypographyProps> = ({ + count, + ...props +}) => ( + + Decode {count} + +); + +export const MetadataLogger = < + S extends any, + M extends any, + C extends ParserConfiguration, + PP extends Record> +>({ + hub, + mapEvent, + decode, + parser, +}: MetadataLoggerProps): JSX.Element => { + const [contributions, setContributions] = React.useState< + MetadataLoggerState + >(new Map()); + + const contributionEntries = pipe(contributions, M.toArray(S.Ord)); + + const getContributions = React.useCallback( + (f, s, a) => { + const sources = contributionEntries.map(([key, c]) => c.source); + log.debug('Get last sources %O', sources); + return Promise.resolve({ sources, errors: 0, overflow: false }); + }, + [contributionEntries] + ); + + const getMetadata = React.useCallback( + async (e: any): Promise => { + log.debug('Store %O', contributions); + const entryId = parser.getEntryId(e); + + const m = pipe( + contributions, + M.lookup(S.Eq)(entryId), + O.filterMap((m) => (m.metadata ? O.some(m.metadata) : O.none)), + O.toNullable + ); + + log.debug('Get metadata from entry id %s => %O', entryId, m); + + return m; + }, + [contributionEntries] + ); + + const saveResults = React.useCallback( + (s: any, m: any) => { + log.debug('Save results: source %O, metadata %O', s, m); + const metadataUpsert = pipe( + contributions, + M.lookup(S.Eq)(s.id), + O.fold( + (): MetadataLoggerEntry => ({ + errors: { + parser: [], + decode: [], + }, + source: s, + metadata: m, + }), + (c) => ({ + ...c, + metadata: m, + }) + ) + ); + log.debug('Old contributions %O', contributions); + pipe( + contributions, + M.upsertAt(S.Eq)(parser.getEntryId(s), metadataUpsert), + (cc) => { + log.debug('New contributions %s', cc); + setContributions(cc); + } + ); + + return Promise.resolve({ + metadata: m, + source: s, + count: { metadata: 1 }, + }); + }, + [contributionEntries] + ); + + const parserCtx: ParserContext = { + ...parser, + log, + db: { api: {} as any, read: {} as any, write: {} as any }, + codecs: { metadata: t.any, contribution: t.any }, + getMetadata, + getContributions, + saveResults, + }; + + React.useEffect(() => { + log.debug('Start parsing %O', M.toArray(S.Ord)(contributions)); + const unprocessedContributions = pipe( + contributions, + // once the parser has finished the entry should have either `metadata` or some `errors.parser` + M.filter((s) => s.metadata === undefined && s.errors.parser.length === 0) + ); + + log.debug( + 'Unprocessed contributions %O', + M.toArray(S.Ord)(unprocessedContributions) + ); + + if (unprocessedContributions.size >= 1) { + void executionLoop(parserCtx)({ + singleUse: true, + htmlAmount: 1, + stop: 1, + }).then((r) => { + log.debug('Parser output %O', r); + if (r.type === 'Success') { + const outputContributions = r.payload.reduce((acc, p) => { + const sourceId = parser.getEntryId(p.source); + + let parserErrors: any[] = []; + if (p.failures) { + parserErrors = Object.entries(p.failures).reduce( + (acc, [key, value]) => { + if (value instanceof Error) { + return acc.concat({ + [key]: value.message, + }); + } + + return acc.concat({ + [key]: value, + }); + }, + [] + ); + } + + const entry = pipe( + acc, + M.lookup(S.Eq)(sourceId), + O.fold( + (): MetadataLoggerEntry => ({ + source: p.source, + metadata: undefined, + errors: { + decode: [], + parser: parserErrors, + }, + }), + (m) => ({ + ...m, + metadata: p.metadata, + errors: { + ...m.errors, + parser: parserErrors, + }, + }) + ) + ); + + return pipe(acc, M.upsertAt(S.Eq)(sourceId, entry)); + }, contributions); + + log.debug('Update contributions %O', outputContributions); + setContributions(outputContributions); + } + }); + } + }, [contributionEntries]); + + const onAnyEvent = (event: any): void => { + // log.debug('event received', event); + const eventId = uuid(); + const datum = mapEvent(eventId, event); + + if (datum === null) { + log.debug('Avoid parsing event type %s', event.type); + return; + } + + log.debug('Decoding event %O', datum); + // log.debug('datum mapped %O', datum); + const contribution = decode(datum); + // when the received event decode fails + // we update the `decodeErrors` map + // to render the errors + + if (E.isLeft(contribution)) { + const errors = PathReporter.report(contribution); + + log.debug('Decode errors %O', errors); + + const decodeErrorUpsert = pipe( + contributions, + M.lookup(S.Eq)(eventId), + O.fold( + (): MetadataLoggerEntry => ({ + errors: { parser: [], decode: errors }, + metadata: undefined, + source: datum, + }), + (c) => ({ + ...c, + errors: { + ...c.errors, + decode: errors, + }, + }) + ) + ); + + pipe( + contributions, + M.upsertAt(S.Eq)(eventId, decodeErrorUpsert), + setContributions + ); + } else { + log.debug('Contribution id %s', eventId); + + pipe( + contributions, + M.lookup(S.Eq)(eventId), + O.fold( + () => ({ + metadata: undefined, + source: contribution.right, + errors: { + decode: [], + parser: [], + }, + }), + (c) => ({ ...c, source: contribution.right }) + ), + (c) => { + log.debug('Contribution %O', c); + pipe(contributions, M.upsertAt(S.Eq)(eventId, c), setContributions); + } + ); + } + }; + + React.useEffect(() => { + hub.onAnyEvent(onAnyEvent); + }, []); + + log.debug('contributions', contributions); + + const { summary, details } = React.useMemo(() => { + const metadata = pipe( + contributions, + M.filter((c) => !!c.metadata) + ); + + const decodeErrors = pipe( + contributions, + M.filter((c) => c.errors.decode.length > 0) + ); + + const parserErrors = pipe( + contributions, + M.filter((c) => c.errors.parser.length > 0) + ); + + return { + summary: ( +
+ + + +
+ ), + details: ( +
+
    + {pipe( + contributions, + M.mapWithIndex((key, data) => ( +
  • +
    + + {(data.metadata as any)?.type}:{' '} + {parser.getEntryId(data.source as any)} + + + + +
    + {data.errors.parser.length > 0 ? ( +
    + + {data.errors.parser.map((l, i) => ( + + {JSON.stringify(l)} + + ))} +
    + ) : null} + + {data.errors.decode.length > 0 ? ( +
    + + + + + {data.errors.decode.map((l, i) => ( + + {l} + + ))} +
    + ) : null} +
    • + )), + M.toArray(S.Ord), + (entries) => entries.map(([k, e]) => e) + )} +
+
+ ), + }; + }, [contributionEntries]); + + return ( + + + {summary} + {details} + + + ); +}; diff --git a/packages/shared/src/extension/tooltip/components/tooltip.tsx b/packages/shared/src/extension/ui/components/tooltip.tsx similarity index 100% rename from packages/shared/src/extension/tooltip/components/tooltip.tsx rename to packages/shared/src/extension/ui/components/tooltip.tsx diff --git a/packages/shared/src/extension/ui/index.tsx b/packages/shared/src/extension/ui/index.tsx new file mode 100644 index 000000000..3cb66badc --- /dev/null +++ b/packages/shared/src/extension/ui/index.tsx @@ -0,0 +1,69 @@ +import { createTheme, ThemeProvider } from '@material-ui/core/styles'; +import React from 'react'; +import ReactDOM from 'react-dom'; +// import Tooltip from './components/tooltip'; +import { ErrorBoundary } from '../../components/Error/ErrorBoundary'; +import { ParserConfiguration, ParserFn } from '../../providers/parser.provider'; +import { Hub } from '../hub'; +import { + MetadataLogger, + MetadataLoggerProps +} from './components/MetadataLogger'; + +const theme = createTheme({ + typography: { + fontFamily: 'Trex-Regular', + }, +}); + +// const TOOLTIP_ID = 'trex--tooltip'; +const METADATA_LOGGER_ID = 'trex--metadata-logger'; + +export interface RenderUIProps< + S, + M, + C extends ParserConfiguration, + PP extends Record> +> { + hub: Hub; + metadataLogger: Omit, 'hub'>; +} + +export function renderUI< + S, + M, + C extends ParserConfiguration, + PP extends Record> +>({ hub, metadataLogger }: RenderUIProps): void { + // const tooltipNode = document.getElementById(TOOLTIP_ID); + // if (!tooltipNode) { + // const node = document.createElement('div'); + + // node.id = TOOLTIP_ID; + // document.body.append(node); + // ReactDOM.render( + // + // + // , + // node + // ); + // } + + const metadataLoggerNode = document.getElementById(METADATA_LOGGER_ID); + if (!metadataLoggerNode) { + const node = document.createElement('div'); + + node.id = METADATA_LOGGER_ID; + document.body.append(node); + ReactDOM.render( + + + + + + + , + node + ); + } +} diff --git a/platforms/storybook/.storybook/main.js b/platforms/storybook/.storybook/main.js index 991c7910d..249b6d650 100644 --- a/platforms/storybook/.storybook/main.js +++ b/platforms/storybook/.storybook/main.js @@ -54,6 +54,7 @@ module.exports = { }, env: (config) => ({ ...config, + DEBUG: '*', BUILD: 'BUILD env variable', BUILD_DATE: new Date().toISOString(), DEVELOPMENT: true, diff --git a/platforms/storybook/src/stories/MetadataLogger.stories.tsx b/platforms/storybook/src/stories/MetadataLogger.stories.tsx new file mode 100644 index 000000000..b8c6a681d --- /dev/null +++ b/platforms/storybook/src/stories/MetadataLogger.stories.tsx @@ -0,0 +1,136 @@ +import { + MetadataLogger, + MetadataLoggerProps, +} from '@shared/extension/ui/components/MetadataLogger'; +import { ParserConfiguration } from '@shared/providers/parser.provider'; +import { Meta, Story } from '@storybook/react'; +import tkHub from '@tktrex/extension/src/handlers/hub'; +import * as tkContributionsArb from '@tktrex/shared/arbitraries/ContributionEvent.arb'; +import * as ytContributionsArb from '@yttrex/shared/arbitraries/ContributionEvent.arb'; +import { metadataLoggerProps as tkMetadataLoggerParserProps } from '@tktrex/shared/parser/metadata-logger'; +import ytHub from '@yttrex/extension/src/handlers/hub'; +import { metadataLoggerParserProps } from '@yttrex/shared/parser/metadata-logger'; +import fc from 'fast-check'; +import * as React from 'react'; + +const meta: Meta> = { + title: 'Example/MetadataLogger', + component: MetadataLogger, +}; + +export default meta; + +const Template: Story< + MetadataLoggerProps & { + platform: 'yttrex' | 'tktrex'; + arbs: { + [key: string]: () => fc.Arbitrary; + }; + } +> = ({ platform, arbs, ...args }) => { + const handleCollect = React.useCallback((k: keyof typeof arbs) => { + const arb = arbs[k](); + args.hub.dispatch(fc.sample(arb, 1)[0]); + }, []); + + return ( +
+
+ {Object.keys(arbs).map((key) => ( + + ))} +
+ +
+ ); +}; + +export const YoutubeBasic = Template.bind({}); +YoutubeBasic.args = { + platform: 'yttrex', + hub: ytHub, + arbs: { + home: () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const fixture = require(`../../../yttrex/backend/__tests__/fixtures/htmls/home/1c144b8476d67668f19cc9e8d4f235890181d26a.json`); + const { html, href } = fixture.sources[0]; + return ytContributionsArb.ContributionEventArb.map((e) => ({ + type: 'NewVideo', + payload: { + ...e, + element: html, + href, + clientTime: new Date(), + }, + })); + }, + search: () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const fixture = require(`../../../yttrex/backend/__tests__/fixtures/htmls/search/0b1f85779539e6a0bd95304902cfd8117a69c82b.json`); + const { html, href } = fixture.sources[0]; + return ytContributionsArb.ContributionEventArb.map((e) => ({ + type: 'NewVideo', + payload: { + ...e, + html, + href, + clientTime: new Date(), + }, + })); + }, + video: () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const fixture = require(`../../../yttrex/backend/__tests__/fixtures/htmls/video/004fdccdb8bddf7a2588fcbf09916015ff898238.json`); + const { html, href } = fixture.sources[0]; + return ytContributionsArb.ContributionEventArb.map((e) => ({ + type: 'NewVideo', + payload: { + ...e, + html, + href, + clientTime: new Date(), + }, + })); + }, + }, + ...metadataLoggerParserProps, +}; + +export const TikTokBasic = Template.bind({}); +TikTokBasic.args = { + platform: 'tktrex', + hub: tkHub, + arbs: { + search: () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const fixture = require(`../../../tktrex/backend/__tests__/fixtures/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json`); + const { html, href, } = fixture.sources[0]; + return tkContributionsArb.ContributionEventArb.map((e) => ({ + type: 'search', + payload: { + ...e, + html, + href, + clientTime: new Date(), + }, + })); + }, + native: () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const fixture = require(`../../../tktrex/backend/__tests__/fixtures/native/1c144b8476d67668f19cc9e8d4f235890181d26a.json`); + const { html, href } = fixture.sources[0]; + return tkContributionsArb.ContributionEventArb.map((e) => ({ + type: 'native', + payload: { + ...e, + html, + href, + clientTime: new Date(), + }, + })); + }, + }, + ...tkMetadataLoggerParserProps, +}; diff --git a/platforms/storybook/tsconfig.json b/platforms/storybook/tsconfig.json index 76cca4abb..317515d3a 100644 --- a/platforms/storybook/tsconfig.json +++ b/platforms/storybook/tsconfig.json @@ -22,14 +22,20 @@ "moduleResolution": "node", "isolatedModules": true, "paths": { - "@shared/*": [ - "../../../packages/shared/src/*" - ] + "@shared/*": ["../../../packages/shared/src/*"], + "@yttrex/shared/*": ["../../yttrex/shared/src/*"], + "@tktrex/shared/*": ["../../tktrex/shared/src/*"] } }, "references": [ { "path": "../../packages/shared" + }, + { + "path": "../yttrex/shared" + }, + { + "path": "../tktrex/shared" } ], "include": ["./src"], diff --git a/platforms/tktrex/extension/src/app/index.ts b/platforms/tktrex/extension/src/app/index.ts index e8b85b3da..314c5008c 100644 --- a/platforms/tktrex/extension/src/app/index.ts +++ b/platforms/tktrex/extension/src/app/index.ts @@ -2,7 +2,9 @@ import { boot } from '@shared/extension/app'; import { tiktokDomainRegExp } from '@tktrex/parser/v2/constant'; import { registerTkHandlers } from './handlers'; import { feedId, onLocationChange, tkHandlers, tkTrexActions } from './app'; +import { metadataLoggerProps } from '@tktrex/parser/metadata-logger'; import tkHub from '../handlers/hub'; +import config from '@shared/extension/config'; // Boot the app script. This is the first function called. void boot({ @@ -13,8 +15,8 @@ void boot({ mapLocalConfig: (c, p) => ({ ...c, ...p }), hub: { hub: tkHub, - onRegister: (hub, config) => { - registerTkHandlers(hub, config); + onRegister: (hub, conf) => { + registerTkHandlers(hub, conf); }, }, observe: { @@ -23,4 +25,7 @@ void boot({ onLocationChange, }, onAuthenticated: tkTrexActions, + ui: config.DEVELOPMENT ? { + metadataLogger: metadataLoggerProps, + }: undefined, }); diff --git a/platforms/tktrex/shared/src/parser/metadata-logger.ts b/platforms/tktrex/shared/src/parser/metadata-logger.ts new file mode 100644 index 000000000..ce05cccf1 --- /dev/null +++ b/platforms/tktrex/shared/src/parser/metadata-logger.ts @@ -0,0 +1,61 @@ +import { MetadataLoggerProps } from '@shared/extension/ui/components/MetadataLogger'; +import { trexLogger } from '@shared/logger'; +import { ParserConfiguration } from '@shared/providers/parser.provider'; +import { v4 as uuid } from 'uuid'; +import { TKMetadata } from '../models/metadata/index'; +import { toMetadata } from './metadata'; +import { parsers } from './parsers'; +import { HTMLSource } from './source'; + +const metadataLogger = trexLogger.extend('tk:parser'); + +export const metadataLoggerProps: Omit< + MetadataLoggerProps, + 'hub' +> = { + parser: { + name: 'tk:parser', + log: metadataLogger, + parsers, + addDom: (h) => ({ + ...h, + jsdom: new DOMParser().parseFromString(h.html.html, 'text/html'), + }), + getEntryId: (e) => e.html.id, + getEntryDate: (e) => e.html.savingTime, + getEntryNatureType: (e) => e.html.nature?.type ?? e.html.type, + buildMetadata: toMetadata, + config: {}, + }, + decode: HTMLSource.decode, + mapEvent(id, e) { + metadataLogger.debug('Map event %O', e); + const ev: any = e; + + return { + html: { + ...ev.payload, + id, + metadataId: uuid(), + blang: '', + counters: [], + clientTime: new Date(), + savingTime: new Date(), + publicKey: '', + type: ev.type, + videoId: '', + authorId: '', + processed: false, + nature: { type: ev.type } + }, + supporter: { + publicKey: '', + version: '', + lastActivity: new Date(), + creationTime: new Date(), + p: '', + }, + jsdom: {}, + }; + }, +}; diff --git a/platforms/tktrex/shared/src/parser/parsers/nature.ts b/platforms/tktrex/shared/src/parser/parsers/nature.ts index 05dce2597..1f16e6eb3 100644 --- a/platforms/tktrex/shared/src/parser/parsers/nature.ts +++ b/platforms/tktrex/shared/src/parser/parsers/nature.ts @@ -4,15 +4,14 @@ import { Nature } from '../../models/Nature'; import { TKParserConfig } from '../config'; import { HTMLSource } from '../source'; import { Either, left, right } from 'fp-ts/lib/Either'; -import { URLError } from '../v2/models/Error'; const HOSTNAMES = ['www.tiktok.com', 'tiktok.com']; -export const getNatureByHref = (href: string): Either => { +export const getNatureByHref = (href: string): Either => { const url = new URL(href); const chunks = url.pathname.split('/'); if (!HOSTNAMES.includes(url.hostname)) { - return left(new URLError('URL is not from tiktok', url)); + return left(new Error(`URL is not from tiktok: ${url}`)); } if ( @@ -50,7 +49,7 @@ export const getNatureByHref = (href: string): Either => { }); } - return left(new URLError('unexpected condition from URL', url)); + return left(new Error(`Unexpected condition from URL: ${url}`)); }; const nature: ParserFn = async( diff --git a/platforms/yttrex/extension/src/app/index.ts b/platforms/yttrex/extension/src/app/index.ts index 453c6d5b0..97772b005 100644 --- a/platforms/yttrex/extension/src/app/index.ts +++ b/platforms/yttrex/extension/src/app/index.ts @@ -2,16 +2,18 @@ // with some values we can retrieve only from the `chrome` space. import { boot } from '@shared/extension/app'; +import config from '@shared/extension/config'; import { bo } from '@shared/extension/utils/browser.utils'; +import { metadataLoggerParserProps } from '@yttrex/shared/parser/metadata-logger'; import { youtubeDomainRegExp } from '@yttrex/shared/parser/selectors'; import * as hubHandlers from '../handlers/events'; import ytHub from '../handlers/hub'; import { onLocationChange, watchedPaths, ytLogger, ytTrexActions } from './app'; -bo.runtime.sendMessage({ type: 'chromeConfig' }, (config) => { - ytLogger.info('Booting app with config %O', config); +bo.runtime.sendMessage({ type: 'chromeConfig' }, (chromeConfig) => { + ytLogger.info('Booting app with config %O', chromeConfig); try { - const { ui, ...settings } = config; + const { ui, ...settings } = chromeConfig; void boot({ payload: { href: window.location.href, @@ -36,6 +38,11 @@ bo.runtime.sendMessage({ type: 'chromeConfig' }, (config) => { }, }, onAuthenticated: ytTrexActions, + ui: config.DEVELOPMENT + ? { + metadataLogger: metadataLoggerParserProps, + } + : undefined, }); } catch (e) { // eslint-disable-next-line diff --git a/platforms/yttrex/extension/tsconfig.json b/platforms/yttrex/extension/tsconfig.json index e1fb8c8d7..5e393a57b 100644 --- a/platforms/yttrex/extension/tsconfig.json +++ b/platforms/yttrex/extension/tsconfig.json @@ -29,7 +29,7 @@ "path": "../../../packages/shared" } ], - "include": ["./"], + "include": ["./src"], "exclude": [ "./jest.config.js", "./jest.setup.ts", diff --git a/platforms/yttrex/shared/src/parser/metadata-logger.ts b/platforms/yttrex/shared/src/parser/metadata-logger.ts new file mode 100644 index 000000000..23e5989b9 --- /dev/null +++ b/platforms/yttrex/shared/src/parser/metadata-logger.ts @@ -0,0 +1,66 @@ +import { MetadataLoggerProps } from '@shared/extension/ui/components/MetadataLogger'; +import { trexLogger } from '@shared/logger'; +import { ParserConfiguration } from '@shared/providers/parser.provider'; +import { v4 as uuid } from 'uuid'; +import { Metadata } from '../models/metadata/Metadata'; +import { toMetadata } from './metadata'; +import nature, { getNatureFromURL } from './parsers/nature'; +import search from './parsers/searches'; +import { HTMLSource } from './source'; + +const metadataParserLogger = trexLogger.extend('metadata-logger'); + +export const metadataLoggerParserProps: Omit< + MetadataLoggerProps, + 'hub' +> = { + parser: { + name: 'yt-metadata', + log: metadataParserLogger, + addDom: (h) => ({ + ...h, + jsdom: new DOMParser().parseFromString(h.html.html, 'text/html'), + }), + buildMetadata: toMetadata as any, + parsers: { + nature, + search, + }, + getEntryDate: (e) => e.html.clientTime, + getEntryNatureType: (e) => e.html.nature.type, + getEntryId: (e) => e.html.id, + config: {}, + }, + decode: HTMLSource.decode, + mapEvent: (id, e) => { + const ev: any = e; + if (ev.type === 'NewVideo') { + const { element, ...payload } = ev.payload; + return { + html: { + id, + metadataId: uuid(), + blang: '', + publicKey: '', + clientTime: new Date(), + savingTime: new Date(), + nature: getNatureFromURL(payload.href), + counters: [], + ...payload, + html: element, + processed: false, + }, + supporter: { + publicKey: '', + lastActivity: new Date(), + version: '', + creationTime: new Date(), + p: '', + }, + jsdom: {}, + }; + } + + return null; + }, +}; diff --git a/platforms/yttrex/shared/tsconfig.json b/platforms/yttrex/shared/tsconfig.json index 47978c905..a8eb64780 100644 --- a/platforms/yttrex/shared/tsconfig.json +++ b/platforms/yttrex/shared/tsconfig.json @@ -8,6 +8,7 @@ "baseUrl": "./src", "rootDir": "./src", "noEmit": false, + "jsx": "react", "esModuleInterop": true, "skipLibCheck": true, "isolatedModules": true, diff --git a/yarn.lock b/yarn.lock index 8a39c215f..93dfd4597 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6145,6 +6145,7 @@ __metadata: react: ^17.0.2 react-dom: ^17.0.2 react-i18next: ^11.18.0 + react-json-view: ^1.21.3 react-refresh: ^0.12.0 react-refresh-typescript: ^2.0.7 sass-loader: ^12.6.0