From 2f8fc0bb5b45a3e8986cf5ed2581cb350d34ff2c Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Sat, 7 Nov 2020 17:51:06 +0100 Subject: [PATCH 01/16] fix(scraper): fix tocs not updated toc specific values should only be saved in tocs. a new medium should only have the necessary values as its values should be filled by the user --- src/server/bin/jobHandler.ts | 81 ++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/src/server/bin/jobHandler.ts b/src/server/bin/jobHandler.ts index 98f30b51..c7c9f26b 100644 --- a/src/server/bin/jobHandler.ts +++ b/src/server/bin/jobHandler.ts @@ -101,6 +101,14 @@ async function feedHandler({link, result}: { link: string; result: News[] }): Pr } } +/** + * Map toc contents to their respective Media. + * Creates a new Medium if a corresponding one for a ToC does not exist. + * Adds Medium to the uuid + * + * @param tocs tocs to map + * @param uuid a user uuid + */ async function getTocMedia(tocs: Toc[], uuid?: Uuid) : Promise> { @@ -113,51 +121,48 @@ async function getTocMedia(tocs: Toc[], uuid?: Uuid) // @ts-ignore medium = await mediumStorage.getSimpleMedium(toc.mediumId); } else { - const likeMedium = await mediumStorage.getLikeMedium({title: toc.title, link: ""}); + // get likemedium with similar title and same media type + const likeMedium = await mediumStorage.getLikeMedium({title: toc.title, type: toc.mediumType, link: ""}); medium = likeMedium.medium; } + // if no such medium exists, create a new medium and toc if (!medium) { - const author = toc.authors ? toc.authors[0].name : undefined; - const artist = toc.artists ? toc.artists[0].name : undefined; - medium = await mediumStorage.addMedium({ - medium: toc.mediumType, - title: toc.title, - author, - artist, - stateOrigin: toc.statusCOO, - stateTL: toc.statusTl, - languageOfOrigin: toc.langCOO, - lang: toc.langTL, - }, uuid); - } else { - const author = toc.authors ? toc.authors[0].name : undefined; - const artist = toc.artists ? toc.artists[0].name : undefined; - await mediumStorage.updateMediumToc({ - id: 0, - mediumId: medium.id as number, - link: toc.link, - medium: toc.mediumType, - author, - artist, - stateOrigin: toc.statusCOO, - stateTL: toc.statusTl, - languageOfOrigin: toc.langCOO, - lang: toc.langTL, - }); + // create medium with minimal values + medium = await mediumStorage.addMedium( + { + medium: toc.mediumType, + title: toc.title + }, + uuid + ); + + await mediumStorage.addToc(medium.id as number, toc.link); } + // TODO: how to handle multiple authors, artists?, json array, csv, own table? + const author = toc.authors ? toc.authors[0].name : undefined; + const artist = toc.artists ? toc.artists[0].name : undefined; + // update toc specific values + await mediumStorage.updateMediumToc({ + id: 0, + title: toc.title, + mediumId: medium.id as number, + link: toc.link, + medium: toc.mediumType, + author, + artist, + stateOrigin: toc.statusCOO, + stateTL: toc.statusTl, + languageOfOrigin: toc.langCOO, + lang: toc.langTL, + }); + // ensure synonyms exist + // TODO: shouldnt these synonyms be toc specific? if (toc.synonyms) { await mediumStorage.addSynonyms({mediumId: medium.id as number, synonym: toc.synonyms}); } - if (medium.id && toc.link) { - await mediumStorage.addToc(medium.id, toc.link); - } else { - // TODO: 26.02.2020 what does this imply, should something be done? is this an error? - logger.warn(`missing toc and id for ${JSON.stringify(medium)} and ${JSON.stringify(toc)}`); - } - const mediumValue = getElseSet(media, medium, () => { return { episodes: [], @@ -165,6 +170,7 @@ async function getTocMedia(tocs: Toc[], uuid?: Uuid) }; }); + // map toc contents to their medium toc.content.forEach((content) => { if (!content || content.totalIndex == null) { throw Error(`invalid tocContent for mediumId:'${medium && medium.id}' and link:'${toc.link}'`); @@ -350,14 +356,15 @@ export async function tocHandler(result: { tocs: Toc[]; uuid?: Uuid }): Promise< } const tocs = result.tocs; const uuid = result.uuid; - logger.debug(`handling toc: ${tocs.map((value) => { + logger.debug(`handling tocs ${tocs.length}: ${tocs.map((value) => { return {...value, content: value.content.length}; })} ${uuid}`); - if (!(tocs && tocs.length)) { + if (!tocs || !tocs.length) { return; } + // map tocs contents to medium const media: Map = await getTocMedia(tocs, uuid); const promises: Array>>> = Array.from(media.entries()) From 439214fa809fbb15be5b480d91f201b9c244d8f5 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Sat, 7 Nov 2020 20:00:34 +0100 Subject: [PATCH 02/16] fix(scraper): fix property access on undefined --- src/server/bin/jobHandler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/bin/jobHandler.ts b/src/server/bin/jobHandler.ts index c7c9f26b..bd0b43e9 100644 --- a/src/server/bin/jobHandler.ts +++ b/src/server/bin/jobHandler.ts @@ -140,8 +140,8 @@ async function getTocMedia(tocs: Toc[], uuid?: Uuid) await mediumStorage.addToc(medium.id as number, toc.link); } // TODO: how to handle multiple authors, artists?, json array, csv, own table? - const author = toc.authors ? toc.authors[0].name : undefined; - const artist = toc.artists ? toc.artists[0].name : undefined; + const author = toc.authors?.length ? toc.authors[0].name : undefined; + const artist = toc.artists?.length ? toc.artists[0].name : undefined; // update toc specific values await mediumStorage.updateMediumToc({ id: 0, From 70faf6c852745a26b135c452c19e39f71ca28d04 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Sat, 7 Nov 2020 21:41:46 +0100 Subject: [PATCH 03/16] feat(scraper): scrape authors and artists when possible --- .../bin/externals/direct/boxNovelScraper.ts | 7 ++++-- .../bin/externals/direct/directTools.ts | 25 +++++++++++++++++++ .../bin/externals/direct/mangaHasuScraper.ts | 7 ++++-- .../bin/externals/direct/mangadexScraper.ts | 4 +++ .../bin/externals/direct/novelFullScraper.ts | 10 +++++--- 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/server/bin/externals/direct/boxNovelScraper.ts b/src/server/bin/externals/direct/boxNovelScraper.ts index 1e016c5b..dea9bb89 100644 --- a/src/server/bin/externals/direct/boxNovelScraper.ts +++ b/src/server/bin/externals/direct/boxNovelScraper.ts @@ -4,7 +4,7 @@ import {queueCheerioRequest, queueRequest} from "../queueManager"; import * as url from "url"; import {equalsIgnore, extractIndices, MediaType, relativeToAbsoluteTime, sanitizeString} from "../../tools"; import logger from "../../logger"; -import {getTextContent, SearchResult as TocSearchResult, searchToc} from "./directTools"; +import {getTextContent, SearchResult as TocSearchResult, searchToc, extractLinkable} from "./directTools"; import {checkTocContent} from "../scraperTools"; import {MissingResourceError, UrlError} from "../errors"; import {StatusCodeError} from "cloudscraper/errors"; @@ -262,13 +262,16 @@ async function tocAdapter(tocLink: string): Promise { end = false; releaseState = ReleaseState.Ongoing; } + const authors = extractLinkable($, ".author-content a", uri); + return [{ link: tocLink, content, title: mediumTitle, end, statusTl: releaseState, - mediumType: MediaType.TEXT + mediumType: MediaType.TEXT, + authors, }]; } diff --git a/src/server/bin/externals/direct/directTools.ts b/src/server/bin/externals/direct/directTools.ts index 076ea221..1c41639b 100644 --- a/src/server/bin/externals/direct/directTools.ts +++ b/src/server/bin/externals/direct/directTools.ts @@ -4,6 +4,7 @@ import {queueCheerioRequest} from "../queueManager"; import {combiIndex, equalsIgnore, extractIndices, MediaType, sanitizeString, stringify} from "../../tools"; import * as url from "url"; import {ReleaseState, TocSearchMedium} from "../../types"; +import cheerio from "cheerio"; export function getTextContent(novelTitle: string, episodeTitle: string, urlString: string, content: string): EpisodeContent[] { if (!novelTitle || !episodeTitle) { @@ -33,6 +34,30 @@ export function getTextContent(novelTitle: string, episodeTitle: string, urlStri return [episodeContent]; } +/** + * Extracts the Links described by the css selector into + * each an Object of link text and href. + * + * @param $ the cheerio root of the document + * @param selector a valid css selector + * @param uri a valid base url + */ +export function extractLinkable($: cheerio.Root, selector: string, uri: string): + Array<{ name: string; link: string }> { + const elements = $(selector); + const result = []; + + for (let i = 0; i < elements.length; i++) { + const element = elements.eq(i); + + const name = sanitizeString(element.text()); + const link = url.resolve(uri, element.attr("href") as string); + + result.push({ name, link }); + } + return result; +} + export async function searchTocCheerio(medium: TocSearchMedium, tocScraper: TocScraper, uri: string, searchLink: (parameter: string) => string, linkSelector: string): Promise { logger.info(`searching for ${medium.title} on ${uri}`); diff --git a/src/server/bin/externals/direct/mangaHasuScraper.ts b/src/server/bin/externals/direct/mangaHasuScraper.ts index 854a5891..88b4fbbc 100644 --- a/src/server/bin/externals/direct/mangaHasuScraper.ts +++ b/src/server/bin/externals/direct/mangaHasuScraper.ts @@ -5,7 +5,7 @@ import {queueCheerioRequest} from "../queueManager"; import logger from "../../logger"; import {equalsIgnore, extractIndices, MediaType, sanitizeString} from "../../tools"; import {checkTocContent} from "../scraperTools"; -import {SearchResult as TocSearchResult, searchToc} from "./directTools"; +import {SearchResult as TocSearchResult, searchToc, extractLinkable} from "./directTools"; import {MissingResourceError, UrlError} from "../errors"; async function scrapeNews(): Promise<{ news?: News[]; episodes?: EpisodeNews[] } | undefined> { @@ -180,7 +180,7 @@ async function scrapeToc(urlString: string): Promise { const indexPartMap: Map = new Map(); const chapterContents: TocEpisode[] = []; let releaseState: ReleaseState = ReleaseState.Unknown; - const releaseStateElement = $("div.col-md-12:nth-child(5) > span:nth-child(2) > a:nth-child(1)"); + const releaseStateElement = $("a[href^=\"/advanced-search.html?status=\"]"); const releaseStateString = releaseStateElement.text().toLowerCase(); if (releaseStateString.includes("complete")) { @@ -196,6 +196,9 @@ async function scrapeToc(urlString: string): Promise { mediumType: MediaType.IMAGE }; + toc.authors = extractLinkable($, "a[href^=\"/advanced-search.html?author=\"]", uri); + toc.artists = extractLinkable($, "a[href^=\"/advanced-search.html?artist=\"]", uri); + const endReg = /\[END]\s*$/i; const volChapReg = /Vol\.?\s*((\d+)(\.(\d+))?)\s*Chapter\s*((\d+)(\.(\d+))?)(:\s*(.+))?/i; const chapReg = /Chapter\s*((\d+)(\.(\d+))?)(:\s*(.+))?/i; diff --git a/src/server/bin/externals/direct/mangadexScraper.ts b/src/server/bin/externals/direct/mangadexScraper.ts index 50cc7ee9..0917f4dc 100644 --- a/src/server/bin/externals/direct/mangadexScraper.ts +++ b/src/server/bin/externals/direct/mangadexScraper.ts @@ -8,6 +8,7 @@ import * as request from "request"; import {checkTocContent} from "../scraperTools"; import {episodeStorage} from "../../database/storages/storage"; import {MissingResourceError, UrlError} from "../errors"; +import { extractLinkable } from './directTools'; const jar = request.jar(); jar.setCookie( @@ -315,6 +316,9 @@ async function scrapeTocPage(toc: Toc, endReg: RegExp, volChapReg: RegExp, chapR toc.statusTl = releaseState; const ignoreTitles = /(oneshot)|(special.+chapter)/i; + toc.authors = extractLinkable($, "a[href^=\"/search?author\"]", uri); + toc.artists = extractLinkable($, "a[href^=\"/search?artist\"]", uri); + const chapters = contentElement.find(".chapter-container .chapter-row"); if (!chapters.length) { diff --git a/src/server/bin/externals/direct/novelFullScraper.ts b/src/server/bin/externals/direct/novelFullScraper.ts index e4d1959a..623c451d 100644 --- a/src/server/bin/externals/direct/novelFullScraper.ts +++ b/src/server/bin/externals/direct/novelFullScraper.ts @@ -4,7 +4,7 @@ import {queueCheerioRequest} from "../queueManager"; import * as url from "url"; import {extractIndices, isTocEpisode, isTocPart, MediaType, relativeToAbsoluteTime, sanitizeString} from "../../tools"; import logger from "../../logger"; -import {EpisodePiece, getTextContent, scrapeToc, searchTocCheerio, TocMetaPiece, TocPiece} from "./directTools"; +import {EpisodePiece, getTextContent, scrapeToc, searchTocCheerio, TocMetaPiece, TocPiece, extractLinkable} from "./directTools"; import {checkTocContent} from "../scraperTools"; import {UrlError} from "../errors"; @@ -70,7 +70,7 @@ async function contentDownloadAdapter(urlString: string): Promise div:nth-child(4) > a:nth-child(2)"); + const releaseStateElement = $("a[href^=\"/status/\"]"); const releaseStateString = releaseStateElement.text().toLowerCase(); if (releaseStateString.includes("complete")) { @@ -82,13 +82,17 @@ function extractTocSnippet($: cheerio.Root, link: string): Toc { } const mediumTitleElement = $(".desc .title").first(); const mediumTitle = sanitizeString(mediumTitleElement.text()); + + const authors = extractLinkable($, "a[href^=\"/author/\"]", "https://novelfull.com/"); + return { content: [], mediumType: MediaType.TEXT, end, statusTl: releaseState, title: mediumTitle, - link + link, + authors }; } From 63f67cbeec9e05b402718464733add4b923030cb Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 11:04:40 +0100 Subject: [PATCH 04/16] refactor(database): use Error enum instead of number codes using the enum allows to better understand the reason of the error code --- src/server/bin/database/contexts/episodeContext.ts | 2 +- src/server/bin/database/contexts/partContext.ts | 3 ++- src/server/bin/database/storages/storage.ts | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/server/bin/database/contexts/episodeContext.ts b/src/server/bin/database/contexts/episodeContext.ts index 57e79c7c..243262cd 100644 --- a/src/server/bin/database/contexts/episodeContext.ts +++ b/src/server/bin/database/contexts/episodeContext.ts @@ -647,7 +647,7 @@ export class EpisodeContext extends SubContext { } catch (e) { // do not catch if it isn't an duplicate key error // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - if (!e || (e.errno !== 1062 && e.errno !== 1022)) { + if (!e || (e.errno !== MysqlServerError.ER_DUP_KEY && e.errno !== MysqlServerError.ER_DUP_ENTRY)) { throw e; } const result = await this.query( diff --git a/src/server/bin/database/contexts/partContext.ts b/src/server/bin/database/contexts/partContext.ts index 752f36d6..a1b8c172 100644 --- a/src/server/bin/database/contexts/partContext.ts +++ b/src/server/bin/database/contexts/partContext.ts @@ -3,6 +3,7 @@ import {Episode, FullPart, MinPart, MultiSingle, Part, ShallowPart, SimpleEpisod import mySql from "promise-mysql"; import {combiIndex, getElseSetObj, multiSingle, separateIndex} from "../../tools"; import {Query} from "mysql"; +import { MysqlServerError } from "../mysqlError"; export class PartContext extends SubContext { public async getAll(): Promise { @@ -276,7 +277,7 @@ export class PartContext extends SubContext { partId = result.insertId; } catch (e) { // do not catch if it isn't an duplicate key error - if (!e || (e.errno !== 1062 && e.errno !== 1022)) { + if (!e || (e.errno !== MysqlServerError.ER_DUP_KEY && e.errno !== MysqlServerError.ER_DUP_ENTRY)) { throw e; } const result = await this.query( diff --git a/src/server/bin/database/storages/storage.ts b/src/server/bin/database/storages/storage.ts index 60054411..e18b622e 100644 --- a/src/server/bin/database/storages/storage.ts +++ b/src/server/bin/database/storages/storage.ts @@ -20,6 +20,7 @@ import {MediumInWaitStorage} from "./mediumInWaitStorage"; import {NewsStorage} from "./newsStorage"; import {EpisodeStorage} from "./episodeStorage"; import {PartStorage} from "./partStorage"; +import { MysqlServerError } from "../mysqlError"; function inContext(callback: ContextCallback, transaction = true) { return storageInContext(callback, (con) => queryContextProvider(con), transaction); @@ -82,7 +83,7 @@ async function catchTransactionError( await context.rollback(); } // if it is an deadlock or lock wait timeout error, restart transaction after a delay at max five times - if ((e.errno === 1213 || e.errno === 1205) && attempts < 5) { + if ((e.errno === MysqlServerError.ER_LOCK_DEADLOCK || e.errno === MysqlServerError.ER_LOCK_WAIT_TIMEOUT) && attempts < 5) { await delay(500); return doTransaction(callback, context, transaction, attempts + 1); } else { From ee0815977279114318ce99443d93776f46f87885 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 11:05:55 +0100 Subject: [PATCH 05/16] fix(scraper): retry request on internal server error some internal server error occur randomly, retrying yields most times a valid response again --- .../bin/externals/direct/mangaHasuScraper.ts | 30 +++++++++++++++---- src/server/bin/externals/errors.ts | 10 +++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/server/bin/externals/direct/mangaHasuScraper.ts b/src/server/bin/externals/direct/mangaHasuScraper.ts index 88b4fbbc..08ec298b 100644 --- a/src/server/bin/externals/direct/mangaHasuScraper.ts +++ b/src/server/bin/externals/direct/mangaHasuScraper.ts @@ -3,15 +3,35 @@ import {EpisodeNews, News, ReleaseState, SearchResult, TocSearchMedium} from ".. import * as url from "url"; import {queueCheerioRequest} from "../queueManager"; import logger from "../../logger"; -import {equalsIgnore, extractIndices, MediaType, sanitizeString} from "../../tools"; +import {equalsIgnore, extractIndices, MediaType, sanitizeString, delay} from "../../tools"; import {checkTocContent} from "../scraperTools"; import {SearchResult as TocSearchResult, searchToc, extractLinkable} from "./directTools"; -import {MissingResourceError, UrlError} from "../errors"; +import {MissingResourceError, UrlError, UnreachableError} from "../errors"; + +async function tryRequest(link: string, retry = 0): Promise { + try { + return await queueCheerioRequest(link); + } catch (error) { + // mangahasu likes to throw an Internal Server Error every now and then + if (error.statusCode === 500) { + // try at most 3 times + if (retry < 3) { + // wait a bit before trying again + await delay(500); + return tryRequest(link, retry + 1); + } else { + throw new UnreachableError(link); + } + } else { + throw error; + } + } +} async function scrapeNews(): Promise<{ news?: News[]; episodes?: EpisodeNews[] } | undefined> { // TODO scrape more than just the first page if there is an open end const baseUri = "http://mangahasu.se/"; - const $ = await queueCheerioRequest(baseUri + "latest-releases.html"); + const $ = await tryRequest(baseUri + "latest-releases.html"); const newsRows = $("ul.list_manga .info-manga"); const news: EpisodeNews[] = []; @@ -107,7 +127,7 @@ async function scrapeNews(): Promise<{ news?: News[]; episodes?: EpisodeNews[] } } async function contentDownloadAdapter(chapterLink: string): Promise { - const $ = await queueCheerioRequest(chapterLink); + const $ = await tryRequest(chapterLink); if ($("head > title").text() === "Page not found!") { throw new MissingResourceError("Missing Toc on NovelFull", chapterLink); } @@ -156,7 +176,7 @@ async function scrapeToc(urlString: string): Promise { if (!/http:\/\/mangahasu\.se\/[^/]+\.html/.test(urlString)) { throw new UrlError("not a toc link for MangaHasu: " + urlString, urlString); } - const $ = await queueCheerioRequest(urlString); + const $ = await tryRequest(urlString); if ($("head > title").text() === "Page not found!") { throw new MissingResourceError("Missing Toc on NovelFull", urlString); } diff --git a/src/server/bin/externals/errors.ts b/src/server/bin/externals/errors.ts index 89a0699f..80716f6e 100644 --- a/src/server/bin/externals/errors.ts +++ b/src/server/bin/externals/errors.ts @@ -4,6 +4,16 @@ export class ScraperError extends Error { } } +export class UnreachableError extends Error { + public readonly name = "UnreachableError"; + public readonly url: string; + + public constructor(link: string) { + super(); + this.url = link; + } +} + // tslint:disable-next-line:max-classes-per-file export class UrlError extends Error { public readonly name = "UrlError"; From 203fa4f78b8f5542385474c9d0e6b64bc0a54603 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 11:06:29 +0100 Subject: [PATCH 06/16] refactor(scraper): add more descriptive error message --- src/server/bin/externals/scraperTools.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/bin/externals/scraperTools.ts b/src/server/bin/externals/scraperTools.ts index 9cc158ad..346c47f5 100644 --- a/src/server/bin/externals/scraperTools.ts +++ b/src/server/bin/externals/scraperTools.ts @@ -651,18 +651,18 @@ export function checkTocContent(content: TocContent, allowMinusOne = false): voi const index = content.combiIndex; if (index == null || (index < 0 && (index !== -1 || !allowMinusOne))) { - throw Error("invalid toc content, combiIndex invalid"); + throw Error("invalid toc content, combiIndex invalid: '" + index + "'"); } const totalIndex = content.totalIndex; if (totalIndex == null || !Number.isInteger(totalIndex) || (totalIndex < 0 && (totalIndex !== -1 || !allowMinusOne))) { - throw Error("invalid toc content, totalIndex invalid"); + throw Error(`invalid toc content, totalIndex invalid: '${totalIndex}' of ${index}`); } const partialIndex = content.partialIndex; if (partialIndex != null && (partialIndex < 0 || !Number.isInteger(partialIndex))) { - throw Error("invalid toc content, partialIndex invalid"); + throw Error(`invalid toc content, partialIndex invalid: '${partialIndex}' of ${index}`); } } From 150d141dbd42e9f1d0893197e079a50e9edcf711 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 11:08:11 +0100 Subject: [PATCH 07/16] fix(scraper): ignore some errors or unimportant informations which would fail the whole job --- .../bin/externals/direct/directTools.ts | 22 ++++++++++++++++--- .../bin/externals/direct/mangadexScraper.ts | 6 +---- .../bin/externals/direct/webnovelScraper.ts | 9 +++++++- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/server/bin/externals/direct/directTools.ts b/src/server/bin/externals/direct/directTools.ts index 1c41639b..7fc2c8c7 100644 --- a/src/server/bin/externals/direct/directTools.ts +++ b/src/server/bin/externals/direct/directTools.ts @@ -4,7 +4,7 @@ import {queueCheerioRequest} from "../queueManager"; import {combiIndex, equalsIgnore, extractIndices, MediaType, sanitizeString, stringify} from "../../tools"; import * as url from "url"; import {ReleaseState, TocSearchMedium} from "../../types"; -import cheerio from "cheerio"; +import { checkTocContent } from "../scraperTools"; export function getTextContent(novelTitle: string, episodeTitle: string, urlString: string, content: string): EpisodeContent[] { if (!novelTitle || !episodeTitle) { @@ -829,6 +829,7 @@ function adjustTocContentsLinked(contents: TocLinkedList, state: TocScrapeState) lastVolumeLastEpisode = lastVolume && lastVolume.episodes[lastVolume.episodes.length - 1]; volume = node; currentVolumeChecked = false; + checkTocContent(node); } else if (isInternalEpisode(node)) { if (node.partCount) { adjustPartialIndicesLinked(node, ascending, contents); @@ -887,6 +888,7 @@ function adjustTocContentsLinked(contents: TocLinkedList, state: TocScrapeState) } } } + checkTocContent(node); } } } @@ -1072,7 +1074,7 @@ function mark(tocPiece: TocContentPiece, state: TocScrapeState): Node[] { possibleVolume = state.volumeMap.get(volIndices.combi); if (!possibleVolume) { - possibleVolume = { + const internalTocPart = { type: "part", combiIndex: volIndices.combi, totalIndex: volIndices.total, @@ -1081,6 +1083,13 @@ function mark(tocPiece: TocContentPiece, state: TocScrapeState): Node[] { originalTitle: "", episodes: [] } as InternalTocPart; + // need to be valid to be acknowledged + try { + checkTocContent(internalTocPart); + } catch (error) { + continue + } + possibleVolume = internalTocPart; newVolume = true; state.volumeMap.set(volIndices.combi, possibleVolume); } @@ -1124,7 +1133,7 @@ function mark(tocPiece: TocContentPiece, state: TocScrapeState): Node[] { possibleVolume = state.volumeMap.get(volIndices.combi); if (!possibleVolume) { - possibleVolume = { + const internalTocPart = { type: "part", combiIndex: volIndices.combi, totalIndex: volIndices.total, @@ -1133,6 +1142,13 @@ function mark(tocPiece: TocContentPiece, state: TocScrapeState): Node[] { originalTitle: "", episodes: [] } as InternalTocPart; + // need to be valid to be acknowledged + try { + checkTocContent(internalTocPart); + } catch (error) { + continue + } + possibleVolume = internalTocPart; newVolume = true; state.volumeMap.set(volIndices.combi, possibleVolume); } diff --git a/src/server/bin/externals/direct/mangadexScraper.ts b/src/server/bin/externals/direct/mangadexScraper.ts index 0917f4dc..ac76d09c 100644 --- a/src/server/bin/externals/direct/mangadexScraper.ts +++ b/src/server/bin/externals/direct/mangadexScraper.ts @@ -314,7 +314,6 @@ async function scrapeTocPage(toc: Toc, endReg: RegExp, volChapReg: RegExp, chapR } } toc.statusTl = releaseState; - const ignoreTitles = /(oneshot)|(special.+chapter)/i; toc.authors = extractLinkable($, "a[href^=\"/search?author\"]", uri); toc.artists = extractLinkable($, "a[href^=\"/search?artist\"]", uri); @@ -415,11 +414,8 @@ async function scrapeTocPage(toc: Toc, endReg: RegExp, volChapReg: RegExp, chapR checkTocContent(chapterContent); toc.content.push(chapterContent); } else { - if (chapterTitle.match(ignoreTitles)) { - continue; - } logger.warn(`volume - chapter format changed on mangadex: recognized neither of them: '${chapterTitle}', ${urlString}`); - return true; + continue; } } const nextPaging = $(".page-item:last-child:not(.disabled)"); diff --git a/src/server/bin/externals/direct/webnovelScraper.ts b/src/server/bin/externals/direct/webnovelScraper.ts index 671cb35c..a317035d 100644 --- a/src/server/bin/externals/direct/webnovelScraper.ts +++ b/src/server/bin/externals/direct/webnovelScraper.ts @@ -123,7 +123,14 @@ async function scrapeTocPage(bookId: string, mediumId?: number): Promise } const idPattern = /^\d+$/; - const content: TocPart[] = tocJson.data.volumeItems.map((volume: any): TocPart => { + const content: TocPart[] = tocJson.data.volumeItems + // one medium has a volume with negative indices only, this should not be a valid episode + // a volume which consists of only episodes with negative indices should be filtered out + .filter((volume) => { + volume.chapterItems = volume.chapterItems.filter(episode => episode.index >= 0); + return volume.chapterItems.length; + }) + .map((volume: any): TocPart => { if (!volume.name) { volume.name = "Volume " + volume.index; } From 5e2fc8507764af2ac62b749d96220c200a153a6b Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 11:28:06 +0100 Subject: [PATCH 08/16] fix(scraper): retry when part of the response is missing --- .../bin/externals/direct/webnovelScraper.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/server/bin/externals/direct/webnovelScraper.ts b/src/server/bin/externals/direct/webnovelScraper.ts index a317035d..8e5ddc06 100644 --- a/src/server/bin/externals/direct/webnovelScraper.ts +++ b/src/server/bin/externals/direct/webnovelScraper.ts @@ -188,10 +188,21 @@ function loadBody(urlString: string): Promise { return initPromise.then(() => queueCheerioRequest(urlString, undefined, defaultRequest)); } -function loadJson(urlString: string): Promise { +function loadJson(urlString: string, retry = 0): Promise { return initPromise .then(() => queueRequest(urlString, undefined, defaultRequest)) - .then((body) => JSON.parse(body)); + .then((body) => { + try { + return JSON.parse(body) + } catch (error) { + // sometimes the response body is incomplete for whatever reason + // so retry once to get it right, else forget it + if (retry >= 2) { + throw error; + } + return loadJson(urlString, retry + 1); + } + }); } async function scrapeContent(urlString: string): Promise { From 0e8d5c9d11396af2b7ec759ec349cafaf12b9a71 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 11:29:21 +0100 Subject: [PATCH 09/16] refactor: fix indent --- .../bin/externals/direct/webnovelScraper.ts | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/server/bin/externals/direct/webnovelScraper.ts b/src/server/bin/externals/direct/webnovelScraper.ts index 8e5ddc06..00f25a53 100644 --- a/src/server/bin/externals/direct/webnovelScraper.ts +++ b/src/server/bin/externals/direct/webnovelScraper.ts @@ -131,46 +131,46 @@ async function scrapeTocPage(bookId: string, mediumId?: number): Promise return volume.chapterItems.length; }) .map((volume: any): TocPart => { - if (!volume.name) { - volume.name = "Volume " + volume.index; - } - const name = volume.name; + if (!volume.name) { + volume.name = "Volume " + volume.index; + } + const name = volume.name; - const chapters: TocEpisode[] = volume.chapterItems.map((item: ChapterItem): TocEpisode => { - let date = new Date(item.createTime); + const chapters: TocEpisode[] = volume.chapterItems.map((item: ChapterItem): TocEpisode => { + let date = new Date(item.createTime); - if (Number.isNaN(date.getDate())) { - date = relativeToAbsoluteTime(item.createTime) || new Date(); - } + if (Number.isNaN(date.getDate())) { + date = relativeToAbsoluteTime(item.createTime) || new Date(); + } - if (!date) { - throw Error(`invalid date: '${item.createTime}'`); - } + if (!date) { + throw Error(`invalid date: '${item.createTime}'`); + } - if (!idPattern.test(item.id)) { - throw Error("invalid chapterId: " + item.id); - } + if (!idPattern.test(item.id)) { + throw Error("invalid chapterId: " + item.id); + } - const chapterContent: TocEpisode = { - url: `https://www.webnovel.com/book/${bookId}/${item.id}/`, - title: item.name, - combiIndex: item.index, - totalIndex: item.index, - releaseDate: date, - locked: item.isVip !== 0 + const chapterContent: TocEpisode = { + url: `https://www.webnovel.com/book/${bookId}/${item.id}/`, + title: item.name, + combiIndex: item.index, + totalIndex: item.index, + releaseDate: date, + locked: item.isVip !== 0 + }; + checkTocContent(chapterContent); + return chapterContent; + }); + const partContent = { + episodes: chapters, + title: name, + combiIndex: volume.index, + totalIndex: volume.index, }; - checkTocContent(chapterContent); - return chapterContent; + checkTocContent(partContent, true); + return partContent; }); - const partContent = { - episodes: chapters, - title: name, - combiIndex: volume.index, - totalIndex: volume.index, - }; - checkTocContent(partContent, true); - return partContent; - }); const toc: Toc = { link: `https://www.webnovel.com/book/${bookId}/`, synonyms: [tocJson.data.bookInfo.bookSubName], From b74a9f84f527b9ee3b5547366fbb3acf7820c52f Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 14:22:11 +0100 Subject: [PATCH 10/16] refactor(npm): fix script path --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a9e500d8..5054e0b3 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "start:server": "node dist/server/startServer.js", "start:crawler": "node dist/server/startCrawler.js", "test": " LOG_LEVEL=error node ./node_modules/mocha/bin/mocha --ui bdd ./dist/**/test/database/**/*.js ./dist/**/test/*.js", - "migrate": "node dist/server/test/migrate.js", + "migrate": "node dist/server/misc/migrate.js", "commit": "npx git-cz" }, "husky": { From 7c86b05d5d134ca7e10595d9ab560f3e51677e43 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 14:26:47 +0100 Subject: [PATCH 11/16] feat(scraper): store jobs result state, message and context in database the message consists currently of a json of modifications done to the database --- .../bin/database/contexts/episodeContext.ts | 77 ++++++++++++------- .../database/contexts/externalListContext.ts | 5 +- .../database/contexts/externalUserContext.ts | 10 ++- .../database/contexts/internalListContext.ts | 3 + .../bin/database/contexts/jobContext.ts | 27 +++++-- .../bin/database/contexts/mediumContext.ts | 71 +++++++++++++---- .../database/contexts/mediumInWaitContext.ts | 1 + .../bin/database/contexts/newsContext.ts | 4 + .../bin/database/contexts/partContext.ts | 3 + src/server/bin/database/databaseSchema.ts | 9 ++- src/server/bin/database/migrations.ts | 32 ++++++++ src/server/bin/database/sqlTools.ts | 35 +++++++++ src/server/bin/externals/jobScraperManager.ts | 2 +- src/server/bin/externals/scraperTools.ts | 11 +-- src/server/bin/jobHandler.ts | 44 ++++++++--- src/server/bin/jobManager.ts | 6 +- 16 files changed, 270 insertions(+), 70 deletions(-) create mode 100644 src/server/bin/database/sqlTools.ts diff --git a/src/server/bin/database/contexts/episodeContext.ts b/src/server/bin/database/contexts/episodeContext.ts index 243262cd..52f777ae 100644 --- a/src/server/bin/database/contexts/episodeContext.ts +++ b/src/server/bin/database/contexts/episodeContext.ts @@ -30,6 +30,7 @@ import logger from "../../logger"; import { MysqlServerError } from "../mysqlError"; import { escapeLike } from "../storages/storageTools"; import { Query } from "mysql"; +import { storeModifications } from "../sqlTools"; export class EpisodeContext extends SubContext { public async getAll(uuid: Uuid): Promise { @@ -305,7 +306,7 @@ export class EpisodeContext extends SubContext { const teaserMatcher = /\(?teaser\)?$|(\s+$)/i; // @ts-ignore - return promiseMultiSingle(result.result, async (value: MetaResult) => { + return promiseMultiSingle(result.result, async (value: MetaResult): void => { // TODO what if it is not a serial medium but only an article? should it even save such things? if (!value.novel || (!value.chapIndex && !value.chapter) @@ -320,10 +321,12 @@ export class EpisodeContext extends SubContext { ); // if a similar/same result was mapped to an episode before, get episode_id and update read if (resultArray[0] && resultArray[0].episode_id != null) { - return this.query( + const queryResult = await this.query( "INSERT IGNORE INTO user_episode (user_uuid, episode_id,progress) VALUES (?,?,0);", [uuid, resultArray[0].episode_id] ); + storeModifications("progress", "insert", queryResult); + return; } const escapedNovel = escapeLike(value.novel, { singleQuotes: true, noBoundaries: true }); @@ -463,16 +466,18 @@ export class EpisodeContext extends SubContext { // mark the episode as read // normally the progress should be updated by messages of the tracker // it should be inserted only, if there does not exist any progress - await this + let queryResult = await this .query( "INSERT IGNORE INTO user_episode (user_uuid, episode_id, progress) VALUES (?,?,0);", [uuid, episodeId] ); - await this.query( + storeModifications("progress", "insert", queryResult); + queryResult = await this.query( "INSERT INTO result_episode (novel, chapter, chapIndex, volume, volIndex, episode_id) " + "VALUES (?,?,?,?,?,?);", [value.novel, value.chapter, value.chapIndex, value.volume, value.volIndex, episodeId] ); + storeModifications("result_episode", "insert", queryResult); }).then(ignore); } @@ -499,6 +504,7 @@ export class EpisodeContext extends SubContext { release.locked ]; }); + // TODO: storeModifications("progress", "insert", result); return releases; } @@ -568,10 +574,11 @@ export class EpisodeContext extends SubContext { } ); } else if (value.sourceType) { - await this.query( + const result = await this.query( "UPDATE episode_release SET url=? WHERE source_type=? AND url != ? AND title=?", [value.url, value.sourceType, value.url, value.title] ); + storeModifications("release", "update", result); } }).then(ignore); } @@ -631,7 +638,6 @@ export class EpisodeContext extends SubContext { let insertId: number | undefined; const episodeCombiIndex = episode.combiIndex == null ? combiIndex(episode) : episode.combiIndex; try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const result: any = await this.query( "INSERT INTO episode " + "(part_id, totalIndex, partialIndex, combiIndex) " + @@ -643,6 +649,7 @@ export class EpisodeContext extends SubContext { episodeCombiIndex ] ); + storeModifications("episode", "insert", result); insertId = result.insertId; } catch (e) { // do not catch if it isn't an duplicate key error @@ -890,11 +897,12 @@ export class EpisodeContext extends SubContext { ); const changePartIds: number[] = changePartIdsResult.map((value) => value.id); - await this.queryInList( + const result = await this.queryInList( `UPDATE episode SET part_id=${mySql.escape(newPartId)} ` + `WHERE part_id=${mySql.escape(oldPartId)} AND combiIndex`, changePartIds ); + // TODO: storeModifications("release", "update", result); if (!replaceIds.length) { return true; } @@ -903,13 +911,15 @@ export class EpisodeContext extends SubContext { return this.query( "UPDATE episode_release set episode_id=? where episode_id=?", [replaceId.newId, replaceId.oldId] - ).catch((reason) => { - if (reason && MysqlServerError.ER_DUP_ENTRY === reason.errno) { - deleteReleaseIds.push(replaceId.oldId); - } else { - throw reason; - } - }); + ) + .then(value => storeModifications("release", "update", value)) + .catch((reason) => { + if (reason && MysqlServerError.ER_DUP_ENTRY === reason.errno) { + deleteReleaseIds.push(replaceId.oldId); + } else { + throw reason; + } + }); })); const deleteProgressIds: number[] = []; @@ -917,13 +927,15 @@ export class EpisodeContext extends SubContext { return this.query( "UPDATE user_episode set episode_id=? where episode_id=?", [replaceId.newId, replaceId.oldId] - ).catch((reason) => { - if (reason && MysqlServerError.ER_DUP_ENTRY === reason.errno) { - deleteProgressIds.push(replaceId.oldId); - } else { - throw reason; - } - }); + ) + .then(value => storeModifications("progress", "update", value)) + .catch((reason) => { + if (reason && MysqlServerError.ER_DUP_ENTRY === reason.errno) { + deleteProgressIds.push(replaceId.oldId); + } else { + throw reason; + } + }); })); const deleteResultIds: number[] = []; @@ -931,23 +943,29 @@ export class EpisodeContext extends SubContext { return this.query( "UPDATE result_episode set episode_id=? where episode_id=?", [replaceId.newId, replaceId.oldId] - ).catch((reason) => { - if (reason && MysqlServerError.ER_DUP_ENTRY === reason.errno) { - deleteResultIds.push(replaceId.oldId); - } else { - throw reason; - } - }); + ) + .then(value => storeModifications("result_episode", "update", value)) + .catch((reason) => { + if (reason && MysqlServerError.ER_DUP_ENTRY === reason.errno) { + deleteResultIds.push(replaceId.oldId); + } else { + throw reason; + } + }); })); const oldIds = replaceIds.map((value) => value.oldId); // TODO: 26.08.2019 this does not go quite well, throws error with 'cannot delete parent reference' await this.queryInList("DELETE FROM episode_release WHERE episode_id ", deleteReleaseIds); + // TODO: storeModifications("release", "update", result); await this.queryInList("DELETE FROM user_episode WHERE episode_id ", deleteProgressIds); + // TODO: storeModifications("release", "update", result); await this.queryInList("DELETE FROM result_episode WHERE episode_id ", deleteResultIds); + // TODO: storeModifications("release", "update", result); await this.queryInList( `DELETE FROM episode WHERE part_id=${mySql.escape(oldPartId)} AND id`, oldIds, ); + // TODO: storeModifications("release", "update", result); return true; } @@ -1011,7 +1029,7 @@ export class EpisodeContext extends SubContext { } // TODO: 09.03.2020 rework query and input, for now the episodeIndices are only relative to their parts mostly, // not always relative to the medium - await this.query( + const result = await this.query( "UPDATE user_episode, episode, part " + "SET user_episode.progress=1 " + "WHERE user_episode.episode_id=episode.id" + @@ -1022,5 +1040,6 @@ export class EpisodeContext extends SubContext { "AND episode.combiIndex < ?", [uuid, id, partInd, episodeInd] ); + storeModifications("progress", "update", result); } } diff --git a/src/server/bin/database/contexts/externalListContext.ts b/src/server/bin/database/contexts/externalListContext.ts index 5250eedb..b733c23b 100644 --- a/src/server/bin/database/contexts/externalListContext.ts +++ b/src/server/bin/database/contexts/externalListContext.ts @@ -1,6 +1,7 @@ import {SubContext} from "./subContext"; import {ExternalList, Uuid} from "../../types"; import {Errors, promiseMultiSingle} from "../../tools"; +import { storeModifications } from "../sqlTools"; export class ExternalListContext extends SubContext { public async getAll(uuid: Uuid): Promise { @@ -30,7 +31,7 @@ export class ExternalListContext extends SubContext { "VALUES(?,?,?,?);", [externalList.name, userUuid, externalList.medium, externalList.url], ); - + storeModifications("external_list", "insert", result); const insertId = result.insertId; if (!Number.isInteger(insertId)) { @@ -134,6 +135,7 @@ export class ExternalListContext extends SubContext { "VALUES (?,?)", [listId, mediumId], ); + storeModifications("external_list_item", "insert", result); return result.affectedRows > 0; } @@ -162,6 +164,7 @@ export class ExternalListContext extends SubContext { medium.id, (value) => [medium.listId, value] ); + storeModifications("external_list_item", "insert", result); return result.affectedRows > 0; } diff --git a/src/server/bin/database/contexts/externalUserContext.ts b/src/server/bin/database/contexts/externalUserContext.ts index e5c114aa..813b9fea 100644 --- a/src/server/bin/database/contexts/externalUserContext.ts +++ b/src/server/bin/database/contexts/externalUserContext.ts @@ -3,6 +3,7 @@ import {ExternalUser, Uuid} from "../../types"; import {Errors, promiseMultiSingle} from "../../tools"; import {v1 as uuidGenerator} from "uuid"; import {Query} from "mysql"; +import { storeModifications } from "../sqlTools"; export class ExternalUserContext extends SubContext { public async getAll(uuid: Uuid): Promise { @@ -39,11 +40,13 @@ export class ExternalUserContext extends SubContext { } const uuid = uuidGenerator(); - result = await this.query("INSERT INTO external_user " + + result = await this.query( + "INSERT INTO external_user " + "(name, uuid, local_uuid, service, cookies) " + "VALUES (?,?,?,?,?);", - [externalUser.identifier, uuid, localUuid, externalUser.type, externalUser.cookies], + [externalUser.identifier, uuid, localUuid, externalUser.type, externalUser.cookies], ); + storeModifications("external_user", "insert", result); if (!result.affectedRows) { return Promise.reject(new Error(Errors.UNKNOWN)); @@ -61,13 +64,14 @@ export class ExternalUserContext extends SubContext { // would violate the foreign keys restraints // first delete list - medium links - await this.query( + const result = await this.query( "DELETE FROM external_list_medium " + "WHERE list_id " + "IN (SELECT id FROM external_reading_list " + "WHERE user_uuid =?);" , externalUuid, ); + storeModifications("external_list_item", "delete", result); // proceed to delete lists of external user await this.delete("external_reading_list", {column: "user_uuid", value: externalUuid}); // finish by deleting external user itself diff --git a/src/server/bin/database/contexts/internalListContext.ts b/src/server/bin/database/contexts/internalListContext.ts index 9e982ae0..a27447db 100644 --- a/src/server/bin/database/contexts/internalListContext.ts +++ b/src/server/bin/database/contexts/internalListContext.ts @@ -1,6 +1,7 @@ import {SubContext} from "./subContext"; import {List, Medium, Uuid} from "../../types"; import {Errors, promiseMultiSingle} from "../../tools"; +import { storeModifications } from "../sqlTools"; export class InternalListContext extends SubContext { /** @@ -12,6 +13,7 @@ export class InternalListContext extends SubContext { "INSERT INTO reading_list (user_uuid, name, medium) VALUES (?,?,?)", [uuid, name, medium], ); + storeModifications("list", "insert", result); if (!Number.isInteger(result.insertId)) { throw Error(`invalid ID: ${result.insertId}`); } @@ -160,6 +162,7 @@ export class InternalListContext extends SubContext { medium.id, (value) => [medium.listId, value] ); + storeModifications("list_item", "insert", result); return result.affectedRows > 0; } diff --git a/src/server/bin/database/contexts/jobContext.ts b/src/server/bin/database/contexts/jobContext.ts index 013c8f5f..004eff17 100644 --- a/src/server/bin/database/contexts/jobContext.ts +++ b/src/server/bin/database/contexts/jobContext.ts @@ -4,6 +4,8 @@ import {isString, promiseMultiSingle} from "../../tools"; import logger from "../../logger"; import mysql from "promise-mysql"; import {escapeLike} from "../storages/storageTools"; +import { getStore } from "../../asyncStorage"; +import { storeModifications } from "../sqlTools"; export class JobContext extends SubContext { public async removeJobLike(column: string, value: any): Promise { @@ -19,7 +21,8 @@ export class JobContext extends SubContext { noBoundaries: true, singleQuotes: true }); - await this.query(`DELETE FROM jobs WHERE ${mysql.escapeId(column)} LIKE ?`, like); + const result = await this.query(`DELETE FROM jobs WHERE ${mysql.escapeId(column)} LIKE ?`, like); + storeModifications("job", "delete", result); } } @@ -104,6 +107,7 @@ export class JobContext extends SubContext { // @ts-ignore value.id = result.insertId; } + storeModifications("job", "insert", result); } delete value.runImmediately; return value as unknown as JobItem; @@ -119,11 +123,13 @@ export class JobContext extends SubContext { } public async removeJob(key: string | number): Promise { + let result; if (isString(key)) { - await this.query("DELETE FROM jobs WHERE `name` = ?", key); + result = await this.query("DELETE FROM jobs WHERE `name` = ?", key); } else { - await this.query("DELETE FROM jobs WHERE id = ?", key); + result = await this.query("DELETE FROM jobs WHERE id = ?", key); } + storeModifications("job", "delete", result); } public async updateJobs(jobs: JobItem | JobItem[], finished?: Date): Promise { @@ -177,9 +183,17 @@ export class JobContext extends SubContext { if (value.arguments && !isString(value.arguments)) { args = JSON.stringify(value.arguments); } + const store = getStore(); + if (!store) { + throw Error("missing store - is this running outside a AsyncLocalStorage Instance?"); + } + const context = store.get("context"); + const result = store.get("result") || "success"; + const message = store.get("message") || "Missing Message"; + return this.query( - "INSERT INTO job_history (id, type, name, deleteAfterRun, runAfter, start, end, arguments)" + - " VALUES (?,?,?,?,?,?,?,?);", + "INSERT INTO job_history (id, type, name, deleteAfterRun, runAfter, start, end, result, message, context, arguments)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?);", [ value.id, value.type, @@ -188,6 +202,9 @@ export class JobContext extends SubContext { value.runAfter, value.runningSince, finished, + result, + message, + JSON.stringify(context), args ] ); diff --git a/src/server/bin/database/contexts/mediumContext.ts b/src/server/bin/database/contexts/mediumContext.ts index 32bbc939..f444704f 100644 --- a/src/server/bin/database/contexts/mediumContext.ts +++ b/src/server/bin/database/contexts/mediumContext.ts @@ -11,13 +11,15 @@ import { Uuid, SecondaryMedium } from "../../types"; -import {count, Errors, getElseSet, ignore, invalidId, multiSingle, promiseMultiSingle} from "../../tools"; +import {count, Errors, getElseSet, invalidId, multiSingle, promiseMultiSingle} from "../../tools"; import {escapeLike} from "../storages/storageTools"; import {escape, Query} from "mysql"; +import { storeModifications } from "../sqlTools"; export class MediumContext extends SubContext { public async removeToc(tocLink: string): Promise { - await this.query("DELETE FROM medium_toc WHERE link = ?", tocLink); + const result = await this.query("DELETE FROM medium_toc WHERE link = ?", tocLink); + storeModifications("toc", "delete", result); } /** @@ -34,6 +36,7 @@ export class MediumContext extends SubContext { if (!Number.isInteger(result.insertId)) { throw Error(`invalid ID: ${result.insertId}`); } + storeModifications("medium", "insert", result); await this.parentContext.partContext.createStandardPart(result.insertId); @@ -388,19 +391,21 @@ export class MediumContext extends SubContext { params.push([value.mediumId, item]); }); }); - await this.multiInsert( + const result = await this.multiInsert( "INSERT IGNORE INTO medium_synonyms (medium_id, synonym) VALUES", params, (value) => value ); + storeModifications("synonym", "insert", result); return true; } - public addToc(mediumId: number, link: string): Promise { - return this.query( + public async addToc(mediumId: number, link: string): Promise { + const result = await this.query( "INSERT IGNORE INTO medium_toc (medium_id, link) VAlUES (?,?)", [mediumId, link] - ).then(ignore); + ); + storeModifications("toc", "insert", result); } public async getToc(mediumId: number): Promise { @@ -446,7 +451,7 @@ export class MediumContext extends SubContext { } } } - const updatedReleaseResult = await this.query( + const deletedReleaseResult = await this.query( "DELETE er FROM episode_release as er, episode as e, part as p" + " WHERE er.episode_id = e.id" + " AND e.part_id = p.id" + @@ -454,7 +459,9 @@ export class MediumContext extends SubContext { " AND locate(?,er.url) > 0;", [mediumId, domain] ); - const updatedProgressResult = await this.queryInList( + storeModifications("release", "delete", deletedReleaseResult); + + const deletedProgressResult = await this.queryInList( "DELETE ue FROM user_episode as ue, episode as e, part as p" + " WHERE ue.episode_id = e.id" + " AND e.part_id = p.id" + @@ -462,7 +469,8 @@ export class MediumContext extends SubContext { " AND e.id", removeEpisodesAfter ); - const updatedResultResult = await this.queryInList( + // TODO: storeModifications("external_list_item", "delete", result); + const deletedResultResult = await this.queryInList( "DELETE re FROM result_episode as re, episode as e, part as p" + " WHERE re.episode_id = e.id" + " AND e.part_id = p.id" + @@ -470,10 +478,12 @@ export class MediumContext extends SubContext { " AND e.id", removeEpisodesAfter ); + // TODO: storeModifications("external_list_item", "delete", result); const deletedEpisodesResult = await this.queryInList( "DELETE FROM episode WHERE episode.id", removeEpisodesAfter ); + // TODO: storeModifications("external_list_item", "delete", result); return this.delete( "medium_toc", {column: "medium_id", value: mediumId}, @@ -512,25 +522,32 @@ export class MediumContext extends SubContext { // remove all tocs of source await this.delete("medium_toc", {column: "medium_id", value: sourceMediumId}); - await this.query( + let result = await this.query( "UPDATE IGNORE list_medium SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] ); + storeModifications("list_item", "update", result); + await this.delete("list_medium", {column: "medium_id", value: sourceMediumId}); - await this.query( + result = await this.query( "UPDATE IGNORE external_list_medium SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] ); + storeModifications("external_list_item", "update", result); + await this.delete("external_list_medium", {column: "medium_id", value: sourceMediumId}); - await this.query( + result = await this.query( "UPDATE IGNORE medium_synonyms SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] ); + storeModifications("synonym", "update", result); + await this.delete("medium_synonyms", {column: "medium_id", value: sourceMediumId}); await this.query( "UPDATE IGNORE news_medium SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] ); + await this.delete("news_medium", {column: "medium_id", value: sourceMediumId}); const deletedReleaseResult = await this.query( "DELETE er FROM episode_release as er, episode as e, part as p" + @@ -539,6 +556,8 @@ export class MediumContext extends SubContext { " AND p.medium_id = ?", sourceMediumId ); + storeModifications("release", "delete", deletedReleaseResult); + const deletedProgressResult = await this.query( "DELETE ue FROM user_episode as ue, episode as e, part as p" + " WHERE ue.episode_id = e.id" + @@ -546,6 +565,8 @@ export class MediumContext extends SubContext { " AND p.medium_id = ?", sourceMediumId ); + storeModifications("progress", "delete", deletedProgressResult); + const deletedResultResult = await this.query( "DELETE re FROM result_episode as re, episode as e, part as p" + " WHERE re.episode_id = e.id" + @@ -553,22 +574,30 @@ export class MediumContext extends SubContext { " AND p.medium_id = ?", sourceMediumId ); + storeModifications("result_episode", "delete", deletedResultResult); + const deletedEpisodesResult = await this.query( "DELETE e FROM episode as e, part as p" + " WHERE e.part_id = p.id" + " AND p.medium_id = ?", sourceMediumId ); + storeModifications("episode", "delete", deletedEpisodesResult); + const deletedPartResult = await this.query( "DELETE FROM part" + " WHERE medium_id = ?", sourceMediumId ); + storeModifications("part", "delete", deletedPartResult); + const deletedMediumResult = await this.query( "DELETE FROM medium" + " WHERE id = ?", sourceMediumId ); + storeModifications("medium", "delete", deletedMediumResult); + return true; } @@ -583,6 +612,7 @@ export class MediumContext extends SubContext { if (!Number.isInteger(result.insertId)) { throw Error(`invalid ID: ${result.insertId}`); } + storeModifications("medium", "insert", result); let mediumId: number; // medium exists already if insertId == 0 if (result.insertId === 0) { @@ -622,6 +652,8 @@ export class MediumContext extends SubContext { "UPDATE IGNORE medium_toc SET medium_id = ? WHERE (medium_id, link) = (?,?);", [destMediumId, sourceMediumId, toc] ); + storeModifications("toc", "update", updatedTocResult); + const releases = await this.parentContext.episodeContext.getEpisodeLinksByMedium(sourceMediumId); const episodeMap: Map = new Map(); const valueCb = () => []; @@ -652,6 +684,7 @@ export class MediumContext extends SubContext { ` WHERE part.medium_id = ${escape(sourceMediumId)} AND episode.id`, copyEpisodes ); + // TODO: storeModifications("progress", "insert", result); const updatedReleaseResult = await this.query( "UPDATE IGNORE episode_release, episode as src_e, episode as dest_e, part" + " SET episode_release.episode_id = dest_e.id" + @@ -663,6 +696,8 @@ export class MediumContext extends SubContext { " AND locate(?,episode_release.url) > 0;", [sourceMediumId, standardPartId, domain] ); + storeModifications("release", "update", updatedReleaseResult); + const updatedProgressResult = await this.queryInList( "UPDATE IGNORE user_episode, episode as src_e, episode as dest_e, part" + " SET user_episode.episode_id = dest_e.id" + @@ -674,6 +709,8 @@ export class MediumContext extends SubContext { " AND src_e.id", removeEpisodesAfter ); + // TODO: storeModifications("progress", "update", updatedProgressResult); + const updatedResultResult = await this.queryInList( "UPDATE IGNORE result_episode, episode as src_e, episode as dest_e, part" + " SET result_episode.episode_id = dest_e.id" + @@ -685,29 +722,35 @@ export class MediumContext extends SubContext { " AND src_e.id", removeEpisodesAfter ); + // TODO: storeModifications("release", "update", result); + const deletedReleasesResult = await this.queryInList( "DELETE FROM episode_release" + " WHERE episode_id", removeEpisodesAfter ); + // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); const deletedUserEpisodesResult = await this.queryInList( "DELETE FROM user_episode" + " WHERE episode_id", removeEpisodesAfter ); + // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); const deletedResultEpisodesResult = await this.queryInList( "DELETE FROM result_episode" + " WHERE episode_id", removeEpisodesAfter ); + // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); const deletedEpisodesResult = await this.queryInList( "DELETE FROM episode" + " WHERE id", removeEpisodesAfter ); + // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); const copiedOnlyEpisodes: number[] = copyEpisodes.filter((value) => !removeEpisodesAfter.includes(value)); const copiedProgressResult = await this.queryInList( - "INSERT IGNORE INTO user_episode" + + " IGNORE INTO user_episode" + " (user_uuid, episode_id, progress, read_date)" + " SELECT user_episode.user_uuid, dest_e.id, user_episode.progress, user_episode.read_date" + " FROM user_episode, episode as src_e, episode as dest_e, part" + @@ -719,6 +762,7 @@ export class MediumContext extends SubContext { " AND src_e.id", copiedOnlyEpisodes ); + // TODO: storeModifications("progress", "insert", result); const copiedResultResult = await this.queryInList( "INSERT IGNORE INTO result_episode" + " (novel, chapter, chapIndex, volIndex, volume, episode_id)" + @@ -733,6 +777,7 @@ export class MediumContext extends SubContext { " AND src_e.id", copiedOnlyEpisodes ); + // TODO: storeModifications("progress", "insert", result); return true; } } diff --git a/src/server/bin/database/contexts/mediumInWaitContext.ts b/src/server/bin/database/contexts/mediumInWaitContext.ts index 7c9b4ecd..9afeb5c6 100644 --- a/src/server/bin/database/contexts/mediumInWaitContext.ts +++ b/src/server/bin/database/contexts/mediumInWaitContext.ts @@ -90,5 +90,6 @@ export class MediumInWaitContext extends SubContext { mediaInWait, (value: any) => [value.title, value.medium, value.link] ); + // TODO: storeModifications("progress", "insert", result); } } diff --git a/src/server/bin/database/contexts/newsContext.ts b/src/server/bin/database/contexts/newsContext.ts index 76b9ce6a..b910d41c 100644 --- a/src/server/bin/database/contexts/newsContext.ts +++ b/src/server/bin/database/contexts/newsContext.ts @@ -1,6 +1,7 @@ import {SubContext} from "./subContext"; import {News, Uuid} from "../../types"; import {Errors, promiseMultiSingle} from "../../tools"; +import { storeModifications } from "../sqlTools"; export class NewsContext extends SubContext { /** @@ -30,6 +31,7 @@ export class NewsContext extends SubContext { if (!Number.isInteger(result.insertId)) { throw Error(`invalid ID ${result.insertId}`); } + storeModifications("news", "insert", result); if (!result.affectedRows) { return; } @@ -136,6 +138,7 @@ export class NewsContext extends SubContext { await this.query("DELETE FROM news_medium WHERE news_id IN " + "(SELECT news_id FROM news_board WHERE date < NOW() - INTERVAL 30 DAY);"); const result = await this.query("DELETE FROM news_board WHERE date < NOW() - INTERVAL 30 DAY;"); + storeModifications("news", "delete", result); return result.affectedRows > 0; } @@ -148,6 +151,7 @@ export class NewsContext extends SubContext { news, (value) => [uuid, value] ); + // TODO: storeModifications("progress", "insert", result); return true; } diff --git a/src/server/bin/database/contexts/partContext.ts b/src/server/bin/database/contexts/partContext.ts index a1b8c172..fb47f86f 100644 --- a/src/server/bin/database/contexts/partContext.ts +++ b/src/server/bin/database/contexts/partContext.ts @@ -4,6 +4,7 @@ import mySql from "promise-mysql"; import {combiIndex, getElseSetObj, multiSingle, separateIndex} from "../../tools"; import {Query} from "mysql"; import { MysqlServerError } from "../mysqlError"; +import { storeModifications } from "../sqlTools"; export class PartContext extends SubContext { public async getAll(): Promise { @@ -275,6 +276,7 @@ export class PartContext extends SubContext { [part.mediumId, part.title, part.totalIndex, part.partialIndex, partCombiIndex], ); partId = result.insertId; + storeModifications("part", "insert", result); } catch (e) { // do not catch if it isn't an duplicate key error if (!e || (e.errno !== MysqlServerError.ER_DUP_KEY && e.errno !== MysqlServerError.ER_DUP_ENTRY)) { @@ -359,6 +361,7 @@ export class PartContext extends SubContext { "INSERT IGNORE INTO part (medium_id,title, totalIndex, combiIndex) VALUES (?,?,?,?);", [mediumId, partName, -1, -1] ).then((value): ShallowPart => { + storeModifications("part", "insert", value); return { totalIndex: -1, title: partName, diff --git a/src/server/bin/database/databaseSchema.ts b/src/server/bin/database/databaseSchema.ts index f7e6d561..04222fe6 100644 --- a/src/server/bin/database/databaseSchema.ts +++ b/src/server/bin/database/databaseSchema.ts @@ -1,7 +1,7 @@ import {DataBaseBuilder} from "./databaseBuilder"; import {Migrations} from "./migrations"; -const dataBaseBuilder = new DataBaseBuilder(10); +const dataBaseBuilder = new DataBaseBuilder(11); dataBaseBuilder.getTableBuilder() .setName("user") @@ -291,8 +291,11 @@ dataBaseBuilder.getTableBuilder() .parseColumn("name VARCHAR(200) NOT NULL") .parseColumn("deleteAfterRun BOOLEAN NOT NULL") .parseColumn("runAfter INT") - .parseColumn("start DATETIME") - .parseColumn("end DATETIME") + .parseColumn("start DATETIME NOT NULL") + .parseColumn("end DATETIME NOT NULL") + .parseColumn("result VARCHAR(100) NOT NULL") + .parseColumn("message VARCHAR(200) NOT NULL") + .parseColumn("context TEXT NOT NULL") .parseColumn("arguments TEXT") .parseMeta("PRIMARY KEY(id, start)") .build(); diff --git a/src/server/bin/database/migrations.ts b/src/server/bin/database/migrations.ts index 4df85e85..8f812132 100644 --- a/src/server/bin/database/migrations.ts +++ b/src/server/bin/database/migrations.ts @@ -271,5 +271,37 @@ export const Migrations: Migration[] = [ // add index to speed up queries on episode_release where releaseDate is a big factor await context.addIndex("episode_release", "episode_release_releaseDate_Index", ["releaseDate"]); } + }, + { + fromVersion: 10, + toVersion: 11, + async migrate(context: DatabaseContext): Promise { + // TODO: should i ask for user input before? + // remove all data, because this change is destructive + // one cannot/should not simulate the data for the new columns + await context.query("TRUNCATE job_history;"); + // add columns and ignore duplicate column error + await Promise.all([ + "result VARCHAR(100) NOT NULL", + "message VARCHAR(200) NOT NULL", + "context TEXT NOT NULL" + ].map((value) => ignoreError( + () => context.addColumn( + "job_history", + value, + ), + [MysqlServerError.ER_DUP_FIELDNAME] + ))); + // add not null restraint + await context.alterColumn( + "job_history", + "start DATETIME NOT NULL", + ); + // add not null restraint + await context.alterColumn( + "job_history", + "end DATETIME NOT NULL", + ); + } } ]; diff --git a/src/server/bin/database/sqlTools.ts b/src/server/bin/database/sqlTools.ts new file mode 100644 index 00000000..76772e99 --- /dev/null +++ b/src/server/bin/database/sqlTools.ts @@ -0,0 +1,35 @@ +import { OkPacket } from "mysql"; +import { getStore } from "../asyncStorage"; +import { getElseSet, getElseSetObj } from "../tools"; + +interface Modification { + created: number; + deleted: number; + updated: number; +} + +export type QueryType = "select" | "update" | "insert" | "delete"; +export type ModificationKey = "progress" | "result_episode" | "medium" | "part" | "episode" | "release" | "list" | "external_list" | "external_user" | "user" | "list_item" | "external_list_item" | "job" | "synonym" | "toc" | "news"; + +/** + * Store the type of modification in the Async Storage associated with this context. + * + * @param key + */ +export function storeModifications(key: ModificationKey, queryType: QueryType, result: OkPacket): void { + const store = getStore(); + + if (!store) { + return; + } + const modifications = getElseSet(store, "modifications", () => { return {}; }); + const modification: Modification = getElseSetObj(modifications, key, () => { return { created: 0, deleted: 0, updated: 0 }; }); + + if (queryType === "delete") { + modification.deleted += result.affectedRows; + } else if (queryType === "insert") { + modification.created += result.affectedRows; + } else if (queryType === "update") { + modification.updated += result.changedRows; + } +} \ No newline at end of file diff --git a/src/server/bin/externals/jobScraperManager.ts b/src/server/bin/externals/jobScraperManager.ts index 50d479aa..d2be4eed 100644 --- a/src/server/bin/externals/jobScraperManager.ts +++ b/src/server/bin/externals/jobScraperManager.ts @@ -72,7 +72,7 @@ export class JobScraperManager { this.helper.init(); } - public on(event: string, callback: (value: any) => void): void { + public on(event: string, callback: (value: any) => void | Promise): void { this.helper.on(event, callback); } diff --git a/src/server/bin/externals/scraperTools.ts b/src/server/bin/externals/scraperTools.ts index 346c47f5..57edb2f9 100644 --- a/src/server/bin/externals/scraperTools.ts +++ b/src/server/bin/externals/scraperTools.ts @@ -683,7 +683,7 @@ export enum ScrapeEvent { } export class ScraperHelper { - private readonly eventMap: Map void>> = new Map(); + private readonly eventMap: Map void | Promise>> = new Map(); public get redirects(): RegExp[] { return redirects; @@ -705,18 +705,19 @@ export class ScraperHelper { return newsAdapter; } - public on(event: string, callback: (value: any) => void): void { + public on(event: string, callback: (value: any) => void | Promise): void { const callbacks = getElseSet(this.eventMap, event, () => []); callbacks.push(callback); } - public emit(event: string, value: any): void { + public emit(event: string, value: any): Promise { if (env.stopScrapeEvents) { logger.info("not emitting events"); - return; + return Promise.resolve(); } const callbacks = getElseSet(this.eventMap, event, () => []); - callbacks.forEach((cb) => cb(value)); + // return a promise of all callbacks yielding a promise + return Promise.all(callbacks.map((cb) => cb(value)).filter(cbValue => cbValue)).then(() => undefined); } public init(): void { diff --git a/src/server/bin/jobHandler.ts b/src/server/bin/jobHandler.ts index bd0b43e9..a6931873 100644 --- a/src/server/bin/jobHandler.ts +++ b/src/server/bin/jobHandler.ts @@ -41,6 +41,7 @@ import { storage } from "./database/storages/storage"; import {MissingResourceError, UrlError} from "./externals/errors"; +import { getStore } from "./asyncStorage"; const scraper = DefaultJobScraper; @@ -716,6 +717,11 @@ async function newsHandler({link, result}: { link: string; result: News[] }) { } async function tocErrorHandler(error: Error) { + const store = getStore(); + if (store) { + store.set("result", "failed"); + store.set("message", error.message); + } // TODO: 10.03.2020 remove any releases associated? with this toc // to do that, it needs to be checked if there are other toc from this domain (unlikely) // and if there are to scrape them and delete any releases that are not contained in them @@ -737,16 +743,36 @@ async function tocErrorHandler(error: Error) { } } -scraper.on("feed:error", (errorValue: any) => logger.error(errorValue)); +scraper.on("feed:error", (errorValue: any) => { + const store = getStore(); + if (store) { + store.set("result", "failed"); + store.set("message", errorValue.message); + } + logger.error(errorValue); +}); scraper.on("toc:error", (errorValue: any) => tocErrorHandler(errorValue)); -scraper.on("list:error", (errorValue: any) => logger.error(errorValue)); - -scraper.on("news:error", (errorValue: any) => logger.error(errorValue)); -scraper.on("news", (result) => newsHandler(result).catch((error) => logger.error(error))); -scraper.on("toc", (result) => tocHandler(result).catch((error) => logger.error(error))); -scraper.on("feed", (result) => feedHandler(result).catch((error) => logger.error(error))); - -scraper.on("list", (result) => listHandler(result).catch((error) => logger.error(error))); +scraper.on("list:error", (errorValue: any) => { + const store = getStore(); + if (store) { + store.set("result", "failed"); + store.set("message", errorValue.message); + } + logger.error(errorValue); +}); + +scraper.on("news:error", (errorValue: any) => { + const store = getStore(); + if (store) { + store.set("result", "failed"); + store.set("message", errorValue.message); + } + logger.error(errorValue); +}); +scraper.on("news", (result) => newsHandler(result)); +scraper.on("toc", (result) => tocHandler(result)); +scraper.on("feed", (result) => feedHandler(result)); +scraper.on("list", (result) => listHandler(result)); export const startCrawler = (): void => { scraper diff --git a/src/server/bin/jobManager.ts b/src/server/bin/jobManager.ts index f02ae377..dc52a3a7 100644 --- a/src/server/bin/jobManager.ts +++ b/src/server/bin/jobManager.ts @@ -1,4 +1,4 @@ -import {remove, removeLike, stringify} from "./tools"; +import {remove, removeLike, stringify, getElseSet} from "./tools"; import logger from "./logger"; import {JobRequest} from "./types"; import {getStore, runAsync, setContext, removeContext} from "./asyncStorage"; @@ -357,8 +357,12 @@ export class JobQueue { logger.info("executing job: " + toExecute.jobId); return toExecute.job(() => this._done(toExecute)); }); + getElseSet(store, "result", () => "success"); + store.set("message", JSON.stringify(store.get("modifications") || {})); } catch (error) { remove(this.waitingJobs, toExecute); + store.set("result", "failed"); + store.set("message", error.message); logger.error(`Job ${toExecute.jobId} threw an error somewhere ${stringify(error)}`); } finally { removeContext("Job"); From 70df3c45fd13cd267c9c4aed6c7b5e8059ffd198 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 18:01:52 +0100 Subject: [PATCH 12/16] feat(scraper): store more modifications, store query count --- .../bin/database/contexts/episodeContext.ts | 71 ++++++++++----- .../database/contexts/externalListContext.ts | 39 +++++--- .../database/contexts/externalUserContext.ts | 17 ++-- .../database/contexts/internalListContext.ts | 34 +++++-- .../bin/database/contexts/jobContext.ts | 8 +- .../bin/database/contexts/mediumContext.ts | 88 +++++++++++++------ .../database/contexts/mediumInWaitContext.ts | 36 ++++---- .../bin/database/contexts/newsContext.ts | 3 +- .../bin/database/contexts/partContext.ts | 6 +- .../bin/database/contexts/queryContext.ts | 51 ++++++++--- .../bin/database/contexts/subContext.ts | 15 ++-- .../bin/database/contexts/userContext.ts | 7 +- src/server/bin/database/sqlTools.ts | 22 ++++- src/server/bin/jobManager.ts | 8 +- 14 files changed, 285 insertions(+), 120 deletions(-) diff --git a/src/server/bin/database/contexts/episodeContext.ts b/src/server/bin/database/contexts/episodeContext.ts index 52f777ae..0efd4473 100644 --- a/src/server/bin/database/contexts/episodeContext.ts +++ b/src/server/bin/database/contexts/episodeContext.ts @@ -29,7 +29,7 @@ import { import logger from "../../logger"; import { MysqlServerError } from "../mysqlError"; import { escapeLike } from "../storages/storageTools"; -import { Query } from "mysql"; +import { Query, OkPacket } from "mysql"; import { storeModifications } from "../sqlTools"; export class EpisodeContext extends SubContext { @@ -225,21 +225,23 @@ export class EpisodeContext extends SubContext { if (progress < 0 || progress > 1) { return Promise.reject(new Error(Errors.INVALID_INPUT)); } - await this.multiInsert( + const results = await this.multiInsert( "REPLACE INTO user_episode " + "(user_uuid, episode_id, progress, read_date) " + "VALUES ", episodeId, (value) => [uuid, value, progress, readDate] ); + // @ts-expect-error + multiSingle(results, (value: OkPacket) => storeModifications("progress", "update", value)) return true; } /** * Removes progress of an user in regard to an episode. */ - public removeProgress(uuid: Uuid, episodeId: number): Promise { - return this.delete( + public async removeProgress(uuid: Uuid, episodeId: number): Promise { + const result = await this.delete( "user_episode", { column: "user_uuid", @@ -250,6 +252,8 @@ export class EpisodeContext extends SubContext { value: episodeId, }, ); + storeModifications("progress", "delete", result); + return result.affectedRows > 0; } /** @@ -486,7 +490,7 @@ export class EpisodeContext extends SubContext { public async addRelease(releases: EpisodeRelease | EpisodeRelease[]): Promise { - await this.multiInsert( + const results = await this.multiInsert( "INSERT IGNORE INTO episode_release " + "(episode_id, title, url, source_type, releaseDate, locked) " + "VALUES", @@ -504,7 +508,8 @@ export class EpisodeContext extends SubContext { release.locked ]; }); - // TODO: storeModifications("progress", "insert", result); + // @ts-expect-error + multiSingle(results, (value: OkPacket) => storeModifications("release", "insert", value)) return releases; } @@ -546,7 +551,7 @@ export class EpisodeContext extends SubContext { // @ts-ignore return promiseMultiSingle(releases, async (value: EpisodeRelease): Promise => { if (value.episodeId) { - await this.update( + const result = await this.update( "episode_release", (updates, values) => { if (value.title) { @@ -573,6 +578,7 @@ export class EpisodeContext extends SubContext { value: value.url, } ); + storeModifications("release", "update", result); } else if (value.sourceType) { const result = await this.query( "UPDATE episode_release SET url=? WHERE source_type=? AND url != ? AND title=?", @@ -583,8 +589,8 @@ export class EpisodeContext extends SubContext { }).then(ignore); } - public deleteRelease(release: EpisodeRelease): Promise { - return this.delete( + public async deleteRelease(release: EpisodeRelease): Promise { + const result = await this.delete( "episode_release", { column: "episode_id", @@ -594,7 +600,8 @@ export class EpisodeContext extends SubContext { column: "url", value: release.url } - ).then(ignore); + ); + storeModifications("release", "delete", result); } public async getEpisodeContentData(chapterLink: string): Promise { @@ -851,7 +858,7 @@ export class EpisodeContext extends SubContext { * Updates an episode from the storage. */ public async updateEpisode(episode: SimpleEpisode): Promise { - return this.update("episode", (updates, values) => { + const result = await this.update("episode", (updates, values) => { if (episode.partId) { updates.push("part_id = ?"); values.push(episode.partId); @@ -873,6 +880,8 @@ export class EpisodeContext extends SubContext { }, { column: "id", value: episode.id }); + storeModifications("episode", "update", result); + return result.changedRows > 0; } /** @@ -897,12 +906,13 @@ export class EpisodeContext extends SubContext { ); const changePartIds: number[] = changePartIdsResult.map((value) => value.id); - const result = await this.queryInList( + let result = await this.queryInList( `UPDATE episode SET part_id=${mySql.escape(newPartId)} ` + `WHERE part_id=${mySql.escape(oldPartId)} AND combiIndex`, changePartIds ); - // TODO: storeModifications("release", "update", result); + // @ts-expect-error + multiSingle(result, value => storeModifications("release", "update", value)); if (!replaceIds.length) { return true; } @@ -955,17 +965,24 @@ export class EpisodeContext extends SubContext { })); const oldIds = replaceIds.map((value) => value.oldId); // TODO: 26.08.2019 this does not go quite well, throws error with 'cannot delete parent reference' - await this.queryInList("DELETE FROM episode_release WHERE episode_id ", deleteReleaseIds); - // TODO: storeModifications("release", "update", result); - await this.queryInList("DELETE FROM user_episode WHERE episode_id ", deleteProgressIds); - // TODO: storeModifications("release", "update", result); - await this.queryInList("DELETE FROM result_episode WHERE episode_id ", deleteResultIds); - // TODO: storeModifications("release", "update", result); - await this.queryInList( + result = await this.queryInList("DELETE FROM episode_release WHERE episode_id ", deleteReleaseIds); + // @ts-expect-error + multiSingle(result, value => storeModifications("release", "delete", value)); + + result = await this.queryInList("DELETE FROM user_episode WHERE episode_id ", deleteProgressIds); + // @ts-expect-error + multiSingle(result, value => storeModifications("progress", "delete", value)); + + result = await this.queryInList("DELETE FROM result_episode WHERE episode_id ", deleteResultIds); + // @ts-expect-error + multiSingle(result, value => storeModifications("result_episode", "delete", value)); + + result = await this.queryInList( `DELETE FROM episode WHERE part_id=${mySql.escape(oldPartId)} AND id`, oldIds, ); - // TODO: storeModifications("release", "update", result); + // @ts-expect-error + multiSingle(result, value => storeModifications("episode", "delete", value)); return true; } @@ -974,10 +991,16 @@ export class EpisodeContext extends SubContext { */ public async deleteEpisode(id: number): Promise { // remove episode from progress first - await this.delete("user_episode", { column: "episode_id", value: id }); - await this.delete("episode_release", { column: "episode_id", value: id }); + let result = await this.delete("user_episode", { column: "episode_id", value: id }); + storeModifications("progress", "delete", result); + + result = await this.delete("episode_release", { column: "episode_id", value: id }); + storeModifications("release", "delete", result); + // lastly remove episode itself - return this.delete("episode", { column: "id", value: id }); + result = await this.delete("episode", { column: "id", value: id }); + storeModifications("episode", "delete", result); + return result.affectedRows > 0; } public async getChapterIndices(mediumId: number): Promise { diff --git a/src/server/bin/database/contexts/externalListContext.ts b/src/server/bin/database/contexts/externalListContext.ts index b733c23b..9e973b7d 100644 --- a/src/server/bin/database/contexts/externalListContext.ts +++ b/src/server/bin/database/contexts/externalListContext.ts @@ -1,7 +1,8 @@ import {SubContext} from "./subContext"; import {ExternalList, Uuid} from "../../types"; -import {Errors, promiseMultiSingle} from "../../tools"; +import {Errors, promiseMultiSingle, multiSingle} from "../../tools"; import { storeModifications } from "../sqlTools"; +import { OkPacket } from 'mysql'; export class ExternalListContext extends SubContext { public async getAll(uuid: Uuid): Promise { @@ -50,8 +51,8 @@ export class ExternalListContext extends SubContext { /** * Updates an external list. */ - public updateExternalList(externalList: ExternalList): Promise { - return this.update("external_reading_list", (updates, values) => { + public async updateExternalList(externalList: ExternalList): Promise { + const result = await this.update("external_reading_list", (updates, values) => { if (externalList.medium) { updates.push("medium = ?"); values.push(externalList.medium); @@ -62,6 +63,8 @@ export class ExternalListContext extends SubContext { values.push(externalList.name); } }, {column: "user_uuid", value: externalList.id}); + storeModifications("external_list", "delete", result); + return result.changedRows > 0; } /** @@ -70,11 +73,13 @@ export class ExternalListContext extends SubContext { public async removeExternalList(uuid: Uuid, externalListId: number | number[]): Promise { // TODO: 29.06.2019 replace with id IN (...) and list_id IN (...) // @ts-ignore - return promiseMultiSingle(externalListId, async (item) => { + const results = await promiseMultiSingle(externalListId, async (item) => { // first delete any references of externalList: list-media links - await this.delete("external_list_medium", {column: "list_id", value: item}); + let result = await this.delete("external_list_medium", {column: "list_id", value: item}); + storeModifications("external_list_item", "delete", result); + // then delete list itself - return this.delete("external_reading_list", + result = await this.delete("external_reading_list", { column: "user_uuid", value: uuid, @@ -83,7 +88,10 @@ export class ExternalListContext extends SubContext { column: "id", value: item, }); + storeModifications("external_list", "delete", result); + return result.affectedRows > 0; }); + return Array.isArray(results) ? results.some(v => v) : results; } @@ -164,16 +172,24 @@ export class ExternalListContext extends SubContext { medium.id, (value) => [medium.listId, value] ); - storeModifications("external_list_item", "insert", result); - return result.affectedRows > 0; + let added = false; + // @ts-expect-error + multiSingle(result, (value: OkPacket) => { + storeModifications("external_list_item", "insert", value); + + if (value.affectedRows > 0) { + added = true + } + }) + return added; } /** * Removes an item from a list. */ public removeMedium(listId: number, mediumId: number | number[]): Promise { - return promiseMultiSingle(mediumId, (value) => { - return this.delete( + return promiseMultiSingle(mediumId, async (value) => { + const result = await this.delete( "external_list_medium", { column: "list_id", @@ -183,7 +199,8 @@ export class ExternalListContext extends SubContext { column: "medium_id", value, }); - + storeModifications("external_list_item", "delete", result); + return result.affectedRows > 0; }).then(() => true); } } diff --git a/src/server/bin/database/contexts/externalUserContext.ts b/src/server/bin/database/contexts/externalUserContext.ts index 813b9fea..d93aac19 100644 --- a/src/server/bin/database/contexts/externalUserContext.ts +++ b/src/server/bin/database/contexts/externalUserContext.ts @@ -64,7 +64,7 @@ export class ExternalUserContext extends SubContext { // would violate the foreign keys restraints // first delete list - medium links - const result = await this.query( + let result = await this.query( "DELETE FROM external_list_medium " + "WHERE list_id " + "IN (SELECT id FROM external_reading_list " + @@ -72,10 +72,15 @@ export class ExternalUserContext extends SubContext { , externalUuid, ); storeModifications("external_list_item", "delete", result); + // proceed to delete lists of external user - await this.delete("external_reading_list", {column: "user_uuid", value: externalUuid}); + result = await this.delete("external_reading_list", {column: "user_uuid", value: externalUuid}); + storeModifications("external_list", "delete", result); + // finish by deleting external user itself - return this.delete("external_user", {column: "uuid", value: externalUuid}); + result = await this.delete("external_user", {column: "uuid", value: externalUuid}); + storeModifications("external_user", "delete", result); + return result.affectedRows > 0; } /** @@ -152,8 +157,8 @@ export class ExternalUserContext extends SubContext { /** * Updates an external user. */ - public updateExternalUser(externalUser: ExternalUser): Promise { - return this.update("external_user", (updates, values) => { + public async updateExternalUser(externalUser: ExternalUser): Promise { + const result = await this.update("external_user", (updates, values) => { if (externalUser.identifier) { updates.push("name = ?"); values.push(externalUser.identifier); @@ -171,5 +176,7 @@ export class ExternalUserContext extends SubContext { updates.push("cookies = NULL"); } }, {column: "uuid", value: externalUser.uuid}); + storeModifications("external_user", "update", result); + return result.changedRows > 0; } } diff --git a/src/server/bin/database/contexts/internalListContext.ts b/src/server/bin/database/contexts/internalListContext.ts index a27447db..80c8d417 100644 --- a/src/server/bin/database/contexts/internalListContext.ts +++ b/src/server/bin/database/contexts/internalListContext.ts @@ -84,9 +84,9 @@ export class InternalListContext extends SubContext { */ public async updateList(list: List): Promise { if (!list.userUuid) { - return Promise.reject(new Error(Errors.INVALID_INPUT)); + throw new Error(Errors.INVALID_INPUT); } - return this.update("reading_list", (updates, values) => { + const result = await this.update("reading_list", (updates, values) => { if (list.name) { updates.push("name = ?"); values.push(list.name); @@ -100,6 +100,8 @@ export class InternalListContext extends SubContext { column: "id", value: list.id }); + storeModifications("list", "update", result); + return result.changedRows > 0; } @@ -117,9 +119,13 @@ export class InternalListContext extends SubContext { return Promise.reject(new Error(Errors.DOES_NOT_EXIST)); } // first remove all links between a list and their media - await this.delete("list_medium", {column: "list_id", value: listId}); + let deleteResult = await this.delete("list_medium", {column: "list_id", value: listId}); + storeModifications("list_item", "delete", deleteResult); + // lastly delete the list itself - return this.delete("reading_list", {column: "id", value: listId}); + deleteResult = await this.delete("reading_list", {column: "id", value: listId}); + storeModifications("list", "delete", deleteResult); + return deleteResult.affectedRows > 0; } /** @@ -162,8 +168,16 @@ export class InternalListContext extends SubContext { medium.id, (value) => [medium.listId, value] ); - storeModifications("list_item", "insert", result); - return result.affectedRows > 0; + let added = false; + // @ts-expect-error + multiSingle(result, (value: OkPacket) => { + storeModifications("list_item", "insert", value); + + if (value.affectedRows > 0) { + added = true + } + }) + return added; } /** @@ -182,15 +196,17 @@ export class InternalListContext extends SubContext { * Removes an item from a list. */ public async removeMedium(listId: number, mediumId: number | number[], external = false): Promise { - await promiseMultiSingle(mediumId, (value) => { - return this.delete("list_medium", { + const results = await promiseMultiSingle(mediumId, async (value) => { + const result = await this.delete("list_medium", { column: "list_id", value: listId, }, { column: "medium_id", value, }); + storeModifications("list_item", "delete", result); + return result.affectedRows > 0; }); - return true; + return Array.isArray(results) ? results.some(v => v) : results; } } diff --git a/src/server/bin/database/contexts/jobContext.ts b/src/server/bin/database/contexts/jobContext.ts index 004eff17..606ce1c0 100644 --- a/src/server/bin/database/contexts/jobContext.ts +++ b/src/server/bin/database/contexts/jobContext.ts @@ -1,6 +1,6 @@ import {SubContext} from "./subContext"; import {JobItem, JobRequest, JobState} from "../../types"; -import {isString, promiseMultiSingle} from "../../tools"; +import {isString, promiseMultiSingle, multiSingle} from "../../tools"; import logger from "../../logger"; import mysql from "promise-mysql"; import {escapeLike} from "../storages/storageTools"; @@ -115,7 +115,9 @@ export class JobContext extends SubContext { } public async removeJobs(jobs: JobItem | JobItem[], finished?: Date): Promise { - await this.queryInList("DELETE FROM jobs WHERE id", jobs, undefined, (value) => value.id); + const result = await this.queryInList("DELETE FROM jobs WHERE id", jobs, undefined, (value) => value.id); + // @ts-expect-error + multiSingle(result, value => storeModifications("job", "delete", value)); if (finished) { await this.addJobHistory(jobs, finished); @@ -187,7 +189,7 @@ export class JobContext extends SubContext { if (!store) { throw Error("missing store - is this running outside a AsyncLocalStorage Instance?"); } - const context = store.get("context"); + const context = store.get("history"); const result = store.get("result") || "success"; const message = store.get("message") || "Missing Message"; diff --git a/src/server/bin/database/contexts/mediumContext.ts b/src/server/bin/database/contexts/mediumContext.ts index f444704f..f78fae65 100644 --- a/src/server/bin/database/contexts/mediumContext.ts +++ b/src/server/bin/database/contexts/mediumContext.ts @@ -284,7 +284,7 @@ export class MediumContext extends SubContext { /** * Updates a medium from the storage. */ - public updateMediumToc(mediumToc: FullMediumToc): Promise { + public async updateMediumToc(mediumToc: FullMediumToc): Promise { const keys = [ "countryOfOrigin", "languageOfOrigin", "author", "title", "medium", "artist", "lang", "stateOrigin", "stateTL", "series", "universe" @@ -301,7 +301,7 @@ export class MediumContext extends SubContext { } else { conditions.push({column: "id", value: mediumToc.id}); } - return this.update("medium_toc", (updates, values) => { + const result = await this.update("medium_toc", (updates, values) => { for (const key of keys) { const value = mediumToc[key]; @@ -313,12 +313,14 @@ export class MediumContext extends SubContext { } } }, ...conditions); + storeModifications("toc", "update", result); + return result.changedRows > 0; } /** * Updates a medium from the storage. */ - public updateMedium(medium: UpdateMedium): Promise { + public async updateMedium(medium: UpdateMedium): Promise { const keys = [ "countryOfOrigin", "languageOfOrigin", "author", "title", "medium", "artist", "lang", "stateOrigin", "stateTL", "series", "universe" @@ -333,7 +335,7 @@ export class MediumContext extends SubContext { if (!Number.isInteger(medium.id) || medium.id <= 0) { throw Error("invalid medium, id, title or medium is invalid: " + JSON.stringify(medium)); } - return this.update("medium", (updates, values) => { + const result = await this.update("medium", (updates, values) => { for (const key of keys) { const value = medium[key]; @@ -345,6 +347,8 @@ export class MediumContext extends SubContext { } } }, {column: "id", value: medium.id}); + storeModifications("medium", "update", result); + return result.changedRows > 0; } public async getSynonyms(mediumId: number | number[]): Promise { @@ -368,8 +372,8 @@ export class MediumContext extends SubContext { public removeSynonyms(synonyms: Synonyms | Synonyms[]): Promise { // @ts-ignore return promiseMultiSingle(synonyms, (value: Synonyms) => { - return promiseMultiSingle(value.synonym, (item) => { - return this.delete("medium_synonyms", + return promiseMultiSingle(value.synonym, async (item) => { + const result = await this.delete("medium_synonyms", { column: "synonym", value: item @@ -378,6 +382,8 @@ export class MediumContext extends SubContext { column: "medium_id", value: value.mediumId }); + storeModifications("synonym", "delete", result); + return result.affectedRows > 0; }); }).then(() => true); } @@ -396,7 +402,8 @@ export class MediumContext extends SubContext { params, (value) => value ); - storeModifications("synonym", "insert", result); + // @ts-expect-error + multiSingle(result, (value) => storeModifications("synonym", "insert", value)); return true; } @@ -469,7 +476,9 @@ export class MediumContext extends SubContext { " AND e.id", removeEpisodesAfter ); - // TODO: storeModifications("external_list_item", "delete", result); + // @ts-expect-error + multiSingle(deletedProgressResult, value => storeModifications("progress", "delete", value)); + const deletedResultResult = await this.queryInList( "DELETE re FROM result_episode as re, episode as e, part as p" + " WHERE re.episode_id = e.id" + @@ -478,17 +487,23 @@ export class MediumContext extends SubContext { " AND e.id", removeEpisodesAfter ); - // TODO: storeModifications("external_list_item", "delete", result); + // @ts-expect-error + multiSingle(deletedResultResult, value => storeModifications("result_episode", "delete", value)); + const deletedEpisodesResult = await this.queryInList( "DELETE FROM episode WHERE episode.id", removeEpisodesAfter ); - // TODO: storeModifications("external_list_item", "delete", result); - return this.delete( + // @ts-expect-error + multiSingle(deletedEpisodesResult, value => storeModifications("episode", "delete", value)); + + const result = await this.delete( "medium_toc", {column: "medium_id", value: mediumId}, {column: "link", value: link} ); + storeModifications("toc", "delete", result); + return result.affectedRows > 0; } public getAllMediaTocs(): Promise> { @@ -521,28 +536,36 @@ export class MediumContext extends SubContext { ); // remove all tocs of source - await this.delete("medium_toc", {column: "medium_id", value: sourceMediumId}); - let result = await this.query( + let result = await this.delete("medium_toc", {column: "medium_id", value: sourceMediumId}); + storeModifications("toc", "delete", result); + + result = await this.query( "UPDATE IGNORE list_medium SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] ); storeModifications("list_item", "update", result); - await this.delete("list_medium", {column: "medium_id", value: sourceMediumId}); + result = await this.delete("list_medium", {column: "medium_id", value: sourceMediumId}); + storeModifications("list_item", "delete", result); + result = await this.query( "UPDATE IGNORE external_list_medium SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] ); storeModifications("external_list_item", "update", result); - await this.delete("external_list_medium", {column: "medium_id", value: sourceMediumId}); + result = await this.delete("external_list_medium", {column: "medium_id", value: sourceMediumId}); + storeModifications("external_list_item", "delete", result); + result = await this.query( "UPDATE IGNORE medium_synonyms SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] ); storeModifications("synonym", "update", result); - await this.delete("medium_synonyms", {column: "medium_id", value: sourceMediumId}); + result = await this.delete("medium_synonyms", {column: "medium_id", value: sourceMediumId}); + storeModifications("synonym", "delete", result); + await this.query( "UPDATE IGNORE news_medium SET medium_id=? WHERE medium_id=?", [destMediumId, sourceMediumId] @@ -684,7 +707,9 @@ export class MediumContext extends SubContext { ` WHERE part.medium_id = ${escape(sourceMediumId)} AND episode.id`, copyEpisodes ); - // TODO: storeModifications("progress", "insert", result); + // @ts-expect-error + multiSingle(copyEpisodesResult, value => storeModifications("episode", "insert", value)); + const updatedReleaseResult = await this.query( "UPDATE IGNORE episode_release, episode as src_e, episode as dest_e, part" + " SET episode_release.episode_id = dest_e.id" + @@ -709,7 +734,8 @@ export class MediumContext extends SubContext { " AND src_e.id", removeEpisodesAfter ); - // TODO: storeModifications("progress", "update", updatedProgressResult); + // @ts-expect-error + multiSingle(updatedProgressResult, value => storeModifications("progress", "update", value)); const updatedResultResult = await this.queryInList( "UPDATE IGNORE result_episode, episode as src_e, episode as dest_e, part" + @@ -722,32 +748,41 @@ export class MediumContext extends SubContext { " AND src_e.id", removeEpisodesAfter ); - // TODO: storeModifications("release", "update", result); + // @ts-expect-error + multiSingle(updatedResultResult, value => storeModifications("result_episode", "update", value)); const deletedReleasesResult = await this.queryInList( "DELETE FROM episode_release" + " WHERE episode_id", removeEpisodesAfter ); - // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); + // @ts-expect-error + multiSingle(deletedReleasesResult, value => storeModifications("release", "delete", value)); + const deletedUserEpisodesResult = await this.queryInList( "DELETE FROM user_episode" + " WHERE episode_id", removeEpisodesAfter ); - // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); + // @ts-expect-error + multiSingle(deletedUserEpisodesResult, value => storeModifications("progress", "delete", value)); + const deletedResultEpisodesResult = await this.queryInList( "DELETE FROM result_episode" + " WHERE episode_id", removeEpisodesAfter ); - // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); + // @ts-expect-error + multiSingle(deletedResultEpisodesResult, value => storeModifications("result_episode", "delete", value)); + const deletedEpisodesResult = await this.queryInList( "DELETE FROM episode" + " WHERE id", removeEpisodesAfter ); - // TODO: storeModifications("result_episode", "delete", deletedReleaseResult); + // @ts-expect-error + multiSingle(deletedEpisodesResult, value => storeModifications("episode", "delete", value)); + const copiedOnlyEpisodes: number[] = copyEpisodes.filter((value) => !removeEpisodesAfter.includes(value)); const copiedProgressResult = await this.queryInList( " IGNORE INTO user_episode" + @@ -762,7 +797,9 @@ export class MediumContext extends SubContext { " AND src_e.id", copiedOnlyEpisodes ); - // TODO: storeModifications("progress", "insert", result); + // @ts-expect-error + multiSingle(copiedProgressResult, value => storeModifications("progress", "insert", value)); + const copiedResultResult = await this.queryInList( "INSERT IGNORE INTO result_episode" + " (novel, chapter, chapIndex, volIndex, volume, episode_id)" + @@ -777,7 +814,8 @@ export class MediumContext extends SubContext { " AND src_e.id", copiedOnlyEpisodes ); - // TODO: storeModifications("progress", "insert", result); + // @ts-expect-error + multiSingle(copiedResultResult, value => storeModifications("result_episode", "insert", value)); return true; } } diff --git a/src/server/bin/database/contexts/mediumInWaitContext.ts b/src/server/bin/database/contexts/mediumInWaitContext.ts index 9afeb5c6..d222a25e 100644 --- a/src/server/bin/database/contexts/mediumInWaitContext.ts +++ b/src/server/bin/database/contexts/mediumInWaitContext.ts @@ -1,7 +1,8 @@ import {SubContext} from "./subContext"; import {MediumInWait} from "../databaseTypes"; import {Medium, MultiSingle, SimpleMedium} from "../../types"; -import {equalsIgnore, ignore, promiseMultiSingle, sanitizeString} from "../../tools"; +import {equalsIgnore, ignore, promiseMultiSingle, sanitizeString, multiSingle} from "../../tools"; +import { storeModifications } from '../sqlTools'; export class MediumInWaitContext extends SubContext { public async createFromMediaInWait(medium: MediumInWait, same?: MediumInWait[], listId?: number): Promise { @@ -70,26 +71,31 @@ export class MediumInWaitContext extends SubContext { return; } // @ts-ignore - return promiseMultiSingle(mediaInWait, (value: MediumInWait) => this.delete( - "medium_in_wait", - { - column: "title", value: value.title - }, - { - column: "medium", value: value.medium - }, - { - column: "link", value: value.link - }, - )).then(ignore); + return promiseMultiSingle(mediaInWait, async (value: MediumInWait) => { + const result = await this.delete( + "medium_in_wait", + { + column: "title", value: value.title + }, + { + column: "medium", value: value.medium + }, + { + column: "link", value: value.link + }, + ); + storeModifications("medium_in_wait", "delete", result); + return result.affectedRows > 0; + }).then(ignore); } public async addMediumInWait(mediaInWait: MultiSingle): Promise { - await this.multiInsert( + const results = await this.multiInsert( "INSERT IGNORE INTO medium_in_wait (title, medium, link) VALUES ", mediaInWait, (value: any) => [value.title, value.medium, value.link] ); - // TODO: storeModifications("progress", "insert", result); + // @ts-expect-error + multiSingle(results, (result) => storeModifications("medium_in_wait", "insert", result)); } } diff --git a/src/server/bin/database/contexts/newsContext.ts b/src/server/bin/database/contexts/newsContext.ts index b910d41c..2cd76d27 100644 --- a/src/server/bin/database/contexts/newsContext.ts +++ b/src/server/bin/database/contexts/newsContext.ts @@ -151,7 +151,6 @@ export class NewsContext extends SubContext { news, (value) => [uuid, value] ); - // TODO: storeModifications("progress", "insert", result); return true; } @@ -207,6 +206,6 @@ export class NewsContext extends SubContext { value: mediumId, }); } - return this.delete("news_medium", ...columns); + return this.delete("news_medium", ...columns).then(value => value.affectedRows > 0); } } diff --git a/src/server/bin/database/contexts/partContext.ts b/src/server/bin/database/contexts/partContext.ts index fb47f86f..ea5959d1 100644 --- a/src/server/bin/database/contexts/partContext.ts +++ b/src/server/bin/database/contexts/partContext.ts @@ -317,8 +317,8 @@ export class PartContext extends SubContext { /** * Updates a part. */ - public updatePart(part: Part): Promise { - return this.update( + public async updatePart(part: Part): Promise { + const result = await this.update( "part", (updates, values) => { if (part.title) { @@ -345,6 +345,8 @@ export class PartContext extends SubContext { value: part.id } ); + storeModifications("part", "update", result); + return result.changedRows > 0; } /** diff --git a/src/server/bin/database/contexts/queryContext.ts b/src/server/bin/database/contexts/queryContext.ts index 165f4c57..da50df87 100644 --- a/src/server/bin/database/contexts/queryContext.ts +++ b/src/server/bin/database/contexts/queryContext.ts @@ -3,7 +3,7 @@ import {Invalidation, MetaResult, Result, Uuid} from "../../types"; import {Errors, getElseSet, getElseSetObj, ignore, multiSingle, promiseMultiSingle} from "../../tools"; import logger from "../../logger"; import * as validate from "validate.js"; -import {Query} from "mysql"; +import {Query, OkPacket} from "mysql"; import {DatabaseContext} from "./databaseContext"; import {UserContext} from "./userContext"; import {ExternalUserContext} from "./externalUserContext"; @@ -18,6 +18,7 @@ import {MediumInWaitContext} from "./mediumInWaitContext"; import {ConnectionContext} from "../databaseTypes"; import env from "../../env"; import { setContext, removeContext } from "../../asyncStorage"; +import { storeCount } from "../sqlTools"; const database = "enterprise"; @@ -31,6 +32,22 @@ export interface DbTrigger { Table: string; } +export interface Condition { + column: string; + value: any; +} + +function emptyPacket() { + return { + affectedRows: 0, + changedRows: 0, + fieldCount: 0, + insertId: 0, + message: "Not queried", + protocol41: false, + }; +} + /** * A Class for consecutive queries on the same connection. */ @@ -299,6 +316,7 @@ export class QueryContext implements ConnectionContext { try { setContext("sql-query"); result = await this.con.query(query, parameter); + storeCount("queryCount"); } finally { removeContext("sql-query"); } @@ -309,11 +327,22 @@ export class QueryContext implements ConnectionContext { return result; } + /** + * Convenience function for correct return type. + * Should only be used for data manipulation queries like INSERT, UPDATE, DELETE. + * + * @param query sql query + * @param parameter parameter for the sql query + */ + public async dmlQuery(query: string, parameter?: any | any[]): Promise { + return this.query(query, parameter); + } + /** * Deletes one or multiple entries from one specific table, * with only one conditional. */ - public async delete(table: string, ...condition: Array<{ column: string; value: any }>): Promise { + public async delete(table: string, ...condition: Array<{ column: string; value: any }>): Promise { if (!condition || (Array.isArray(condition) && !condition.length)) { return Promise.reject(new Error(Errors.INVALID_INPUT)); } @@ -330,17 +359,14 @@ export class QueryContext implements ConnectionContext { values.push(value.value); }); - const result = await this.query(query, values); - - return result.affectedRows >= 0; + return this.query(query, values); } /** * Updates data from the storage. + * May return a empty OkPacket if no values are to be updated. */ - public async update(table: string, cb: UpdateCallback, ...condition: Array<{ column: string; value: any }>) - : Promise { - + public async update(table: string, cb: UpdateCallback, ...condition: Condition[]): Promise { if (!condition || (Array.isArray(condition) && !condition.length)) { return Promise.reject(new Error(Errors.INVALID_INPUT)); } @@ -353,7 +379,7 @@ export class QueryContext implements ConnectionContext { } if (!updates.length) { - return Promise.resolve(false); + return Promise.resolve(emptyPacket()); } let query = `UPDATE ${mySql.escapeId(table)} SET ${updates.join(", ")} @@ -367,13 +393,12 @@ export class QueryContext implements ConnectionContext { } values.push(value.value); }); - const result = await this.query(query, values); - return result.affectedRows > 0; + return this.query(query, values); } - public multiInsert(query: string, value: T | T[], paramCallback: ParamCallback): Promise { + public multiInsert(query: string, value: T | T[], paramCallback: ParamCallback): Promise { if (!value || (Array.isArray(value) && !value.length)) { - return Promise.resolve(); + return Promise.resolve(emptyPacket()); } if (Array.isArray(value) && value.length > 100) { // @ts-ignore diff --git a/src/server/bin/database/contexts/subContext.ts b/src/server/bin/database/contexts/subContext.ts index f54b89d2..5f2b3d80 100644 --- a/src/server/bin/database/contexts/subContext.ts +++ b/src/server/bin/database/contexts/subContext.ts @@ -1,5 +1,5 @@ -import {QueryContext} from "./queryContext"; -import {Query} from "mysql"; +import {QueryContext, Condition} from "./queryContext"; +import {Query, OkPacket} from "mysql"; import {ConnectionContext} from "../databaseTypes"; @@ -26,23 +26,26 @@ export class SubContext implements ConnectionContext { return this.parentContext.query(query, parameter); } + public dmlQuery(query: string, parameter?: any | any[]): Promise { + return this.parentContext.dmlQuery(query, parameter); + } + /** * Deletes one or multiple entries from one specific table, * with only one conditional. */ - protected async delete(table: string, ...condition: Array<{ column: string; value: any }>): Promise { + protected async delete(table: string, ...condition: Condition[]): Promise { return this.parentContext.delete(table, ...condition); } /** * Updates data from the storage. */ - protected async update(table: string, cb: UpdateCallback, ...condition: Array<{ column: string; value: any }>) - : Promise { + protected async update(table: string, cb: UpdateCallback, ...condition: Condition[]): Promise { return this.parentContext.update(table, cb, ...condition); } - protected multiInsert(query: string, value: T | T[], paramCallback: ParamCallback): Promise { + protected multiInsert(query: string, value: T | T[], paramCallback: ParamCallback): Promise { return this.parentContext.multiInsert(query, value, paramCallback); } diff --git a/src/server/bin/database/contexts/userContext.ts b/src/server/bin/database/contexts/userContext.ts index dd6127eb..6c3f7b18 100644 --- a/src/server/bin/database/contexts/userContext.ts +++ b/src/server/bin/database/contexts/userContext.ts @@ -168,7 +168,7 @@ export class UserContext extends SubContext { * Logs a user out. */ public logoutUser(uuid: Uuid, ip: string): Promise { - return this.delete("user_log", {column: "ip", value: ip}); + return this.delete("user_log", {column: "ip", value: ip}).then(v => v.affectedRows > 0); } @@ -225,7 +225,8 @@ export class UserContext extends SubContext { // TODO check if delete was successful, what if not? // in case the deletion was unsuccessful, just 'ban' any further access to that account // and delete it manually? - return this.delete("user", {column: "uuid", value: uuid}); + const result = await this.delete("user", {column: "uuid", value: uuid}); + return result.affectedRows > 0; } /** @@ -266,7 +267,7 @@ export class UserContext extends SubContext { }, { column: "uuid", value: uuid - }); + }).then(value => value.changedRows > 0); } /** diff --git a/src/server/bin/database/sqlTools.ts b/src/server/bin/database/sqlTools.ts index 76772e99..6d877700 100644 --- a/src/server/bin/database/sqlTools.ts +++ b/src/server/bin/database/sqlTools.ts @@ -9,7 +9,7 @@ interface Modification { } export type QueryType = "select" | "update" | "insert" | "delete"; -export type ModificationKey = "progress" | "result_episode" | "medium" | "part" | "episode" | "release" | "list" | "external_list" | "external_user" | "user" | "list_item" | "external_list_item" | "job" | "synonym" | "toc" | "news"; +export type ModificationKey = "progress" | "result_episode" | "medium" | "part" | "episode" | "release" | "list" | "external_list" | "external_user" | "user" | "list_item" | "external_list_item" | "job" | "synonym" | "toc" | "news" | "medium_in_wait"; /** * Store the type of modification in the Async Storage associated with this context. @@ -17,6 +17,9 @@ export type ModificationKey = "progress" | "result_episode" | "medium" | "part" * @param key */ export function storeModifications(key: ModificationKey, queryType: QueryType, result: OkPacket): void { + if (!result.affectedRows || (!result.changedRows && queryType === "update")) { + return; + } const store = getStore(); if (!store) { @@ -32,4 +35,21 @@ export function storeModifications(key: ModificationKey, queryType: QueryType, r } else if (queryType === "update") { modification.updated += result.changedRows; } +} + +type CountKey = "queryCount"; + +/** + * Increases the counter for the given key. + * + * @param key + */ +export function storeCount(key: CountKey): void { + const store = getStore(); + + if (!store) { + return; + } + const count = store.get(key) || 0; + store.set(key, count + 1); } \ No newline at end of file diff --git a/src/server/bin/jobManager.ts b/src/server/bin/jobManager.ts index dc52a3a7..38115643 100644 --- a/src/server/bin/jobManager.ts +++ b/src/server/bin/jobManager.ts @@ -358,7 +358,13 @@ export class JobQueue { return toExecute.job(() => this._done(toExecute)); }); getElseSet(store, "result", () => "success"); - store.set("message", JSON.stringify(store.get("modifications") || {})); + if (!store.get("message")) { + const message = { + "modifications": store.get("modifications") || {}, + "queryCount": store.get("queryCount") || 0, + } + store.set("message", JSON.stringify(message)); + } } catch (error) { remove(this.waitingJobs, toExecute); store.set("result", "failed"); From 3d494db3c0a7f8fed157ca5e552738447d1d6992 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Mon, 9 Nov 2020 19:04:19 +0100 Subject: [PATCH 13/16] refactor: rename redeclared block variable --- src/server/bin/database/contexts/episodeContext.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/bin/database/contexts/episodeContext.ts b/src/server/bin/database/contexts/episodeContext.ts index 0efd4473..667802b5 100644 --- a/src/server/bin/database/contexts/episodeContext.ts +++ b/src/server/bin/database/contexts/episodeContext.ts @@ -325,11 +325,11 @@ export class EpisodeContext extends SubContext { ); // if a similar/same result was mapped to an episode before, get episode_id and update read if (resultArray[0] && resultArray[0].episode_id != null) { - const queryResult = await this.query( + const insertResult = await this.query( "INSERT IGNORE INTO user_episode (user_uuid, episode_id,progress) VALUES (?,?,0);", [uuid, resultArray[0].episode_id] ); - storeModifications("progress", "insert", queryResult); + storeModifications("progress", "insert", insertResult); return; } From b1be4d099bd690a62b3b9025ba91e9f31e6ac5d3 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Tue, 10 Nov 2020 12:22:35 +0100 Subject: [PATCH 14/16] fix: fix imports --- src/server/bin/database/contexts/externalListContext.ts | 2 +- src/server/bin/database/contexts/internalListContext.ts | 2 +- src/server/bin/database/contexts/mediumInWaitContext.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/bin/database/contexts/externalListContext.ts b/src/server/bin/database/contexts/externalListContext.ts index 9e973b7d..541da0f5 100644 --- a/src/server/bin/database/contexts/externalListContext.ts +++ b/src/server/bin/database/contexts/externalListContext.ts @@ -2,7 +2,7 @@ import {SubContext} from "./subContext"; import {ExternalList, Uuid} from "../../types"; import {Errors, promiseMultiSingle, multiSingle} from "../../tools"; import { storeModifications } from "../sqlTools"; -import { OkPacket } from 'mysql'; +import { OkPacket } from "mysql"; export class ExternalListContext extends SubContext { public async getAll(uuid: Uuid): Promise { diff --git a/src/server/bin/database/contexts/internalListContext.ts b/src/server/bin/database/contexts/internalListContext.ts index 80c8d417..533492df 100644 --- a/src/server/bin/database/contexts/internalListContext.ts +++ b/src/server/bin/database/contexts/internalListContext.ts @@ -1,6 +1,6 @@ import {SubContext} from "./subContext"; import {List, Medium, Uuid} from "../../types"; -import {Errors, promiseMultiSingle} from "../../tools"; +import {Errors, promiseMultiSingle, multiSingle} from "../../tools"; import { storeModifications } from "../sqlTools"; export class InternalListContext extends SubContext { diff --git a/src/server/bin/database/contexts/mediumInWaitContext.ts b/src/server/bin/database/contexts/mediumInWaitContext.ts index d222a25e..1136b136 100644 --- a/src/server/bin/database/contexts/mediumInWaitContext.ts +++ b/src/server/bin/database/contexts/mediumInWaitContext.ts @@ -2,7 +2,7 @@ import {SubContext} from "./subContext"; import {MediumInWait} from "../databaseTypes"; import {Medium, MultiSingle, SimpleMedium} from "../../types"; import {equalsIgnore, ignore, promiseMultiSingle, sanitizeString, multiSingle} from "../../tools"; -import { storeModifications } from '../sqlTools'; +import { storeModifications } from "../sqlTools"; export class MediumInWaitContext extends SubContext { public async createFromMediaInWait(medium: MediumInWait, same?: MediumInWait[], listId?: number): Promise { From 9d6d664536fe657c55bad5958cf16372ced4b822 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Sat, 14 Nov 2020 00:15:57 +0100 Subject: [PATCH 15/16] feat(scraper): track network --- src/server/bin/externals/jobScraperManager.ts | 4 +- src/server/bin/externals/queueManager.ts | 74 ++++++++++++++++++- src/server/bin/externals/scraperTools.ts | 4 +- src/server/bin/jobManager.ts | 13 +++- src/server/bin/tools.ts | 3 +- 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/src/server/bin/externals/jobScraperManager.ts b/src/server/bin/externals/jobScraperManager.ts index d2be4eed..32af9fdd 100644 --- a/src/server/bin/externals/jobScraperManager.ts +++ b/src/server/bin/externals/jobScraperManager.ts @@ -532,8 +532,8 @@ export class JobScraperManager { // TODO: 23.06.2019 add timeout? return value .then((content) => this.helper.emit(eventName, content)) - .catch((reason) => { - this.helper.emit(eventName + ":error", reason); + .catch(async (reason) => { + await this.helper.emit(eventName + ":error", reason); return reason; }); } diff --git a/src/server/bin/externals/queueManager.ts b/src/server/bin/externals/queueManager.ts index 70154817..a90b91b5 100644 --- a/src/server/bin/externals/queueManager.ts +++ b/src/server/bin/externals/queueManager.ts @@ -7,7 +7,75 @@ import {BufferToStringStream} from "../transform"; import {StatusCodeError} from "request-promise-native/errors"; import requestPromise from "request-promise-native"; import {MissingResourceError} from "./errors"; -import { setContext, removeContext } from "../asyncStorage"; +import { setContext, removeContext, getStore } from "../asyncStorage"; +import http from "http"; +import https from "https"; +import { Socket } from "net"; +import { isString, getElseSet, stringify } from "../tools"; +import logger from "../logger"; +import { AsyncResource } from "async_hooks"; + + +type RequestOptions = string | http.RequestOptions | https.RequestOptions; +type RequestCallback = (res: http.IncomingMessage) => void | undefined; + +function patchRequest(module: { request: (opt: RequestOptions, callback?: RequestCallback) => http.ClientRequest }, protocol: string) { + const originalRequest = module.request; + + module.request = function(opt, callback) { + const target = isString(opt) ? opt : (protocol + "://" + opt.host + "" + opt.path); + + const clientRequest = originalRequest(opt); + clientRequest.on("response", (res) => { + function listener() { + let socket: Socket; + + if (clientRequest.socket) { + socket = clientRequest.socket; + } else { + console.error("No sockets available on request" + stringify(res) + stringify(request)); + return; + } + const bytesSend = socket.bytesWritten; + const bytesReceived = socket.bytesRead; + + const store = getStore(); + + if (!store) { + return; + } + + const stats = getElseSet(store, "network", () => { return {count: 0, sent: 0, received: 0, history: []}}); + stats.count += 1; + stats.sent += bytesSend; + stats.received += bytesReceived; + stats.history.push({ + url: target, + method: clientRequest.method, + statusCode: res.statusCode, + send: bytesSend, + received: bytesReceived, + }); + + const url = target.slice(0, 70).padEnd(70); + const method = clientRequest.method.slice(0, 5).padEnd(5); + const httpCode = ((res.statusCode + "") || "?").slice(0, 5).padEnd(5); + const send = ("" + bytesSend).slice(0, 10).padEnd(10); + const received = ("" + bytesReceived).slice(0, 10).padEnd(10); + + logger.debug(`${url} ${method} ${httpCode} ${send} ${received}`); + } + res.once("close", AsyncResource.bind(listener)); + + if (callback) { + AsyncResource.bind(callback)(res); + } + }); + return clientRequest; + } +} +patchRequest(http, "http"); +patchRequest(https, "https"); type CheerioStatic = cheerio.Root; @@ -25,6 +93,8 @@ export class Queue { } public push(callback: Callback): Promise { + callback = AsyncResource.bind(callback); + return new Promise((resolve, reject) => { const worker = () => { return new Promise((subResolve, subReject) => { @@ -248,7 +318,7 @@ export const queueCheerioRequestStream: QueueRequest = (uri, opti return queue.push(() => new Promise((resolve, reject) => streamHtmlParser2(resolve, reject, uri, options))); }; -export const queueCheerioRequest = queueCheerioRequestStream; +export const queueCheerioRequest = queueCheerioRequestBuffered; const transformCheerio = (body: string): CheerioStatic => cheerio.load(body, {decodeEntities: false}); diff --git a/src/server/bin/externals/scraperTools.ts b/src/server/bin/externals/scraperTools.ts index 57edb2f9..8b4cc24a 100644 --- a/src/server/bin/externals/scraperTools.ts +++ b/src/server/bin/externals/scraperTools.ts @@ -710,14 +710,14 @@ export class ScraperHelper { callbacks.push(callback); } - public emit(event: string, value: any): Promise { + public async emit(event: string, value: any): Promise { if (env.stopScrapeEvents) { logger.info("not emitting events"); return Promise.resolve(); } const callbacks = getElseSet(this.eventMap, event, () => []); // return a promise of all callbacks yielding a promise - return Promise.all(callbacks.map((cb) => cb(value)).filter(cbValue => cbValue)).then(() => undefined); + await Promise.all(callbacks.map((cb) => cb(value)).filter(cbValue => cbValue)); } public init(): void { diff --git a/src/server/bin/jobManager.ts b/src/server/bin/jobManager.ts index 38115643..da5dfe54 100644 --- a/src/server/bin/jobManager.ts +++ b/src/server/bin/jobManager.ts @@ -344,7 +344,7 @@ export class JobQueue { if (toExecute.jobInfo.onStart) { try { setContext("Job-OnStart"); - await this.executeCallback(toExecute.jobInfo.onStart) + await this.executeCallback(toExecute.jobInfo.onStart); } catch (error) { logger.error(`Job ${toExecute.jobId} onStart threw an error!: ${stringify(error)}`); } finally { @@ -362,13 +362,22 @@ export class JobQueue { const message = { "modifications": store.get("modifications") || {}, "queryCount": store.get("queryCount") || 0, + "network": store.get("network") || {}, } store.set("message", JSON.stringify(message)); } } catch (error) { remove(this.waitingJobs, toExecute); store.set("result", "failed"); - store.set("message", error.message); + if (!store.get("message")) { + const message = { + "modifications": store.get("modifications") || {}, + "queryCount": store.get("queryCount") || 0, + "network": store.get("network") || {}, + "reason": error.message, + } + store.set("message", JSON.stringify(message)); + } logger.error(`Job ${toExecute.jobId} threw an error somewhere ${stringify(error)}`); } finally { removeContext("Job"); diff --git a/src/server/bin/tools.ts b/src/server/bin/tools.ts index ad9bf605..0b936572 100644 --- a/src/server/bin/tools.ts +++ b/src/server/bin/tools.ts @@ -13,6 +13,7 @@ import * as dns from "dns"; import EventEmitter from "events"; import { validate as validateUuid } from "uuid"; import { isNumber } from "validate.js" +import { AsyncResource } from "async_hooks"; export function remove(array: T[], item: T): boolean { @@ -427,7 +428,7 @@ export function relativeToAbsoluteTime(relative: string): Date | null { */ export function delay(timeout = 1000): Promise { return new Promise((resolve) => { - setTimeout(() => resolve(), timeout); + setTimeout(AsyncResource.bind(() => resolve()), timeout); }); } From e23ec762736fe5765537a044ff026644c3827896 Mon Sep 17 00:00:00 2001 From: Mytlogos Date: Sat, 14 Nov 2020 11:30:38 +0100 Subject: [PATCH 16/16] fix: fix AsyncResource.bind arguments --- src/server/bin/asyncStorage.ts | 17 ++++++++++++++++- src/server/bin/externals/queueManager.ts | 10 +++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/server/bin/asyncStorage.ts b/src/server/bin/asyncStorage.ts index e5c987ab..283860fb 100644 --- a/src/server/bin/asyncStorage.ts +++ b/src/server/bin/asyncStorage.ts @@ -1,4 +1,4 @@ -import {AsyncLocalStorage, createHook} from "async_hooks"; +import {AsyncLocalStorage, createHook, AsyncResource} from "async_hooks"; const localStorage = new AsyncLocalStorage(); @@ -175,3 +175,18 @@ export function runAsync(id: number, store: Map, callback: (...args args ); } + +/** + * Workaround to wrong documentation/functionality of AsyncResource.bind. + * + * @see https://github.com/nodejs/node/issues/36051 + * + * @param func function to bind to current async execution context + */ +export function bindContext any>(func: Func): Func & { asyncResource: AsyncResource } { + // @ts-expect-error + return AsyncResource.bind(function(...args: any[]) { + // @ts-expect-error + return func(this, ...args); + }) as Func & { asyncResource: AsyncResource }; +} \ No newline at end of file diff --git a/src/server/bin/externals/queueManager.ts b/src/server/bin/externals/queueManager.ts index a90b91b5..81ece87d 100644 --- a/src/server/bin/externals/queueManager.ts +++ b/src/server/bin/externals/queueManager.ts @@ -7,7 +7,7 @@ import {BufferToStringStream} from "../transform"; import {StatusCodeError} from "request-promise-native/errors"; import requestPromise from "request-promise-native"; import {MissingResourceError} from "./errors"; -import { setContext, removeContext, getStore } from "../asyncStorage"; +import { setContext, removeContext, getStore, bindContext } from "../asyncStorage"; import http from "http"; import https from "https"; import { Socket } from "net"; @@ -26,7 +26,7 @@ function patchRequest(module: { request: (opt: RequestOptions, callback?: Reques const target = isString(opt) ? opt : (protocol + "://" + opt.host + "" + opt.path); const clientRequest = originalRequest(opt); - clientRequest.on("response", (res) => { + clientRequest.on("response", bindContext((res) => { function listener() { let socket: Socket; @@ -65,12 +65,12 @@ function patchRequest(module: { request: (opt: RequestOptions, callback?: Reques logger.debug(`${url} ${method} ${httpCode} ${send} ${received}`); } - res.once("close", AsyncResource.bind(listener)); + res.once("close", bindContext(listener)); if (callback) { - AsyncResource.bind(callback)(res); + bindContext(callback)(res); } - }); + })); return clientRequest; } }