Optimize static files saving

src/Dump.ts

@@ -29,7 +29,6 @@ interface DumpOpts {
   keepEmptyParagraphs: boolean
   tags?: string
   filenameDate: string
-  isMobileRenderer: boolean
 }
 
 export class Dump {

src/MediaWiki.ts

@@ -183,8 +183,8 @@ class MediaWiki {
     this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath)
     this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
     this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
-    this.desktopApiUrl = baseUrlDirector.buildDesktopApiUrl(this.#apiPath)
-    this.mobileApiUrl = baseUrlDirector.buildMobileApiUrl(this.#apiPath)
+    this.desktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#apiPath)
+    this.mobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#apiPath)
     this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt)
     this.mobileModulePath = baseUrlDirector.buildMobileModuleURL()
     this.wikimediaDesktopUrlDirector = new DesktopURLDirector(this.desktopApiUrl.href)

src/mwoffliner.lib.ts

@@ -339,7 +339,6 @@ async function execute(argv: any) {
         keepEmptyParagraphs,
         tags: customZimTags,
         filenameDate,
-        isMobileRenderer: false,
       },
       { ...mwMetaData, mainPage },
       customProcessor,
@@ -401,12 +400,10 @@ async function execute(argv: any) {
     })
     zimCreator.addArticle(scraperArticle)
 
-    logger.info('Copying Static Resource Files')
-    await saveStaticFiles(config, zimCreator)
-
-    // TODO: refactor sequence, this only needed for mobile renderer
     logger.info('Copying Static PCS Override Files')
     await saveStaticPCSFiles(config, zimCreator)
+    logger.info('Copying Static Resource Files')
+    await saveStaticFiles(config, zimCreator)
 
     logger.info('Finding stylesheets to download')
     const stylesheetsToGet = await dump.getRelevantStylesheetUrls(downloader)

src/renderers/visual-editor.renderer.ts

@@ -62,7 +62,7 @@ export class VisualEditorRenderer extends DesktopRenderer {
           articleDetail,
           _moduleDependencies,
           webp,
-          super.templateDesktopArticle(_moduleDependencies, articleId),
+          super.templateDesktopArticle.bind(this),
         )
         result.push({
           articleId,

src/renderers/wikimedia-desktop.renderer.ts

@@ -35,7 +35,6 @@ export class WikimediaDesktopRenderer extends DesktopRenderer {
   public async render(renderOpts: RenderOpts): Promise<any> {
     const result: RenderOutput = []
     const { data, articleId, articleDetailXId, webp, _moduleDependencies, isMainPage, dump } = renderOpts
-    dump.opts.isMobileRenderer = false
     const articleDetail = await renderOpts.articleDetailXId.get(articleId)
 
     // Paginate when there are more than 200 subCategories

src/renderers/wikimedia-mobile.renderer.ts

@@ -23,7 +23,6 @@ export class WikimediaMobileRenderer extends MobileRenderer {
     try {
       const result: RenderOutput = []
       const { data, articleId, webp, _moduleDependencies, dump } = renderOpts
-      dump.opts.isMobileRenderer = true
       const articleDetail = await renderOpts.articleDetailXId.get(articleId)
 
       const displayTitle = this.getStrippedTitle(renderOpts)

src/util/builders/url/base.director.ts

@@ -21,14 +21,14 @@ export default class BaseURLDirector {
       .build(true, '/')
   }
 
-  buildDesktopApiUrl(path?: string) {
+  buildWikimediaDesktopApiUrl(path?: string) {
     return urlBuilder
       .setDomain(this.baseDomain)
       .setPath(path ?? 'api/rest_v1/page/html')
       .build(true, '/')
   }
 
-  buildMobileApiUrl(path?: string) {
+  buildWikimediaMobileApiUrl(path?: string) {
     return urlBuilder
       .setDomain(this.baseDomain)
       .setPath(path ?? 'api/rest_v1/page/mobile-html')

src/util/misc.ts

@@ -162,49 +162,32 @@ export function interpolateTranslationString(str: string, parameters: { [key: st
   return newString
 }
 
-export function saveStaticFiles(config: Config, zimCreator: ZimCreator) {
-  const cssPromises = config.output.cssResources.concat(config.output.mainPageCssResources).map(async (css) => {
+function saveResourceFile(resource: string, type: 'css' | 'js', basePath: string, config: Config, zimCreator: ZimCreator) {
+  return async () => {
     try {
-      const cssCont = await readFilePromise(pathParser.resolve(__dirname, `../../res/${css}.css`))
-      const article = new ZimArticle({ url: cssPath(css), data: cssCont, ns: '-' })
+      const content = await readFilePromise(pathParser.resolve(__dirname, `../../res/${basePath}${resource}.${type}`))
+      const article = new ZimArticle({
+        url: type === 'css' ? cssPath(resource) : jsPath(resource),
+        data: content,
+        ns: '-',
+      })
       zimCreator.addArticle(article)
     } catch (error) {
-      logger.warn(`Could not create ${css} file : ${error}`)
+      const fileType = type === 'css' ? (basePath.includes('pcs') ? 'style PCS override' : 'style') : 'script'
+      logger.warn(`Could not create ${fileType} ${resource} file : ${error}`)
     }
-  })
+  }
+}
 
-  const jsPromises = config.output.jsResources.map(async (js) => {
-    try {
-      const jsCont = await readFilePromise(pathParser.resolve(__dirname, `../../res/${js}.js`))
-      const article = new ZimArticle({ url: jsPath(js), data: jsCont, ns: '-' })
-      zimCreator.addArticle(article)
-    } catch (error) {
-      logger.warn(`Could not create ${js} file : ${error}`)
-    }
-  })
+export function saveStaticFiles(config: Config, zimCreator: ZimCreator) {
+  const cssPromises = config.output.cssResources.concat(config.output.mainPageCssResources).map((css) => saveResourceFile(css, 'css', '', config, zimCreator)())
+  const jsPromises = config.output.jsResources.map((js) => saveResourceFile(js, 'js', '', config, zimCreator)())
   return Promise.all([...cssPromises, ...jsPromises])
 }
 
 export function saveStaticPCSFiles(config: Config, zimCreator: ZimCreator) {
-  const pcsCssPromises = config.output.pcsCssResources.map(async (pcsCss) => {
-    try {
-      const cssCont = await readFilePromise(pathParser.resolve(__dirname, `../../res/pcs/${pcsCss}.css`))
-      const article = new ZimArticle({ url: cssPath(pcsCss), data: cssCont, ns: '-' })
-      zimCreator.addArticle(article)
-    } catch (error) {
-      logger.warn(`Could not create style PCS override ${pcsCss} file : ${error}`)
-    }
-  })
-
-  const pcsJsPromises = config.output.pcsJsResources.map(async (pcsJs) => {
-    try {
-      const jsCont = await readFilePromise(pathParser.resolve(__dirname, `../../res/pcs/${pcsJs}.js`))
-      const article = new ZimArticle({ url: jsPath(pcsJs), data: jsCont, ns: '-' })
-      zimCreator.addArticle(article)
-    } catch (error) {
-      logger.warn(`Could not create script PCS override ${pcsJs} file : ${error}`)
-    }
-  })
+  const pcsCssPromises = config.output.pcsCssResources.map((pcsCss) => saveResourceFile(pcsCss, 'css', 'pcs/', config, zimCreator)())
+  const pcsJsPromises = config.output.pcsJsResources.map((pcsJs) => saveResourceFile(pcsJs, 'js', 'pcs/', config, zimCreator)())
   return Promise.all([...pcsCssPromises, ...pcsJsPromises])
 }
 

test/e2e/mobileRenderIntegrity.test.ts

@@ -1,5 +1,6 @@
 import 'dotenv/config.js'
 import * as mwoffliner from '../../src/mwoffliner.lib.js'
+import * as logger from '../../src/Logger.js'
 import rimraf from 'rimraf'
 import { execa } from 'execa'
 import { jest } from '@jest/globals'
@@ -34,8 +35,8 @@ async function getOutFiles(testId: string, articleList: string, mwUrl: string):
 const commonTreatmentTest = async (articleList: string, mwUrl: string) => {
   if (!zimcheckIsAvailable || !zimdumpIsAvailable) {
     const missingTool = !zimcheckIsAvailable ? 'Zimcheck' : 'Zimdump'
-    console.log(`${missingTool} not installed, skipping test`)
-    return
+    logger.error(`${missingTool} not installed, exiting test`)
+    process.exit(1)
   }
   const now = new Date()
   const testId = `mwo-test-${+now}`

test/unit/builders/url/base.director.test.ts

@@ -25,29 +25,29 @@ describe('BaseURLDirector', () => {
     })
   })
 
-  describe('buildMobileApiUrl', () => {
+  describe('buildWikimediaMobileApiUrl', () => {
     it('should return mobile rest URL with provided path and trailing char', () => {
-      const url = baseUrlDirector.buildMobileApiUrl('api/rest_v2/page/mobile-html')
+      const url = baseUrlDirector.buildWikimediaMobileApiUrl('api/rest_v2/page/mobile-html')
 
       expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/page/mobile-html/')
     })
 
     it('should return mobile rest URL with default path and trailing char', () => {
-      const url = baseUrlDirector.buildMobileApiUrl()
+      const url = baseUrlDirector.buildWikimediaMobileApiUrl()
 
       expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/page/mobile-html/')
     })
   })
 
-  describe('buildDesktopApiUrl', () => {
+  describe('buildWikimediaDesktopApiUrl', () => {
     it('should return a desktop URL with provided path and trailing char', () => {
-      const url = baseUrlDirector.buildDesktopApiUrl('api/rest_v2/page/html')
+      const url = baseUrlDirector.buildWikimediaDesktopApiUrl('api/rest_v2/page/html')
 
       expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/page/html/')
     })
 
     it('should return a desktop URL with default path and trailing char', () => {
-      const url = baseUrlDirector.buildDesktopApiUrl()
+      const url = baseUrlDirector.buildWikimediaDesktopApiUrl()
 
       expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/page/html/')
     })