Refactor tests (partial impl)

src/Downloader.ts

@@ -171,23 +171,23 @@ class Downloader {
     if (!forceRender) {
       //* Objects order in array matters!
       this.baseUrl = basicURLDirector.buildDownloaderBaseUrl([
-        { condition: await MediaWiki.hasWikimediaMobileRestApi(), value: MediaWiki.mobileRestApiUrl.href },
-        { condition: await MediaWiki.hasWikimediaDesktopRestApi(), value: MediaWiki.desktopRestApiUrl.href },
+        { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.mobileApiUrl.href },
+        { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.desktopApiUrl.href },
         { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
       ])
 
       //* Objects order in array matters!
       this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl([
-        { condition: await MediaWiki.hasWikimediaDesktopRestApi(), value: MediaWiki.desktopRestApiUrl.href },
+        { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.desktopApiUrl.href },
         { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
-        { condition: await MediaWiki.hasWikimediaMobileRestApi(), value: MediaWiki.mobileRestApiUrl.href },
+        { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.mobileApiUrl.href },
       ])
     } else {
       switch (forceRender) {
         case 'WikimediaDesktop':
-          if (MediaWiki.hasWikimediaDesktopRestApi()) {
-            this.baseUrl = MediaWiki.desktopRestApiUrl.href
-            this.baseUrlForMainPage = MediaWiki.desktopRestApiUrl.href
+          if (MediaWiki.hasWikimediaDesktopApi()) {
+            this.baseUrl = MediaWiki.desktopApiUrl.href
+            this.baseUrlForMainPage = MediaWiki.desktopApiUrl.href
             break
           }
           break

src/MediaWiki.ts

@@ -59,11 +59,11 @@ class MediaWiki {
   public modulePath: string // only for reading
   public _modulePathOpt: string // only for whiting to generate modulePath
   public webUrl: URL
-  public desktopRestApiUrl: URL
-  public mobileRestApiUrl: URL
+  public desktopApiUrl: URL
+  public mobileApiUrl: URL
 
-  #hasWikimediaDesktopRestApi: boolean | null
-  #hasWikimediaMobileRestApi: boolean | null
+  #hasWikimediaDesktopApi: boolean | null
+  #hasWikimediaMobileApi: boolean | null
   #hasVisualEditorApi: boolean | null
   #hasCoordinates: boolean | null
 
@@ -123,8 +123,8 @@ class MediaWiki {
       formatversion: '2',
     }
 
-    this.#hasWikimediaDesktopRestApi = null
-    this.#hasWikimediaMobileRestApi = null
+    this.#hasWikimediaDesktopApi = null
+    this.#hasWikimediaMobileApi = null
     this.#hasVisualEditorApi = null
     this.#hasCoordinates = null
   }
@@ -133,20 +133,20 @@ class MediaWiki {
     this.initializeMediaWikiDefaults()
   }
 
-  public async hasWikimediaDesktopRestApi(): Promise<boolean> {
-    if (this.#hasWikimediaDesktopRestApi === null) {
-      this.#hasWikimediaDesktopRestApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
-      return this.#hasWikimediaDesktopRestApi
+  public async hasWikimediaDesktopApi(): Promise<boolean> {
+    if (this.#hasWikimediaDesktopApi === null) {
+      this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
+      return this.#hasWikimediaDesktopApi
     }
-    return this.#hasWikimediaDesktopRestApi
+    return this.#hasWikimediaDesktopApi
   }
 
-  public async hasWikimediaMobileRestApi(): Promise<boolean> {
-    if (this.#hasWikimediaMobileRestApi === null) {
-      this.#hasWikimediaMobileRestApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
-      return this.#hasWikimediaMobileRestApi
+  public async hasWikimediaMobileApi(): Promise<boolean> {
+    if (this.#hasWikimediaMobileApi === null) {
+      this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
+      return this.#hasWikimediaMobileApi
     }
-    return this.#hasWikimediaMobileRestApi
+    return this.#hasWikimediaMobileApi
   }
 
   public async hasVisualEditorApi(): Promise<boolean> {
@@ -182,11 +182,11 @@ class MediaWiki {
     this.apiUrl = baseUrlDirector.buildURL(this.#apiPath)
     this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
     this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
-    this.desktopRestApiUrl = baseUrlDirector.buildDesktopRestApiURL(this.#restApiPath)
-    this.mobileRestApiUrl = baseUrlDirector.buildMobileRestApiURL(this.#restApiPath)
+    this.desktopApiUrl = baseUrlDirector.buildDesktopApiUrl(this.#restApiPath)
+    this.mobileApiUrl = baseUrlDirector.buildMobileApiUrl(this.#restApiPath)
     this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt)
-    this.wikimediaDesktopUrlDirector = new DesktopURLDirector(this.desktopRestApiUrl.href)
-    this.wikimediaMobileUrlDirector = new MobileURLDirector(this.mobileRestApiUrl.href)
+    this.wikimediaDesktopUrlDirector = new DesktopURLDirector(this.desktopApiUrl.href)
+    this.wikimediaMobileUrlDirector = new MobileURLDirector(this.mobileApiUrl.href)
     this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
   }
 

src/mwoffliner.lib.ts

@@ -211,8 +211,8 @@ async function execute(argv: any) {
 
   MediaWiki.apiCheckArticleId = mwMetaData.mainPage
   await MediaWiki.hasCoordinates(downloader)
-  await MediaWiki.hasWikimediaDesktopRestApi()
-  const hasWikimediaMobileRestApi = await MediaWiki.hasWikimediaMobileRestApi()
+  await MediaWiki.hasWikimediaDesktopApi()
+  const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi()
   await MediaWiki.hasVisualEditorApi()
   await downloader.setBaseUrls(forceRender)
 
@@ -421,7 +421,7 @@ async function execute(argv: any) {
 
     logger.log('Getting articles')
     stime = Date.now()
-    const { jsModuleDependencies, cssModuleDependencies } = await saveArticles(zimCreator, downloader, dump, hasWikimediaMobileRestApi, forceRender)
+    const { jsModuleDependencies, cssModuleDependencies } = await saveArticles(zimCreator, downloader, dump, hasWikimediaMobileApi, forceRender)
     logger.log(`Fetching Articles finished in ${(Date.now() - stime) / 1000} seconds`)
 
     logger.log(`Found [${jsModuleDependencies.size}] js module dependencies`)

src/renderers/renderer.builder.ts

@@ -10,15 +10,15 @@ export class RendererBuilder {
   public async createRenderer(options: RendererBuilderOptions): Promise<Renderer> {
     const { renderType, renderName } = options
 
-    const [hasVisualEditorApi, hasWikimediaDesktopRestApi, hasWikimediaMobileRestApi] = await Promise.all([
+    const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi] = await Promise.all([
       MediaWiki.hasVisualEditorApi(),
-      MediaWiki.hasWikimediaDesktopRestApi(),
-      MediaWiki.hasWikimediaMobileRestApi(),
+      MediaWiki.hasWikimediaDesktopApi(),
+      MediaWiki.hasWikimediaMobileApi(),
     ])
 
     switch (renderType) {
       case 'desktop':
-        if (hasWikimediaDesktopRestApi) {
+        if (hasWikimediaDesktopApi) {
           // Choose WikimediaDesktopRenderer if it's present, regardless of hasVisualEditorApi value
           return new WikimediaDesktopRenderer()
         } else if (hasVisualEditorApi) {
@@ -28,18 +28,18 @@ export class RendererBuilder {
           process.exit(1)
         }
       case 'mobile':
-        if (hasWikimediaMobileRestApi) {
+        if (hasWikimediaMobileApi) {
           return new WikimediaMobileRenderer()
         }
         logger.error('No available mobile renderer.')
         process.exit(1)
       case 'auto':
-        if (hasWikimediaDesktopRestApi) {
+        if (hasWikimediaDesktopApi) {
           // Choose WikimediaDesktopRenderer if it's present, regardless of hasVisualEditorApi value
           return new WikimediaDesktopRenderer()
         } else if (hasVisualEditorApi) {
           return new VisualEditorRenderer()
-        } else if (hasWikimediaMobileRestApi) {
+        } else if (hasWikimediaMobileApi) {
           return new WikimediaMobileRenderer()
         } else {
           logger.error('No render available at all.')
@@ -49,7 +49,7 @@ export class RendererBuilder {
         // renderName argument is required for 'specific' mode
         switch (renderName) {
           case 'WikimediaDesktop':
-            if (hasWikimediaDesktopRestApi) {
+            if (hasWikimediaDesktopApi) {
               return new WikimediaDesktopRenderer()
             }
             logger.error('Cannot create an instance of WikimediaDesktop renderer.')
@@ -61,7 +61,7 @@ export class RendererBuilder {
             logger.error('Cannot create an instance of VisualEditor renderer.')
             process.exit(1)
           case 'WikimediaMobile':
-            if (hasWikimediaMobileRestApi) {
+            if (hasWikimediaMobileApi) {
               return new WikimediaMobileRenderer()
             }
             logger.error('No available mobile renderer.')

src/renderers/wikimedia-mobile.renderer.ts

@@ -185,6 +185,9 @@ export class WikimediaMobileRenderer extends Renderer {
       .reference-link::after {
         content: none !important;
       }
+      .mw-body h3, .mw-body h2 {
+        width: auto;
+      }
     `
     doc.head.appendChild(styleEl)
 

src/util/builders/url/base.director.ts

@@ -14,21 +14,21 @@ export default class BaseURLDirector {
     return urlBuilder.setDomain(this.baseDomain).setPath(path).build(true)
   }
 
-  buildRestApiURL(path?: string) {
+  buildWikimediaApiURL(path?: string) {
     return urlBuilder
       .setDomain(this.baseDomain)
       .setPath(path ?? 'api/rest_v1')
       .build(true, '/')
   }
 
-  buildDesktopRestApiURL(path?: string) {
+  buildDesktopApiUrl(path?: string) {
     return urlBuilder
       .setDomain(this.baseDomain)
       .setPath(path ?? 'api/rest_v1/page/html')
       .build(true, '/')
   }
 
-  buildMobileRestApiURL(path?: string) {
+  buildMobileApiUrl(path?: string) {
     return urlBuilder
       .setDomain(this.baseDomain)
       .setPath(path ?? 'api/rest_v1/page/mobile-html')

src/util/saveArticles.ts

@@ -12,7 +12,7 @@ import { config } from '../config.js'
 import { getSizeFromUrl, cleanupAxiosError } from './misc.js'
 import { CONCURRENCY_LIMIT, DELETED_ARTICLE_ERROR, MAX_FILE_DOWNLOAD_RETRIES } from './const.js'
 import urlHelper from './url.helper.js'
-import { RendererBuilderOptions, Renderer } from '../renderers/abstract.renderer.js'
+import { Renderer } from '../renderers/abstract.renderer.js'
 import { RendererBuilder } from '../renderers/renderer.builder.js'
 
 export async function downloadFiles(fileStore: RKVS<FileDetail>, retryStore: RKVS<FileDetail>, zimCreator: ZimCreator, dump: Dump, downloader: Downloader, retryCounter = 0) {
@@ -231,7 +231,7 @@ export function getArticleUrl(downloader: Downloader, dump: Dump, articleId: str
 /*
  * Fetch Articles
  */
-export async function saveArticles(zimCreator: ZimCreator, downloader: Downloader, dump: Dump, hasWikimediaMobileRestApi: boolean, forceRender = null) {
+export async function saveArticles(zimCreator: ZimCreator, downloader: Downloader, dump: Dump, hasWikimediaMobileApi: boolean, forceRender = null) {
   const jsModuleDependencies = new Set<string>()
   const cssModuleDependencies = new Set<string>()
   let jsConfigVars = ''
@@ -241,26 +241,21 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
 
   const rendererBuilder = new RendererBuilder()
 
-  let rendererBuilderOptions: RendererBuilderOptions
-
   let mainPageRenderer
   let articlesRenderer
   if (forceRender) {
-    rendererBuilderOptions = {
+    // All articles and main page will use the same renderer if 'forceRender' is specified
+    const renderer = await rendererBuilder.createRenderer({
       renderType: 'specific',
       renderName: forceRender,
-    }
-    // All articles and main page will use the same renderer if 'forceRender' is specified
-    mainPageRenderer = await rendererBuilder.createRenderer(rendererBuilderOptions)
-    articlesRenderer = await rendererBuilder.createRenderer(rendererBuilderOptions)
+    })
+    mainPageRenderer = renderer
+    articlesRenderer = renderer
   } else {
-    rendererBuilderOptions = {
-      renderType: 'desktop',
-    }
-    mainPageRenderer = await rendererBuilder.createRenderer(rendererBuilderOptions)
-    // If the mobile renderer API is not available, switch articles rendering to the auto mode instead
-    rendererBuilderOptions.renderType = hasWikimediaMobileRestApi ? 'mobile' : 'auto'
-    articlesRenderer = await rendererBuilder.createRenderer(rendererBuilderOptions)
+    mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' })
+    articlesRenderer = await rendererBuilder.createRenderer({
+      renderType: hasWikimediaMobileApi ? 'mobile' : 'auto',
+    })
   }
 
   if (dump.customProcessor?.shouldKeepArticle) {

test/e2e/articleLists.test.ts

@@ -22,6 +22,7 @@ describe('articleList', () => {
     outputDirectory: testId,
     redis: process.env.REDIS,
     format: ['nopic'],
+    forceRender: 'WikimediaDesktop',
   }
 
   test('articleList and articleListIgnore check', async () => {

test/e2e/bm.e2e.test.ts

@@ -18,6 +18,7 @@ describe('bm', () => {
     outputDirectory: testId,
     redis: process.env.REDIS,
     format: ['nopic'],
+    forceRender: 'WikimediaDesktop',
   }
 
   test('Simple articleList', async () => {

test/e2e/downloadImage.e2e.test.ts

@@ -20,6 +20,7 @@ describeIf('Check image downloading from S3 using optimisationCacheUrl parameter
     articleList: 'Paris',
     format: ['nodet'],
     optimisationCacheUrl: process.env.S3_URL,
+    forceRender: 'WikimediaDesktop',
   }
 
   test('right scrapping from fr.wikipedia.org with optimisationCacheUrl parameter', async () => {

test/e2e/en10.e2e.test.ts

@@ -21,6 +21,7 @@ describe('en10', () => {
     redis: process.env.REDIS,
     // format: ['nopic', 'novid', 'nopdf', 'nodet'],
     format: ['nopic', 'nopdf'],
+    forceRender: 'WikimediaDesktop',
   }
 
   test('Simple articleList', async () => {

test/e2e/extra.e2e.test.ts

@@ -36,6 +36,7 @@ AC/DC`
       outputDirectory: testId,
       redis: process.env.REDIS,
       format: ['nopic'],
+      forceRender: 'WikimediaDesktop',
     })
 
     // Created 1 outputs

test/e2e/multimediaContent.test.ts

@@ -18,6 +18,7 @@ describe('Multimedia', () => {
     outputDirectory: testId,
     redis: process.env.REDIS,
     customZimDescription: 'Example of the description',
+    forceRender: 'WikimediaDesktop',
   }
 
   test('check multimedia content from wikipedia test page', async () => {

test/e2e/treatMedia.e2e.test.ts

@@ -18,6 +18,7 @@ describe('treatment test', () => {
     articleList,
     outputDirectory: testId,
     redis: process.env.REDIS,
+    forcdRender: 'WikimediaDesktop',
   }
 
   test('media file from hidden element should not be downloaded', async () => {

test/e2e/vikidia.e2e.test.ts

@@ -18,6 +18,7 @@ describe('vikidia', () => {
     redis: process.env.REDIS,
     articleList: 'Alaska',
     customZimDescription: 'Alaska article',
+    forcdRender: 'WikimediaDesktop',
   }
 
   test('right scrapping from vikidia.org', async () => {

test/e2e/wikisource.e2e.test.ts

@@ -18,6 +18,7 @@ describe('wikisource', () => {
     redis: process.env.REDIS,
     format: ['nopic'],
     noLocalParserFallback: true,
+    forcdRender: 'WikimediaDesktop',
   }
 
   test('Wikisource List', async () => {

test/e2e/zimMetadata.e2e.test.ts

@@ -23,6 +23,7 @@ describe('zimMetadata', () => {
     customZimLongDescription: 'Example of the long description',
     customZimTitle: 'Example of the title',
     publisher: 'Example of the publisher',
+    forcdRender: 'WikimediaDesktop',
   }
 
   test('check all zim metadata using zimdump', async () => {