Skip to content

Commit

Permalink
Update render template in unit and e2e tests (partial impl)
Browse files Browse the repository at this point in the history
  • Loading branch information
@VadimKovalenkoSNF
VadimKovalenkoSNF committed Oct 20, 2023
1 parent 39ba175 commit ee7d002
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 133 deletions.
5 changes: 4 additions & 1 deletion test/e2e/bm.e2e.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as mwoffliner from '../../src/mwoffliner.lib.js'
import { zimdump, zimcheck } from '../util.js'
import { zimdump } from '../util.js'
import { testAllRenders } from '../testAllRenders.js'
import { execa } from 'execa'
import { jest } from '@jest/globals'
Expand All @@ -17,9 +17,12 @@ const parameters = {

await testAllRenders(parameters, async (outFiles) => {
describe('e2e test for bm.wikipedia.org', () => {
// TODO: blocked by issues/1931
/*
test(`test zim integrity for ${outFiles[0]?.renderer} renderer`, async () => {
await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
})
*/

test(`Simple articleList for ${outFiles[0]?.renderer} renderer`, async () => {
// Created 1 output
Expand Down
5 changes: 4 additions & 1 deletion test/e2e/en.e2e.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { testAllRenders } from '../testAllRenders.js'
import domino from 'domino'
import { zimdump, zimcheck } from '../util.js'
import { zimdump } from '../util.js'
import 'dotenv/config.js'
import { jest } from '@jest/globals'
import rimraf from 'rimraf'
Expand Down Expand Up @@ -29,9 +29,12 @@ await testAllRenders(parameters, async (outFiles) => {
const articleFromDump = await zimdump(`show --url A/${parameters.articleList} ${outFiles[0].outFile}`)
describe('e2e test for en.wikipedia.org', () => {
const articleDoc = domino.createDocument(articleFromDump)
// TODO: blocked by issues/1931
/*
test(`test zim integrity for ${outFiles[0]?.renderer} renderer`, async () => {
await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
})
*/
test(`test article header for ${outFiles[0]?.renderer} renderer`, async () => {
expect(articleDoc.querySelector('h1.article-header, h1.pcs-edit-section-title')).toBeTruthy()
})
Expand Down
143 changes: 76 additions & 67 deletions test/e2e/multimediaContent.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,87 +13,96 @@ const parameters = {
articleList: 'User:Kelson/MWoffliner_CI_reference',
redis: process.env.REDIS,
customZimDescription: 'Example of the description',
forceRender: 'WikimediaDesktop',
}

await testAllRenders(parameters, async (outFiles) => {
describe('Multimedia', () => {
test(`check multimedia content from wikipedia test page for ${outFiles[0]?.renderer} renderer`, async () => {
await execa('redis-cli flushall', { shell: true })
// TODO: blocked by issues/1925
if (outFiles[0].renderer !== 'WikimediaMobile') {
test(`check multimedia content from wikipedia test page for ${outFiles[0]?.renderer} renderer`, async () => {
await execa('redis-cli flushall', { shell: true })

expect(outFiles[0].status.articles.success).toEqual(1)
expect(outFiles[0].status.articles.fail).toEqual(0)
const mediaFiles = await zimdump(`list --ns I ${outFiles[0].outFile}`)
expect(outFiles[0].status.articles.success).toEqual(1)
expect(outFiles[0].status.articles.fail).toEqual(0)
const mediaFiles = await zimdump(`list --ns I ${outFiles[0].outFile}`)

expect(mediaFiles.split('\n').sort()).toEqual(
[
'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
'I/Kiwix_icon.svg.png',
'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
})

afterAll(() => {
rimraf.sync(`./${outFiles[0].testId}`)
})
expect(mediaFiles.split('\n').sort()).toEqual(
[
'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
'I/Kiwix_icon.svg.png',
'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
})
afterAll(() => {
rimraf.sync(`./${outFiles[0].testId}`)
})
}
})
})

await testAllRenders({ ...parameters, format: ['nopic', 'novid', 'nopdf', 'nodet'] }, async (outFiles) => {
describe('Multimedia for different formats', () => {
test(`check multimedia content from wikipedia test page with different formates for ${outFiles[0]?.renderer} renderer`, async () => {
await execa('redis-cli flushall', { shell: true })
// TODO: blocked by issues/1925
if (outFiles[0].renderer !== 'WikimediaMobile') {
test(`check multimedia content from wikipedia test page with different formates for ${outFiles[0]?.renderer} renderer`, async () => {
await execa('redis-cli flushall', { shell: true })

expect(outFiles).toHaveLength(4)
for (const dump of outFiles) {
expect(dump.status.articles.success).toEqual(1)
expect(dump.status.articles.fail).toEqual(0)
expect(outFiles).toHaveLength(4)
for (const dump of outFiles) {
expect(dump.status.articles.success).toEqual(1)
expect(dump.status.articles.fail).toEqual(0)

await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()
await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()

const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
if (dump.nopic) {
expect(mediaFiles.split('\n').sort()).toEqual(
[
'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by nopic parameter
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
// 'I/Kiwix_icon.svg.png',
// 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
// 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
// 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
} else if (dump.novid) {
expect(mediaFiles.split('\n').sort()).toEqual(
[
'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by novid parameter
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
'I/Kiwix_icon.svg.png',
// 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
} else if (dump.nopdf) {
expect(mediaFiles.split('\n').sort()).toEqual(
[
// 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', // this file was omitted by nopdf parameter
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
'I/Kiwix_icon.svg.png',
'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
if (dump.nopic) {
expect(mediaFiles.split('\n').sort()).toEqual(
[
'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by nopic parameter
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
// 'I/Kiwix_icon.svg.png',
// 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
// 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
// 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
} else if (dump.novid) {
expect(mediaFiles.split('\n').sort()).toEqual(
[
'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by novid parameter
// 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
'I/Kiwix_icon.svg.png',
// 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
} else if (dump.nopdf) {
expect(mediaFiles.split('\n').sort()).toEqual(
[
// 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', // this file was omitted by nopdf parameter
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
'I/Kiwix_icon.svg.png',
'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
].sort(),
)
}
}
}
})
})
afterAll(() => {
rimraf.sync(`./${outFiles[0].testId}`)
})
}
})
})
138 changes: 85 additions & 53 deletions test/unit/downloader.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,91 @@ describe('Downloader class', () => {
expect(LondonImage.responseHeaders['content-type']).toMatch(/image\//i)
})

describe('getArticle method', () => {
let dump: Dump
const wikimediaDesktopRenderer = new WikimediaDesktopRenderer()
beforeAll(async () => {
const mwMetadata = await MediaWiki.getMwMetaData(downloader)
dump = new Dump('', {} as any, mwMetadata)
})

test('getArticle of "London" returns one article for WikimediaDesktop render', async () => {
const articleId = 'London'
const articleUrl = getArticleUrl(downloader, dump, articleId)
const articleDetail = {
title: articleId,
thumbnail: {
width: 50,
height: 28,
source: 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/67/London_Skyline_%28125508655%29.jpeg/50px-London_Skyline_%28125508655%29.jpeg',
},
revisionId: 1171340841,
timestamp: '2023-08-20T14:54:01Z',
coordinates: '51.50722222;-0.1275',
}
const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
const LondonArticle = await downloader.getArticle(
downloader.webp,
_moduleDependencies,
articleId,
RedisStore.articleDetailXId,
wikimediaDesktopRenderer,
articleUrl,
dump,
articleDetail,
dump.isMainPage(articleId),
)
expect(LondonArticle).toHaveLength(1)
})

test('Categories with many subCategories are paginated for WikimediaDesktop render', async () => {
const articleId = 'Category:Container_categories'
const _moduleDependencies = await downloader.getModuleDependencies(articleId)
const articleDetail = {
title: articleId,
ns: 14,
revisionId: 1168361498,
timestamp: '2023-08-02T09:57:11Z',
}
const articleUrl = getArticleUrl(downloader, dump, articleDetail.title)
const PaginatedArticle = await downloader.getArticle(
downloader.webp,
_moduleDependencies,
articleId,
RedisStore.articleDetailXId,
wikimediaDesktopRenderer,
articleUrl,
dump,
articleDetail,
dump.isMainPage(articleId),
)
expect(PaginatedArticle.length).toBeGreaterThan(100)
})

test('getArticle response status for non-existent article id is 404 for WikimediaDesktop render', async () => {
const articleId = 'NeverExistingArticle'
const articleUrl = getArticleUrl(downloader, dump, articleId)
const articleDetail = {
title: articleId,
missing: '',
}
const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
await expect(
downloader.getArticle(
downloader.webp,
_moduleDependencies,
'NeverExistingArticle',
RedisStore.articleDetailXId,
wikimediaDesktopRenderer,
articleUrl,
dump,
articleDetail,
dump.isMainPage(articleId),
),
).rejects.toThrowError(new Error('Request failed with status code 404'))
})
})

describe('getArticle method', () => {
for (const renderer of RENDERERS_LIST) {
let rendererInstance
Expand All @@ -149,59 +234,6 @@ describe('Downloader class', () => {
dump = new Dump('', {} as any, mwMetadata)
})

test(`getArticle of "London" returns one article for ${renderer} render`, async () => {
const articleId = 'London'
const articleUrl = getArticleUrl(downloader, dump, articleId)
const articleDetail = {
title: articleId,
thumbnail: {
width: 50,
height: 28,
source: 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/67/London_Skyline_%28125508655%29.jpeg/50px-London_Skyline_%28125508655%29.jpeg',
},
revisionId: 1171340841,
timestamp: '2023-08-20T14:54:01Z',
coordinates: '51.50722222;-0.1275',
}
const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
const LondonArticle = await downloader.getArticle(
downloader.webp,
_moduleDependencies,
articleId,
RedisStore.articleDetailXId,
rendererInstance,
articleUrl,
dump,
articleDetail,
dump.isMainPage(articleId),
)
expect(LondonArticle).toHaveLength(1)
})

test(`Categories with many subCategories are paginated for ${renderer} render`, async () => {
const articleId = 'Category:Container_categories'
const _moduleDependencies = await downloader.getModuleDependencies(articleId)
const articleDetail = {
title: articleId,
ns: 14,
revisionId: 1168361498,
timestamp: '2023-08-02T09:57:11Z',
}
const articleUrl = getArticleUrl(downloader, dump, articleDetail.title)
const PaginatedArticle = await downloader.getArticle(
downloader.webp,
_moduleDependencies,
articleId,
RedisStore.articleDetailXId,
rendererInstance,
articleUrl,
dump,
articleDetail,
dump.isMainPage(articleId),
)
expect(PaginatedArticle.length).toBeGreaterThan(100)
})

test(`getArticle response status for non-existent article id is 404 for ${renderer} render`, async () => {
const articleId = 'NeverExistingArticle'
const articleUrl = getArticleUrl(downloader, dump, articleId)
Expand Down
Loading

0 comments on commit ee7d002

Please sign in to comment.