Skip to content

Commit

Permalink
Use progress bar
Browse files Browse the repository at this point in the history
  • Loading branch information
MarvNC committed Jul 12, 2024
1 parent 919a97c commit ea6b027
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 7 deletions.
Binary file modified bun.lockb
Binary file not shown.
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
"start": "bun run src/index.ts"
},
"dependencies": {
"cli-progress": "^3.12.0",
"pinyin-pro": "^3.18.5",
"yomichan-dict-builder": "^2.6.0"
},
"type": "module",
"devDependencies": {
"@types/bun": "latest"
"@types/bun": "latest",
"@types/cli-progress": "^3.11.6"
},
"name": "wikipedia-yomitan",
"peerDependencies": {
Expand Down
1 change: 1 addition & 0 deletions src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ export const languages = {
ja: 'JA',
zh: 'ZH',
};
export const linkCharacter = '⧉';
19 changes: 13 additions & 6 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ import { Dictionary, TermEntry } from 'yomichan-dict-builder';
import { parseArgs } from 'util';

import { parseLine } from './parse/parseLine';
import { languages } from './constants';
import { languages, linkCharacter } from './constants';
import { getVersion } from './util/getVersion';
import type {
StructuredContent,
StructuredContentNode,
} from 'yomichan-dict-builder/dist/types/yomitan/termbank';

const linkCharacter = '⧉';
import * as cliProgress from 'cli-progress';

const outputZipName = (lang: string, date: string, version: string) =>
`${lang} Wikipedia [${date}] (v${version}).zip`;
Expand All @@ -37,6 +37,13 @@ const shortAbstractFile = (lang: string) =>

let processedLines = 0;
let buffer = '';
const progressBar = new cliProgress.SingleBar({
format: `{value} lines | {file}`,
});

progressBar.start(0, 0, {
file: filePath,
});

while (true) {
const { done, value } = await lineReader.read();
Expand All @@ -51,16 +58,16 @@ const shortAbstractFile = (lang: string) =>
buffer = buffer.slice(lineEnd + 1);
processedLines++;

if (processedLines % 1000 === 0) {
console.log(`Processed ${processedLines} lines`);
}
progressBar.update(processedLines);
}
}

progressBar.stop();

console.log(`Processed ${processedLines} lines, exporting zip...`);

await dict.setIndex({
title: `${lang} Wikipedia [${date}] (v${version})`,
title: `${lang.toUpperCase()} Wikipedia [${date}] (v${version})`,
revision: `wikipedia_${version}`,
format: 3,
url: 'https://github.com/MarvNC/wikipedia-yomitan',
Expand Down

0 comments on commit ea6b027

Please sign in to comment.