increase bulk import size for link and create folder if not exist

wikiadventure · May 21, 2024 · f8a38a6 · f8a38a6
1 parent e3d6c99
commit f8a38a6
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/sql-dump-to-arango/index.ts b/sql-dump-to-arango/index.ts
@@ -102,7 +102,7 @@ async function parseAndLoadRedirect() {
         }
 
 
-        if (count % 4096 == 0) {
+        if (count % 32_768 == 0) {
             await previousBatchPromise;
 
             const batch = nextBatch;
@@ -111,7 +111,7 @@ async function parseAndLoadRedirect() {
             previousBatchPromise = insertRedirects(batch, redirect);
             // previousBatchPromise = Promise.resolve() as Promise<any>;
 
-            if (count % 16_384 == 0) {
+            if (count % 32_768*8 == 0) {
                 log(info.bytesRead, count);
             }
         }
@@ -124,7 +124,7 @@ async function parseAndLoadRedirect() {
         nextBatch.push({_from:r[0],_to:id});
 
         count++;
-        if (count % 4096 == 0) {
+        if (count % 32_768 == 0) {
             await previousBatchPromise;
 
             const batch = nextBatch;
@@ -133,7 +133,7 @@ async function parseAndLoadRedirect() {
             previousBatchPromise = insertRedirects(batch, redirect);
             // previousBatchPromise = Promise.resolve() as Promise<any>;
 
-            if (count % 16_384 == 0) {
+            if (count % 32_768*8 == 0) {
                 log(info.bytesRead, count);
             }
         }

diff --git a/sql-dump-to-arango/parser/dumpParser.ts b/sql-dump-to-arango/parser/dumpParser.ts
@@ -1,15 +1,15 @@
-import { createWriteStream, existsSync, ReadStream } from "fs";
+import { createWriteStream, existsSync } from "fs";
 import { createGunzip, type Gunzip } from "node:zlib";
 import { createReadStream } from "node:fs";
 import { stat } from "node:fs/promises";
 import { env } from "../env.js";
-import { DecompressionStream } from "node:stream/web";
+import { mkdir } from "fs/promises";
 
 export type FileType = "page" | "redirect" | "pagelinks";
-await sqlDumpStreamFromWeb("redirect");
 export async function sqlDumpStreamFromCache(fileType:FileType) {
     const path = `./cache/${env.WIKI_LANG}/${env.WIKI_LANG}wiki-latest-${fileType}.sql.gz`;
     if (!existsSync(path)) {
+        await mkdir(`./cache/${env.WIKI_LANG}`, {recursive: true}).catch();
         const writeToFile = createWriteStream(path);
         const response = await fetch(`https://dumps.wikimedia.org/${env.WIKI_LANG}wiki/latest/${env.WIKI_LANG}wiki-latest-${fileType}.sql.gz`);
         const size = parseInt(response.headers.get("Content-Length") || "0");