Cache documents, schema loading + os.cpus for concurrency limit + asy…

…nc fs (#5707)

* Cache documents and schema loading

* changelog

* Use os.cpus to calculate concurrency limit

* fs async

* changes

* revert config on cache (already introduced at codegen level)

* refactor(cli): use native node fs/promises

* fix(cli): imports issue

* fix(changeset): remove useless changeset

* fix(cli): remove duplicated code..

* fix(cli): remove useless async/await

Co-authored-by: Charly POLY <[email protected]>

.changeset/serious-months-enjoy.md

@@ -0,0 +1,5 @@
+---
+'@graphql-codegen/cli': minor
+---
+
+Use os.cpus to calculate concurrency limit

.changeset/two-bears-poke.md

@@ -0,0 +1,5 @@
+---
+'@graphql-codegen/cli': minor
+---
+
+Async File System

packages/graphql-codegen-cli/src/codegen.ts

@@ -20,6 +20,7 @@ import { debugLog } from './utils/debugging';
 import { CodegenContext, ensureContext } from './config';
 import fs from 'fs';
 import path from 'path';
+import { cpus } from 'os';
 // eslint-disable-next-line
 import { createRequire } from 'module';
 import Listr from 'listr';
@@ -381,8 +382,7 @@ export async function executeCodegen(input: CodegenContext | Types.Config): Prom
         {
           // it doesn't stop when one of tasks failed, to finish at least some of outputs
           exitOnError: false,
-          // run 4 at once
-          concurrent: 4,
+          concurrent: cpus().length,
         }
       );
     },

packages/graphql-codegen-cli/src/generate-and-save.ts

@@ -2,7 +2,7 @@ import { lifecycleHooks } from './hooks';
 import { Types } from '@graphql-codegen/plugin-helpers';
 import { executeCodegen } from './codegen';
 import { createWatcher } from './utils/watcher';
-import { fileExists, readSync, writeSync, unlinkFile } from './utils/file-system';
+import { fileExists, readFile, writeFile, unlinkFile } from './utils/file-system';
 import mkdirp from 'mkdirp';
 import { dirname, join, isAbsolute } from 'path';
 import { debugLog } from './utils/debugging';
@@ -26,7 +26,7 @@ export async function generate(
     const staleFilenames = previouslyGeneratedFilenames.filter(f => !filenames.includes(f));
     staleFilenames.forEach(filename => {
       if (shouldOverwrite(config, filename)) {
-        unlinkFile(filename, err => {
+        return unlinkFile(filename, err => {
           const prettyFilename = filename.replace(`${input.cwd || process.cwd()}/`, '');
           if (err) {
             debugLog(`Cannot remove stale file: ${prettyFilename}\n${err}`);
@@ -49,16 +49,15 @@ export async function generate(
       removeStaleFiles(config, generationResult);
     }
 
-    await context.profiler.run(
-      () => lifecycleHooks(config.hooks).beforeAllFileWrite(generationResult.map(r => r.filename)),
-      'Lifecycle: beforeAllFileWrite'
-    );
+    await context.profiler.run(async () => {
+      await lifecycleHooks(config.hooks).beforeAllFileWrite(generationResult.map(r => r.filename));
+    }, 'Lifecycle: beforeAllFileWrite');
 
     await context.profiler.run(
       () =>
         Promise.all(
           generationResult.map(async (result: Types.FileOutput) => {
-            const exists = fileExists(result.filename);
+            const exists = await fileExists(result.filename);
 
             if (!shouldOverwrite(config, result.filename) && exists) {
               return;
@@ -69,11 +68,11 @@ export async function generate(
             let previousHash = recentOutputHash.get(result.filename);
 
             if (!previousHash && exists) {
-              previousHash = hash(readSync(result.filename));
+              previousHash = hash(await readFile(result.filename));
             }
 
             if (previousHash && currentHash === previousHash) {
-              debugLog(`Skipping file (${result.filename}) writing due to identical hash...`);
+              debugLog(`Skipping file (${result.filename}) writing due to indentical hash...`);
 
               return;
             }
@@ -86,11 +85,11 @@ export async function generate(
             const basedir = dirname(result.filename);
             await lifecycleHooks(result.hooks).beforeOneFileWrite(result.filename);
             await lifecycleHooks(config.hooks).beforeOneFileWrite(result.filename);
-            mkdirp.sync(basedir);
+            await mkdirp(basedir);
             const absolutePath = isAbsolute(result.filename)
               ? result.filename
               : join(input.cwd || process.cwd(), result.filename);
-            writeSync(absolutePath, result.content);
+            await writeFile(absolutePath, result.content);
             await lifecycleHooks(result.hooks).afterOneFileWrite(result.filename);
             await lifecycleHooks(config.hooks).afterOneFileWrite(result.filename);
           })
@@ -117,7 +116,7 @@ export async function generate(
   await context.profiler.run(() => lifecycleHooks(config.hooks).beforeDone(), 'Lifecycle: beforeDone');
 
   if (context.profilerOutput) {
-    writeSync(join(context.cwd, context.profilerOutput), JSON.stringify(context.profiler.collect()));
+    await writeFile(join(context.cwd, context.profilerOutput), JSON.stringify(context.profiler.collect()));
   }
 
   return outputFiles;

packages/graphql-codegen-cli/src/utils/file-system.ts

@@ -1,21 +1,22 @@
-import { writeFileSync, statSync, readFileSync, unlink } from 'fs';
+import { unlink as fsUnlink, promises } from 'fs';
+const { writeFile: fsWriteFile, readFile: fsReadFile, stat: fsStat } = promises;
 
-export function writeSync(filepath: string, content: string) {
-  return writeFileSync(filepath, content);
+export function writeFile(filepath: string, content: string) {
+  return fsWriteFile(filepath, content);
 }
 
-export function readSync(filepath: string) {
-  return readFileSync(filepath, 'utf-8');
+export function readFile(filepath: string) {
+  return fsReadFile(filepath, 'utf-8');
 }
 
-export function fileExists(filePath: string): boolean {
+export async function fileExists(filePath: string): Promise<boolean> {
   try {
-    return statSync(filePath).isFile();
+    return (await fsStat(filePath)).isFile();
   } catch (err) {
     return false;
   }
 }
 
 export function unlinkFile(filePath: string, cb?: (err?: Error) => any): void {
-  unlink(filePath, cb);
+  fsUnlink(filePath, cb);
 }

packages/graphql-codegen-cli/tests/generate-and-save.spec.ts

@@ -21,7 +21,7 @@ describe('generate-and-save', () => {
 
   test('allow to specify overwrite for specific output (should write file)', async () => {
     const filename = 'overwrite.ts';
-    const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
+    const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
 
     const output = await generate(
       {
@@ -47,10 +47,10 @@ describe('generate-and-save', () => {
 
   test('allow to specify overwrite for specific output (should not write file)', async () => {
     const filename = 'overwrite.ts';
-    const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
+    const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
     // forces file to exist
     const fileExistsSpy = jest.spyOn(fs, 'fileExists');
-    fileExistsSpy.mockImplementation(file => file === filename);
+    fileExistsSpy.mockImplementation(async file => file === filename);
 
     const output = await generate(
       {
@@ -78,7 +78,7 @@ describe('generate-and-save', () => {
 
   test('should use global overwrite option and write a file', async () => {
     const filename = 'overwrite.ts';
-    const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
+    const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
 
     const output = await generate(
       {
@@ -103,10 +103,10 @@ describe('generate-and-save', () => {
 
   test('should use global overwrite option and not write a file', async () => {
     const filename = 'overwrite.ts';
-    const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
+    const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
     // forces file to exist
     const fileExistsSpy = jest.spyOn(fs, 'fileExists');
-    fileExistsSpy.mockImplementation(file => file === filename);
+    fileExistsSpy.mockImplementation(async file => file === filename);
 
     const output = await generate(
       {
@@ -133,12 +133,12 @@ describe('generate-and-save', () => {
 
   test('should overwrite a file by default', async () => {
     const filename = 'overwrite.ts';
-    const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
-    const readSpy = jest.spyOn(fs, 'readSync').mockImplementation();
-    readSpy.mockImplementation(_f => '');
+    const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
+    const readSpy = jest.spyOn(fs, 'readFile').mockImplementation();
+    readSpy.mockImplementation(async _f => '');
     // forces file to exist
     const fileExistsSpy = jest.spyOn(fs, 'fileExists');
-    fileExistsSpy.mockImplementation(file => file === filename);
+    fileExistsSpy.mockImplementation(async file => file === filename);
 
     const output = await generate(
       {
@@ -197,7 +197,7 @@ describe('generate-and-save', () => {
   });
   test('should extract a document from the gql tag (imported from apollo-server)', async () => {
     const filename = 'overwrite.ts';
-    const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
+    const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
 
     const output = await generate(
       {