From 48d3c5ad5b154b53d882039932311381b87d7db0 Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:18:51 -0700 Subject: [PATCH 1/8] Report unused repos --- src/Sync.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Sync.js b/src/Sync.js index 43e3d24..94f6e62 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -40,6 +40,7 @@ class Snippet { this.ref = ref; this.filePath = filePath; this.lines = []; + this.usedRepos = new Set(); } // fmt creates an array of file lines from the Snippet variables fmt(config) { @@ -210,6 +211,7 @@ class Sync { this.progress.updateOperation("done"); this.progress.stop(); this.logger.info("snipsync operation complete"); + this.reportUnusedRepos(); return; } // clear is the method that will remove snippets from target merge files @@ -286,6 +288,9 @@ class Sync { this.progress.updateTotal(filePaths.length); const extractRootPath = join(rootDir, extractionDir); for (const item of filePaths) { + if (fileSnips.length > 0) { + this.usedRepos.add(`${owner}/${repo}`); + } const ext = determineExtension(item.name); let itemPath = join(item.directory, item.name); if (rtype == "remote") { @@ -449,6 +454,20 @@ class Sync { } return; } + reportUnusedRepos() { + const configRepos = new Set( + this.origins.map((origin) => `${origin.owner}/${origin.repo}`) + ); + const unusedRepos = new Set( + [...configRepos].filter((repo) => !this.usedRepos.has(repo)) + ); + if (unusedRepos.size > 0) { + this.logger.warn("Unused repositories:"); + for (const repo of unusedRepos) { + this.logger.warn(`- ${repo}`); + } + } + } // cleanUp deletes temporary files and folders async cleanUp() { this.progress.updateOperation("cleaning up"); From 8535c2a948ef2b0797bdfd66bee4752a6a4ae0c2 Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:21:04 -0700 Subject: [PATCH 2/8] 2 --- src/Sync.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Sync.js b/src/Sync.js index 94f6e62..ee87abd 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -288,9 +288,7 @@ class Sync { this.progress.updateTotal(filePaths.length); const extractRootPath = join(rootDir, extractionDir); for (const item of filePaths) { - if (fileSnips.length > 0) { - this.usedRepos.add(`${owner}/${repo}`); - } + const ext = determineExtension(item.name); let itemPath = join(item.directory, item.name); if (rtype == "remote") { @@ -313,6 +311,9 @@ class Sync { const snip = new Snippet(id, ext, owner, repo, ref, item); fileSnips.push(snip); } + if (fileSnips.length > 0) { + this.usedRepos.add(`${owner}/${repo}`); + } }); snippets.push(...fileSnips); this.progress.increment(); From 7a15a43f47c87ce48f02e45dadce5b764dbdc280 Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:22:57 -0700 Subject: [PATCH 3/8] modify --- src/Sync.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Sync.js b/src/Sync.js index ee87abd..a5b4ffe 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -227,7 +227,10 @@ class Sync { } // getRepos is the method that downloads all of the Github repos async getRepos() { - const repositories = []; + const reposData = { + repositories: [], + usedRepos: new Set(), + }; this.progress.updateOperation("retrieving source files"); this.progress.updateTotal(this.origins.length); await Promise.all( @@ -237,13 +240,14 @@ class Sync { const filePaths = glob.sync(pattern).map((f) => ({ name: basename(f), directory: dirname(f), })); - repositories.push({ + reposData.repositories.push({ rtype: 'local', owner: origin.files.owner, repo: origin.files.repo, ref: origin.files.ref, filePaths: filePaths, }); + reposData.usedRepos.add(`${origin.files.owner}/${origin.files.repo}`); return; } if (!("owner" in origin && "repo" in origin)) { @@ -256,11 +260,12 @@ class Sync { const buffer = arrayBuffToBuff(byteArray); await writeAsync(fileName, buffer); repository.filePaths = await this.unzip(fileName); - repositories.push(repository); + reposData.repositories.push(repository); + reposData.usedRepos.add(`${owner}/${repo}`); this.progress.increment(); }) ); - return repositories; + return reposData; } // unzip unzips the Github repo archive async unzip(filename) { From c4d765ea6d06f97158f45f079d0685557bf30981 Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:24:52 -0700 Subject: [PATCH 4/8] reports --- src/Sync.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Sync.js b/src/Sync.js index a5b4ffe..182219e 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -40,7 +40,6 @@ class Snippet { this.ref = ref; this.filePath = filePath; this.lines = []; - this.usedRepos = new Set(); } // fmt creates an array of file lines from the Snippet variables fmt(config) { @@ -195,7 +194,7 @@ class Sync { // Download repo as zip file. // Extract to sync_repos directory. // Get repository details and file paths. - const repositories = await this.getRepos(); + const { repositories } = await this.getRepos(); // Search each origin file and scrape the snippets const snippets = await this.extractSnippets(repositories); // Get the infos (name, path) of all the files in the target directories From e64c99da8bc7c39afb58547824c7bddec198c74c Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:26:35 -0700 Subject: [PATCH 5/8] report --- src/Sync.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Sync.js b/src/Sync.js index 182219e..8dd5995 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -194,9 +194,9 @@ class Sync { // Download repo as zip file. // Extract to sync_repos directory. // Get repository details and file paths. - const { repositories } = await this.getRepos(); + const { repositories, usedRepos } = await this.getRepos(); // Search each origin file and scrape the snippets - const snippets = await this.extractSnippets(repositories); + const snippets = await this.extractSnippets(repositories, usedRepos); // Get the infos (name, path) of all the files in the target directories let targetFiles = await this.getTargetFilesInfos(); // Add the lines of each file @@ -283,8 +283,7 @@ class Sync { }); return result.data; } - // extractSnippets returns an array of code snippets that are found in the repositories - async extractSnippets(repositories) { + async extractSnippets(repositories, usedRepos) { const snippets = []; this.progress.updateOperation("extracting snippets"); await Promise.all( @@ -292,7 +291,6 @@ class Sync { this.progress.updateTotal(filePaths.length); const extractRootPath = join(rootDir, extractionDir); for (const item of filePaths) { - const ext = determineExtension(item.name); let itemPath = join(item.directory, item.name); if (rtype == "remote") { @@ -315,10 +313,10 @@ class Sync { const snip = new Snippet(id, ext, owner, repo, ref, item); fileSnips.push(snip); } - if (fileSnips.length > 0) { - this.usedRepos.add(`${owner}/${repo}`); - } }); + if (fileSnips.length > 0) { + usedRepos.add(`${owner}/${repo}`); + } snippets.push(...fileSnips); this.progress.increment(); } From bfde1e79c2dbeba80dd62b59e37de3db84b9ee6f Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:32:59 -0700 Subject: [PATCH 6/8] new --- src/Sync.js | 52 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/src/Sync.js b/src/Sync.js index 8dd5995..90d1e76 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -122,6 +122,7 @@ class Repo { this.repo = repo; this.ref = ref; this.filePaths = []; + this.usedRepos = new Set(); } } // File is the class that contains a filename and lines of the file @@ -206,11 +207,11 @@ class Sync { // Overwrite the files to the target directories await this.writeFiles(splicedFiles); // Delete the sync_repos directory + this.reportUnusedRepos(usedRepos); await this.cleanUp(); this.progress.updateOperation("done"); this.progress.stop(); this.logger.info("snipsync operation complete"); - this.reportUnusedRepos(); return; } // clear is the method that will remove snippets from target merge files @@ -283,6 +284,21 @@ class Sync { }); return result.data; } + async reportUnusedRepos(usedRepos) { + const configRepos = new Set( + this.origins.map((origin) => `${origin.owner}/${origin.repo}`) + ); + const unusedRepos = new Set( + [...configRepos].filter((repo) => !usedRepos.has(repo)) + ); + if (unusedRepos.size > 0) { + this.logger.warn("Unused repositories:"); + for (const repo of unusedRepos) { + this.logger.warn(`- ${repo}`); + } + } + } + // extractSnippets returns an array of code snippets that are found in the repositories async extractSnippets(repositories, usedRepos) { const snippets = []; this.progress.updateOperation("extracting snippets"); @@ -291,6 +307,9 @@ class Sync { this.progress.updateTotal(filePaths.length); const extractRootPath = join(rootDir, extractionDir); for (const item of filePaths) { + if (fileSnips.length > 0) { + usedRepos.add(`${owner}/${repo}`); + } const ext = determineExtension(item.name); let itemPath = join(item.directory, item.name); if (rtype == "remote") { @@ -314,9 +333,6 @@ class Sync { fileSnips.push(snip); } }); - if (fileSnips.length > 0) { - usedRepos.add(`${owner}/${repo}`); - } snippets.push(...fileSnips); this.progress.increment(); } @@ -324,6 +340,20 @@ class Sync { ); return snippets; } + reportUnusedRepos(usedRepos) { + const configRepos = new Set( + this.origins.map((origin) => `${origin.owner}/${origin.repo}`) + ); + const unusedRepos = new Set( + [...configRepos].filter((repo) => !usedRepos.has(repo)) + ); + if (unusedRepos.size > 0) { + this.logger.warn("Unused repositories:"); + for (const repo of unusedRepos) { + this.logger.warn(`- ${repo}`); + } + } +} // getTargetFilesInfos identifies the paths to the target write files async getTargetFilesInfos() { this.progress.updateOperation("gathering information of target files"); @@ -457,20 +487,6 @@ class Sync { } return; } - reportUnusedRepos() { - const configRepos = new Set( - this.origins.map((origin) => `${origin.owner}/${origin.repo}`) - ); - const unusedRepos = new Set( - [...configRepos].filter((repo) => !this.usedRepos.has(repo)) - ); - if (unusedRepos.size > 0) { - this.logger.warn("Unused repositories:"); - for (const repo of unusedRepos) { - this.logger.warn(`- ${repo}`); - } - } - } // cleanUp deletes temporary files and folders async cleanUp() { this.progress.updateOperation("cleaning up"); From b39df74e48fba97adf018c500a50ad7a535d87ed Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:49:14 -0700 Subject: [PATCH 7/8] again --- src/Sync.js | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/Sync.js b/src/Sync.js index 90d1e76..8ab37cf 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -284,7 +284,7 @@ class Sync { }); return result.data; } - async reportUnusedRepos(usedRepos) { + reportUnusedRepos(usedRepos) { const configRepos = new Set( this.origins.map((origin) => `${origin.owner}/${origin.repo}`) ); @@ -298,7 +298,6 @@ class Sync { } } } - // extractSnippets returns an array of code snippets that are found in the repositories async extractSnippets(repositories, usedRepos) { const snippets = []; this.progress.updateOperation("extracting snippets"); @@ -307,9 +306,6 @@ class Sync { this.progress.updateTotal(filePaths.length); const extractRootPath = join(rootDir, extractionDir); for (const item of filePaths) { - if (fileSnips.length > 0) { - usedRepos.add(`${owner}/${repo}`); - } const ext = determineExtension(item.name); let itemPath = join(item.directory, item.name); if (rtype == "remote") { @@ -333,6 +329,9 @@ class Sync { fileSnips.push(snip); } }); + if (fileSnips.length > 0) { + usedRepos.add(`${owner}/${repo}`); + } snippets.push(...fileSnips); this.progress.increment(); } @@ -340,20 +339,10 @@ class Sync { ); return snippets; } - reportUnusedRepos(usedRepos) { - const configRepos = new Set( - this.origins.map((origin) => `${origin.owner}/${origin.repo}`) - ); - const unusedRepos = new Set( - [...configRepos].filter((repo) => !usedRepos.has(repo)) - ); - if (unusedRepos.size > 0) { - this.logger.warn("Unused repositories:"); - for (const repo of unusedRepos) { - this.logger.warn(`- ${repo}`); - } - } -} + + + + // getTargetFilesInfos identifies the paths to the target write files async getTargetFilesInfos() { this.progress.updateOperation("gathering information of target files"); From 1d1b941517f0fc0bf094420de0c3187d3e5c5fc8 Mon Sep 17 00:00:00 2001 From: rachfop Date: Thu, 2 May 2024 08:51:03 -0700 Subject: [PATCH 8/8] print --- src/Sync.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Sync.js b/src/Sync.js index 8ab37cf..5cc6230 100644 --- a/src/Sync.js +++ b/src/Sync.js @@ -207,11 +207,13 @@ class Sync { // Overwrite the files to the target directories await this.writeFiles(splicedFiles); // Delete the sync_repos directory - this.reportUnusedRepos(usedRepos); + await this.cleanUp(); this.progress.updateOperation("done"); this.progress.stop(); this.logger.info("snipsync operation complete"); + this.reportUnusedRepos(usedRepos); + console.log("done"); return; } // clear is the method that will remove snippets from target merge files