Merge branch 'main' into feat-adding-embedding-config

Sma1lboy · Dec 28, 2024 · 97cae53 · 97cae53
2 parents 1340532 + d071f0c
commit 97cae53
Show file tree

Hide file tree

Showing 86 changed files with 18,955 additions and 9,982 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,4 +15,5 @@ models/
 */**/models/
 
 */**/database.sqlite
-./backend/src/database.sqlite
+./backend/src/database.sqlite
+.codefox
diff --git a/backend/package.json b/backend/package.json
@@ -41,6 +41,7 @@
     "@types/fs-extra": "^11.0.4",
     "@types/normalize-path": "^3.0.2",
     "@types/toposort": "^2.0.7",
+    "toposort": "^2.0.2",
     "axios": "^1.7.7",
     "bcrypt": "^5.1.1",
     "class-validator": "^0.14.1",

diff --git a/backend/src/build-system/__tests__/test-file-create.spec.ts b/backend/src/build-system/__tests__/test-file-create.spec.ts
@@ -1,7 +1,7 @@
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import { FileGeneratorHandler } from '../handlers/file-generate'; // Update with actual file path to the handler
-import * as normalizePath from 'normalize-path';
+import normalizePath from 'normalize-path';
+import { FileGeneratorHandler } from '../handlers/file-manager/file-generate';
 
 describe('FileGeneratorHandler', () => {
   const projectSrcPath = normalizePath(

diff --git a/backend/src/build-system/__tests__/test-generate-doc.spec.ts b/backend/src/build-system/__tests__/test-generate-doc.spec.ts
@@ -1,25 +1,16 @@
 /* eslint-disable no-console */
 import { BuilderContext } from 'src/build-system/context';
 import { BuildSequence } from '../types';
-import { BuildSequenceExecutor } from '../executor';
 import * as fs from 'fs';
 import * as path from 'path';
+import { writeToFile } from './utils';
 
 describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
   // Generate a unique folder with a timestamp
   const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-  const logFolderPath = `./log-${timestamp}`;
+  const logFolderPath = `./logs/generate-docs-${timestamp}`;
   fs.mkdirSync(logFolderPath, { recursive: true });
 
-  // Utility function to extract Markdown content and write to .md files
-  const writeMarkdownToFile = (handlerName: string, data: any) => {
-    // Extract "data" field and remove surrounding Markdown code block formatting
-    const markdownContent = data?.data?.replace(/```/g, '') || '';
-    const filePath = path.join(logFolderPath, `${handlerName}.md`);
-    fs.writeFileSync(filePath, markdownContent, 'utf8');
-    console.log(`Logged ${handlerName} result data to ${filePath}`);
-  };
-
   it('should execute the full sequence and log results to individual files', async () => {
     const sequence: BuildSequence = {
       id: 'test-sequence',
@@ -32,10 +23,8 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
           name: 'Generate PRD',
           nodes: [
             {
-              id: 'op:PRD::STATE:GENERATE',
+              id: 'op:PRD',
               name: 'PRD Generation Node',
-              type: 'ANALYSIS',
-              subType: 'PRD',
             },
           ],
         },
@@ -44,11 +33,9 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
           name: 'Generate UX Sitemap Document',
           nodes: [
             {
-              id: 'op:UXSMD::STATE:GENERATE',
+              id: 'op:UX:SMD',
               name: 'UX Sitemap Document Node',
-              type: 'UX',
-              subType: 'SITEMAP',
-              requires: ['op:PRD::STATE:GENERATE'],
+              requires: ['op:PRD'],
             },
           ],
         },
@@ -57,11 +44,9 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
           name: 'Generate UX Sitemap Structure',
           nodes: [
             {
-              id: 'op:UXSMS::STATE:GENERATE',
+              id: 'op:UX:SMS',
               name: 'UX Sitemap Structure Node',
-              type: 'UX',
-              subType: 'VIEWS',
-              requires: ['op:UXSMD::STATE:GENERATE'],
+              requires: ['op:UX:SMD'],
             },
           ],
         },
@@ -70,9 +55,9 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
           name: 'UX Data Map Document',
           nodes: [
             {
-              id: 'op:UX_DATAMAP::STATE:GENERATE',
+              id: 'op:UX:DATAMAP:DOC',
               name: 'UX Data Map Document node',
-              requires: ['op:UXSMD::STATE:GENERATE'],
+              requires: ['op:UX:SMD'],
             },
           ],
         },
@@ -81,12 +66,12 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
           name: 'file structure generation',
           nodes: [
             {
-              id: 'op:FSTRUCT::STATE:GENERATE',
+              id: 'op:FILE:STRUCT',
               name: 'file structure generation',
-              requires: [
-                'op:UXSMD::STATE:GENERATE',
-                'op:UX_DATAMAP::STATE:GENERATE',
-              ],
+              requires: ['op:UX:SMD', 'op:UX:DATAMAP:DOC'],
+              options: {
+                projectPart: 'frontend',
+              },
             },
           ],
         },
@@ -95,11 +80,12 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
           name: 'File_Arch Document',
           nodes: [
             {
-              id: 'op:FILE_ARCH::STATE:GENERATE',
+              id: 'op:FILE:ARCH',
               name: 'File_Arch',
               requires: [
-                'op:FSTRUCT::STATE:GENERATE',
-                'op:UX_DATAMAP::STATE:GENERATE',
+                'op:FILE:STRUCT',
+                //TODO: here use datamap doc rather than datamap struct, we have to change this
+                'op:UX:DATAMAP:DOC',
               ],
             },
           ],
@@ -110,19 +96,19 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
     const context = new BuilderContext(sequence, 'test');
 
     // Set input data for context
-    context.setData('projectName', 'spotify like music web');
-    context.setData('description', 'user can play music');
-    context.setData('platform', 'web');
+    context.setGlobalContext('projectName', 'spotify like music web');
+    context.setGlobalContext('description', 'user can play music');
+    context.setGlobalContext('platform', 'web');
 
     try {
-      await BuildSequenceExecutor.executeSequence(sequence, context);
+      await context.execute();
 
       for (const step of sequence.steps) {
         for (const node of step.nodes) {
-          const resultData = await context.getResult(node.id);
+          const resultData = await context.getNodeData(node.id);
           console.log(resultData);
           if (resultData) {
-            writeMarkdownToFile(node.name.replace(/ /g, '_'), resultData);
+            writeToFile(logFolderPath, node.id, resultData);
           }
         }
       }

diff --git a/backend/src/build-system/__tests__/test.backend-code-generator.spec.ts b/backend/src/build-system/__tests__/test.backend-code-generator.spec.ts
@@ -1,57 +1,32 @@
 /* eslint-disable no-console */
 import { BuilderContext } from 'src/build-system/context';
-import { BuildResult, BuildSequence } from '../types';
-import { BuildSequenceExecutor } from '../executor';
+import { BuildSequence } from '../types';
 import * as fs from 'fs';
 import * as path from 'path';
+import { writeToFile } from './utils';
 
 describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGenerator', () => {
   // Generate a unique folder with a timestamp
   const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
   const logFolderPath = `./logs/backend_code_generator-${timestamp}`;
   fs.mkdirSync(logFolderPath, { recursive: true });
 
-  /**
-   * Utility function to extract content within <GENERATE> tags and write to .md files.
-   * @param handlerName - The name of the handler/node.
-   * @param data - The data returned by the handler/node.
-   */
-  const writeMarkdownToFile = (handlerName: string, data: BuildResult) => {
-    try {
-      // Extract "data" field and ensure it's a string
-      const content: string = data?.data;
-      if (typeof content !== 'string') {
-        throw new Error(`Invalid data format for handler: ${handlerName}`);
-      }
-
-      const sanitizedHandlerName = handlerName.replace(/[^a-zA-Z0-9_-]/g, '_');
-      const filePath = path.join(logFolderPath, `${sanitizedHandlerName}.md`);
-      fs.writeFileSync(filePath, content, 'utf8');
-      console.log(`Logged ${handlerName} result data to ${filePath}`);
-    } catch (error) {
-      console.error(`Failed to write markdown for ${handlerName}:`, error);
-      throw error;
-    }
-  };
-
   it('should execute the backend code generation sequence and log results to individual files', async () => {
     // Define the build sequence up to Backend Code Generator
     const sequence: BuildSequence = {
       id: 'test-backend-sequence',
       version: '1.0.0',
-      name: 'Test PRD to Backend Code Generation Sequence',
-      description:
-        'Testing sequence execution from PRD to Backend Code Generation',
+      name: 'Spotify-like Music Web',
+      description: 'Users can play music',
+      databaseType: 'SQLite',
       steps: [
         {
           id: 'step-1',
           name: 'Generate PRD',
           nodes: [
             {
-              id: 'op:PRD::STATE:GENERATE',
+              id: 'op:PRD',
               name: 'PRD Generation Node',
-              type: 'ANALYSIS',
-              subType: 'PRD',
             },
           ],
         },
@@ -60,11 +35,9 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
           name: 'Generate UX Sitemap Document',
           nodes: [
             {
-              id: 'op:UXSMD::STATE:GENERATE',
+              id: 'op:UX:SMD',
               name: 'UX Sitemap Document Node',
-              type: 'UX',
-              subType: 'SITEMAP',
-              requires: ['op:PRD::STATE:GENERATE'],
+              requires: ['op:PRD'],
             },
           ],
         },
@@ -73,11 +46,9 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
           name: 'Generate UX Data Map Document',
           nodes: [
             {
-              id: 'op:UX_DATAMAP::STATE:GENERATE',
+              id: 'op:UX:DATAMAP:DOC',
               name: 'UX Data Map Document Node',
-              type: 'UX',
-              subType: 'DATAMAP',
-              requires: ['op:UXSMD::STATE:GENERATE'],
+              requires: ['op:UX:SMD'],
             },
           ],
         },
@@ -86,11 +57,9 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
           name: 'Generate Database Requirements',
           nodes: [
             {
-              id: 'op:DATABASE_REQ::STATE:GENERATE',
+              id: 'op:DATABASE_REQ',
               name: 'Database Requirements Node',
-              type: 'DATABASE',
-              subType: 'SCHEMAS',
-              requires: ['op:UX_DATAMAP::STATE:GENERATE'],
+              requires: ['op:UX:DATAMAP:DOC'],
             },
           ],
         },
@@ -101,9 +70,7 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
             {
               id: 'op:DATABASE:SCHEMAS',
               name: 'Database Schemas Node',
-              type: 'DATABASE',
-              subType: 'SCHEMAS',
-              requires: ['op:DATABASE_REQ::STATE:GENERATE'],
+              requires: ['op:DATABASE_REQ'],
             },
           ],
         },
@@ -112,13 +79,9 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
           name: 'Generate Backend Code',
           nodes: [
             {
-              id: 'op:BACKEND_CODE::STATE:GENERATE',
+              id: 'op:BACKEND:CODE',
               name: 'Backend Code Generator Node',
-              type: 'BACKEND',
-              requires: [
-                'op:DATABASE:SCHEMAS',
-                'op:UX_DATAMAP::STATE:GENERATE',
-              ],
+              requires: ['op:DATABASE:SCHEMAS', 'op:UX:DATAMAP:DOC'],
             },
           ],
         },
@@ -128,30 +91,23 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
     // Initialize the BuilderContext with the defined sequence and environment
     const context = new BuilderContext(sequence, 'test-env');
 
-    // Set input data for context
-    context.setData('projectName', 'Spotify-like Music Web');
-    context.setData('description', 'Users can play music');
-    context.setData('platform', 'web');
-    context.setData('databaseType', 'SQLite'); // Can be 'PostgreSQL', 'MongoDB', etc., based on your needs
-
     try {
       // Execute the build sequence
-      await BuildSequenceExecutor.executeSequence(sequence, context);
+      await context.execute();
 
       // Iterate through each step and node to retrieve and log results
       for (const step of sequence.steps) {
         for (const node of step.nodes) {
-          const resultData = await context.getResult(node.id);
+          const resultData = await context.getNodeData(node.id);
           console.log(`Result for ${node.name}:`, resultData);
 
-          if (resultData && resultData.success) {
-            writeMarkdownToFile(node.name, resultData);
-          } else if (resultData && !resultData.success) {
+          if (resultData) {
+            writeToFile(logFolderPath, node.name, resultData);
+          } else {
             console.error(
               `Handler ${node.name} failed with error:`,
               resultData.error,
             );
-            // Optionally, you can log this to a separate file or handle it as needed
           }
         }
       }

diff --git a/backend/src/build-system/__tests__/test.copy-project-template.spec.ts b/backend/src/build-system/__tests__/test.copy-project-template.spec.ts
@@ -0,0 +1,17 @@
+import { copyProjectTemplate } from 'src/build-system/utils/files';
+import { promises as fs } from 'fs';
+import { v4 as uuidv4 } from 'uuid';
+import { getTemplatePath } from 'src/config/common-path';
+import { Logger } from '@nestjs/common';
+
+describe('Copy Project Template', () => {
+  it('should copy the template to the specified UUID folder', async () => {
+    const templatePath = getTemplatePath('template-backend');
+    const projectUUID = uuidv4();
+
+    Logger.log('template-path:', templatePath);
+    const projectPath = await copyProjectTemplate(templatePath, projectUUID);
+    expect(await fs.access(projectPath)).toBeUndefined(); // Project folder exists
+    await fs.rm(projectPath, { recursive: true, force: true });
+  });
+});
diff --git a/backend/src/build-system/__tests__/test.file-arch.spec.ts b/backend/src/build-system/__tests__/test.file-arch.spec.ts
@@ -1,7 +1,7 @@
 import { BuilderContext } from 'src/build-system/context';
-import { FileArchGenerateHandler } from '../handlers/file-arch';
 import markdownToTxt from 'markdown-to-txt';
 import { readFileSync } from 'fs-extra';
+import { FileArchGenerateHandler } from '../handlers/file-manager/file-arch';
 
 describe('FileArchGenerateHandler', () => {
   it('should generate file architecture document', async () => {
@@ -23,8 +23,9 @@ describe('FileArchGenerateHandler', () => {
     const dataMapStruct = markdownToTxt(
       readFileSync('./datamap-structure.md', 'utf-8'),
     );
-
-    const result = await handler.run(context, fileStructure, dataMapStruct);
+    context.setNodeData('op:FILE:STRUCT', fileStructure);
+    context.setNodeData('op:UX:DATAMAP:DOC', dataMapStruct);
+    const result = await handler.run(context);
     console.log(result);
   }, 30000);
 });