feat(backend): enhance config loader with embedding support and impro…

backend/src/config/common-path.ts

@@ -1,10 +1,10 @@
 import * as path from 'path';
-import { existsSync, mkdirSync, promises } from 'fs-extra';
+import { existsSync, mkdirSync, promises, writeFileSync } from 'fs-extra';
 
 // Constants for base directories
 const APP_NAME = 'codefox';
 // TODO: hack way to get the root directory of the workspace
-const WORKSPACE_ROOT = path.resolve(__dirname, '../../../');
+const WORKSPACE_ROOT = path.resolve(__dirname, '../../../../');
 const ROOT_DIR = path.join(WORKSPACE_ROOT, `.${APP_NAME}`);
 
 export const TEMPLATE_PATH = path.join(WORKSPACE_ROOT, 'backend/template');
@@ -27,10 +27,17 @@ const ensureDir = (dirPath: string): string => {
 export const getRootDir = (): string => ensureDir(ROOT_DIR);
 
 // Configuration Paths
-export const getConfigDir = (): string =>
-  ensureDir(path.join(getRootDir(), 'config'));
-export const getConfigPath = (configName: string): string =>
-  path.join(getConfigDir(), `${configName}.json`);
+export const getConfigPath = (): string => {
+  const rootPath = ensureDir(path.join(getRootDir()));
+  return path.join(rootPath, 'config.json');
+};
+
+export const getModelStatusPath = (): string => {
+  const rootPath = ensureDir(getRootDir());
+  const modelStatusPath = path.join(rootPath, 'model-status.json');
+  writeFileSync(modelStatusPath, '{}');
+  return modelStatusPath;
+};
 
 // Models Directory
 export const getModelsDir = (): string =>

backend/src/config/config-loader.ts

@@ -2,6 +2,8 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as _ from 'lodash';
 import { getConfigPath } from './common-path';
+import { Logger } from '@nestjs/common';
+
 export interface ChatConfig {
   model: string;
   endpoint?: string;
@@ -10,46 +12,237 @@ export interface ChatConfig {
   task?: string;
 }
 
-export class ConfigLoader {
-  private chatsConfig: ChatConfig[];
+export interface EmbeddingConfig {
+  model: string;
+  endpoint?: string;
+  token?: string;
+}
+
+export interface AppConfig {
+  chats?: ChatConfig[];
+  embeddings?: EmbeddingConfig;
+}
 
+export const exampleConfigContent = `{
+  // Chat models configuration
+  // You can configure multiple chat models
+  "chats": [
+    // Example of OpenAI GPT configuration
+    {
+      "model": "gpt-3.5-turbo",
+      "endpoint": "https://api.openai.com/v1",
+      "token": "your-openai-token",  // Replace with your OpenAI token
+      "default": true                // Set as default chat model
+    },
+
+    // Example of local model configuration
+    {
+      "model": "llama2",
+      "endpoint": "http://localhost:11434/v1",
+      "task": "chat"
+    }
+  ],
+
+  // Embedding model configuration (optional)
+  "embeddings": {
+    "model": "text-embedding-ada-002",
+    "endpoint": "https://api.openai.com/v1",
+    "token": "your-openai-token"     // Replace with your OpenAI token
+  }
+}`;
+
+export class ConfigLoader {
+  private static instance: ConfigLoader;
+  private config: AppConfig;
   private readonly configPath: string;
 
-  constructor() {
-    this.configPath = getConfigPath('config');
+  private constructor() {
+    this.configPath = getConfigPath();
+    this.initConfigFile();
     this.loadConfig();
   }
 
-  private loadConfig() {
-    const file = fs.readFileSync(this.configPath, 'utf-8');
+  public static getInstance(): ConfigLoader {
+    if (!ConfigLoader.instance) {
+      ConfigLoader.instance = new ConfigLoader();
+    }
+    return ConfigLoader.instance;
+  }
+
+  public initConfigFile(): void {
+    Logger.log('Creating example config file', 'ConfigLoader');
 
-    this.chatsConfig = JSON.parse(file);
-    console.log('Raw file content:', this.chatsConfig);
+    const config = getConfigPath();
+    if (fs.existsSync(config)) {
+      return;
+    }
+
+    if (!fs.existsSync(config)) {
+      //make file
+      fs.writeFileSync(config, exampleConfigContent, 'utf-8');
+    }
+    Logger.log('Creating example config file', 'ConfigLoader');
   }
 
-  get<T>(path: string) {
+  public reload(): void {
+    this.loadConfig();
+  }
+
+  private loadConfig() {
+    try {
+      Logger.log(
+        `Loading configuration from ${this.configPath}`,
+        'ConfigLoader',
+      );
+      const file = fs.readFileSync(this.configPath, 'utf-8');
+      const jsonContent = file.replace(
+        /\\"|"(?:\\"|[^"])*"|(\/\/.*|\/\*[\s\S]*?\*\/)/g,
+        (m, g) => (g ? '' : m),
+      );
+      this.config = JSON.parse(jsonContent);
+      this.validateConfig();
+    } catch (error) {
+      if (
+        error.code === 'ENOENT' ||
+        error.message.includes('Unexpected end of JSON input')
+      ) {
+        this.config = {};
+        this.saveConfig();
+      } else {
+        throw error;
+      }
+    }
+  }
+
+  get<T>(path?: string): T {
     if (!path) {
-      return this.chatsConfig as unknown as T;
+      return this.config as unknown as T;
     }
-    return _.get(this.chatsConfig, path) as T;
+    return _.get(this.config, path) as T;
   }
 
   set(path: string, value: any) {
-    _.set(this.chatsConfig, path, value);
+    _.set(this.config, path, value);
     this.saveConfig();
   }
 
   private saveConfig() {
+    const configDir = path.dirname(this.configPath);
+    if (!fs.existsSync(configDir)) {
+      fs.mkdirSync(configDir, { recursive: true });
+    }
     fs.writeFileSync(
       this.configPath,
-      JSON.stringify(this.chatsConfig, null, 4),
+      JSON.stringify(this.config, null, 2),
       'utf-8',
     );
   }
 
+  getAllChatConfigs(): ChatConfig[] {
+    return this.config.chats || [];
+  }
+
+  getChatConfig(modelName?: string): ChatConfig | null {
+    if (!this.config.chats || !Array.isArray(this.config.chats)) {
+      return null;
+    }
+
+    const chats = this.config.chats;
+
+    if (modelName) {
+      const foundChat = chats.find((chat) => chat.model === modelName);
+      if (foundChat) {
+        return foundChat;
+      }
+    }
+
+    return (
+      chats.find((chat) => chat.default) || (chats.length > 0 ? chats[0] : null)
+    );
+  }
+
+  addChatConfig(config: ChatConfig) {
+    if (!this.config.chats) {
+      this.config.chats = [];
+    }
+
+    const index = this.config.chats.findIndex(
+      (chat) => chat.model === config.model,
+    );
+    if (index !== -1) {
+      this.config.chats.splice(index, 1);
+    }
+
+    if (config.default) {
+      this.config.chats.forEach((chat) => {
+        chat.default = false;
+      });
+    }
+
+    this.config.chats.push(config);
+    this.saveConfig();
+  }
+
+  removeChatConfig(modelName: string): boolean {
+    if (!this.config.chats) {
+      return false;
+    }
+
+    const initialLength = this.config.chats.length;
+    this.config.chats = this.config.chats.filter(
+      (chat) => chat.model !== modelName,
+    );
+
+    if (this.config.chats.length !== initialLength) {
+      this.saveConfig();
+      return true;
+    }
+
+    return false;
+  }
+
+  getEmbeddingConfig(): EmbeddingConfig | null {
+    return this.config.embeddings || null;
+  }
+
   validateConfig() {
-    if (!this.chatsConfig) {
-      throw new Error("Invalid configuration: 'chats' section is missing.");
+    if (!this.config) {
+      this.config = {};
+    }
+
+    if (typeof this.config !== 'object') {
+      throw new Error('Invalid configuration: Must be an object');
     }
+
+    if (this.config.chats) {
+      if (!Array.isArray(this.config.chats)) {
+        throw new Error("Invalid configuration: 'chats' must be an array");
+      }
+
+      this.config.chats.forEach((chat, index) => {
+        if (!chat.model) {
+          throw new Error(
+            `Invalid chat configuration at index ${index}: 'model' is required`,
+          );
+        }
+      });
+
+      const defaultChats = this.config.chats.filter((chat) => chat.default);
+      if (defaultChats.length > 1) {
+        throw new Error(
+          'Invalid configuration: Multiple default chat configurations found',
+        );
+      }
+    }
+
+    if (this.config.embeddings) {
+      if (!this.config.embeddings.model) {
+        throw new Error("Invalid embedding configuration: 'model' is required");
+      }
+    }
+  }
+
+  getConfig(): AppConfig {
+    return this.config;
   }
 }

backend/src/main.ts

@@ -1,7 +1,7 @@
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
 import 'reflect-metadata';
-import { downloadAllModels } from './model/utils';
+import { checkAndDownloadAllModels } from './model/utils';
 
 async function bootstrap() {
   const app = await NestFactory.create(AppModule);
@@ -17,7 +17,7 @@ async function bootstrap() {
       'Access-Control-Allow-Credentials',
     ],
   });
-  await downloadAllModels();
+  await checkAndDownloadAllModels();
   await app.listen(process.env.PORT ?? 3000);
 }
 

backend/src/model/__tests__/loadAllChatsModels.spec.ts

@@ -1,9 +1,9 @@
 import path from 'path';
 import * as fs from 'fs';
 import { ConfigLoader } from '../../config/config-loader';
-import { ModelDownloader } from '../model-downloader';
+import { ModelDownloader } from '../downloader/downloader';
 import { downloadAllModels } from '../utils';
-import { getConfigDir, getConfigPath } from 'src/config/common-path';
+import { getConfigPath, getConfigPath } from 'src/config/common-path';
 
 const originalIsArray = Array.isArray;