feat(backend): enhance config loader with embedding support and impro…

backend/src/config/common-path.ts

@@ -1,5 +1,6 @@
 import * as path from 'path';
-import { existsSync, mkdirSync, promises } from 'fs-extra';
+import { existsSync, mkdirSync, promises, writeFileSync } from 'fs-extra';
+import { ConfigType } from '@nestjs/config';
 
 // Constants for base directories
 const APP_NAME = 'codefox';
@@ -27,17 +28,30 @@ const ensureDir = (dirPath: string): string => {
 export const getRootDir = (): string => ensureDir(ROOT_DIR);
 
 // Configuration Paths
-export const getConfigDir = (): string =>
-  ensureDir(path.join(getRootDir(), 'config'));
-export const getConfigPath = (configName: string): string =>
-  path.join(getConfigDir(), `${configName}.json`);
+export const getConfigPath = (): string => {
+  const rootPath = ensureDir(path.join(getRootDir()));
+  return path.join(rootPath, 'config.json');
+};
+
+export const getModelStatusPath = (): string => {
+  const rootPath = ensureDir(getRootDir());
+  const modelStatusPath = path.join(rootPath, `model-status.json`);
+  writeFileSync(modelStatusPath, '{}');
+  return modelStatusPath;
+};
 
 // Models Directory
 export const getModelsDir = (): string =>
   ensureDir(path.join(getRootDir(), 'models'));
 export const getModelPath = (modelName: string): string =>
   path.join(getModelsDir(), modelName);
 
+// Embs Directory
+export const getEmbDir = (): string =>
+  ensureDir(path.join(getRootDir(), 'embeddings'));
+export const getEmbPath = (modelName: string): string =>
+  path.join(getModelsDir(), modelName);
+
 // Project-Specific Paths
 export const getProjectsDir = (): string =>
   ensureDir(path.join(getRootDir(), 'projects'));

backend/src/config/config-loader.ts

@@ -2,54 +2,252 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as _ from 'lodash';
 import { getConfigPath } from './common-path';
-export interface ChatConfig {
+import { ConfigType } from 'src/downloader/universal-utils';
+import { Logger } from '@nestjs/common';
+
+export interface ModelConfig {
   model: string;
   endpoint?: string;
   token?: string;
   default?: boolean;
   task?: string;
 }
 
-export class ConfigLoader {
-  private chatsConfig: ChatConfig[];
+export interface EmbeddingConfig {
+  model: string;
+  endpoint?: string;
+  default?: boolean;
+  token?: string;
+}
 
+export interface AppConfig {
+  models?: ModelConfig[];
+  embeddings?: EmbeddingConfig[];
+}
+
+export const exampleConfigContent = `{
+  // Chat models configuration
+  // You can configure multiple chat models
+  "models": [
+    // Example of OpenAI GPT configuration
+    {
+      "model": "gpt-3.5-turbo",
+      "endpoint": "https://api.openai.com/v1",
+      "token": "your-openai-token",  // Replace with your OpenAI token
+      "default": true                // Set as default chat model
+    },
+
+    // Example of local model configuration
+    {
+      "model": "llama2",
+      "endpoint": "http://localhost:11434/v1"
+    }
+  ],
+
+  // Embedding model configuration (optional)
+  "embeddings": [{
+    "model": "text-embedding-ada-002",
+    "endpoint": "https://api.openai.com/v1",
+    "token": "your-openai-token",     // Replace with your OpenAI token
+    "default": true                // Set as default embedding
+  }]
+}`;
+
+export class ConfigLoader {
+  readonly logger = new Logger(ConfigLoader.name);
+  private type: string;
+  private static instances: Map<ConfigType, ConfigLoader> = new Map();
+  private static config: AppConfig;
   private readonly configPath: string;
 
-  constructor() {
-    this.configPath = getConfigPath('config');
+  private constructor(type: ConfigType) {
+    this.type = type;
+    this.configPath = getConfigPath();
+    this.initConfigFile();
+    this.loadConfig();
+  }
+
+  public static getInstance(type: ConfigType): ConfigLoader {
+    if (!ConfigLoader.instances.has(type)) {
+      ConfigLoader.instances.set(type, new ConfigLoader(type));
+    }
+    return ConfigLoader.instances.get(type)!;
+  }
+
+  public initConfigFile(): void {
+    Logger.log('Creating example config file', 'ConfigLoader');
+
+    const config = getConfigPath();
+    if (fs.existsSync(config)) {
+      return;
+    }
+
+    if (!fs.existsSync(config)) {
+      //make file
+      fs.writeFileSync(config, exampleConfigContent, 'utf-8');
+    }
+    Logger.log('Creating example config file', 'ConfigLoader');
+  }
+
+  public reload(): void {
     this.loadConfig();
   }
 
   private loadConfig() {
-    const file = fs.readFileSync(this.configPath, 'utf-8');
+    try {
+      Logger.log(
+        `Loading configuration from ${this.configPath}`,
+        'ConfigLoader',
+      );
+      const file = fs.readFileSync(this.configPath, 'utf-8');
+      const jsonContent = file.replace(
+        /\\"|"(?:\\"|[^"])*"|(\/\/.*|\/\*[\s\S]*?\*\/)/g,
+        (m, g) => (g ? '' : m),
+      );
+      ConfigLoader.config = JSON.parse(jsonContent);
+      this.validateConfig();
+    } catch (error) {
+      if (
+        error.code === 'ENOENT' ||
+        error.message.includes('Unexpected end of JSON input')
+      ) {
+        ConfigLoader.config = {};
+        this.saveConfig();
+      } else {
+        throw error;
+      }
+    }
 
-    this.chatsConfig = JSON.parse(file);
-    console.log('Raw file content:', this.chatsConfig);
+    this.logger.log(ConfigLoader.config);
   }
 
-  get<T>(path: string) {
+  get<T>(path?: string): T {
     if (!path) {
-      return this.chatsConfig as unknown as T;
+      return ConfigLoader.config as unknown as T;
     }
-    return _.get(this.chatsConfig, path) as T;
+    return _.get(ConfigLoader.config, path) as T;
   }
 
   set(path: string, value: any) {
-    _.set(this.chatsConfig, path, value);
+    _.set(ConfigLoader.config, path, value);
     this.saveConfig();
   }
 
   private saveConfig() {
+    const configDir = path.dirname(this.configPath);
+    if (!fs.existsSync(configDir)) {
+      fs.mkdirSync(configDir, { recursive: true });
+    }
     fs.writeFileSync(
       this.configPath,
-      JSON.stringify(this.chatsConfig, null, 4),
+      JSON.stringify(ConfigLoader.config, null, 2),
       'utf-8',
     );
   }
 
+  addConfig(config: ModelConfig | EmbeddingConfig) {
+    if (!ConfigLoader.config[this.type]) {
+      ConfigLoader.config[this.type] = [];
+    }
+    this.logger.log(ConfigLoader.config);
+    const index = ConfigLoader.config[this.type].findIndex(
+      (chat) => chat.model === config.model,
+    );
+    if (index !== -1) {
+      ConfigLoader.config[this.type].splice(index, 1);
+    }
+
+    if (config.default) {
+      ConfigLoader.config.models.forEach((chat) => {
+        chat.default = false;
+      });
+    }
+
+    ConfigLoader.config[this.type].push(config);
+    this.saveConfig();
+  }
+
+  removeConfig(modelName: string): boolean {
+    if (!ConfigLoader.config[this.type]) {
+      return false;
+    }
+
+    const initialLength = ConfigLoader.config[this.type].length;
+    ConfigLoader.config.models = ConfigLoader.config[this.type].filter(
+      (chat) => chat.model !== modelName,
+    );
+
+    if (ConfigLoader.config[this.type].length !== initialLength) {
+      this.saveConfig();
+      return true;
+    }
+
+    return false;
+  }
+
+  getAllConfigs(): EmbeddingConfig[] | ModelConfig[] | null {
+    const res = ConfigLoader.config[this.type];
+    return Array.isArray(res) ? res : null;
+  }
+
   validateConfig() {
-    if (!this.chatsConfig) {
-      throw new Error("Invalid configuration: 'chats' section is missing.");
+    if (!ConfigLoader.config) {
+      ConfigLoader.config = {};
+    }
+
+    if (typeof ConfigLoader.config !== 'object') {
+      throw new Error('Invalid configuration: Must be an object');
+    }
+
+    if (ConfigLoader.config.models) {
+      if (!Array.isArray(ConfigLoader.config.models)) {
+        throw new Error("Invalid configuration: 'chats' must be an array");
+      }
+
+      ConfigLoader.config.models.forEach((chat, index) => {
+        if (!chat.model) {
+          throw new Error(
+            `Invalid chat configuration at index ${index}: 'model' is required`,
+          );
+        }
+      });
+
+      const defaultChats = ConfigLoader.config.models.filter(
+        (chat) => chat.default,
+      );
+      if (defaultChats.length > 1) {
+        throw new Error(
+          'Invalid configuration: Multiple default chat configurations found',
+        );
+      }
     }
+
+    if (ConfigLoader.config[ConfigType.EMBEDDINGS]) {
+      this.logger.log(ConfigLoader.config[ConfigType.EMBEDDINGS]);
+      if (!Array.isArray(ConfigLoader.config[ConfigType.EMBEDDINGS])) {
+        throw new Error("Invalid configuration: 'embeddings' must be an array");
+      }
+
+      ConfigLoader.config.models.forEach((emb, index) => {
+        if (!emb.model) {
+          throw new Error(
+            `Invalid chat configuration at index ${index}: 'model' is required`,
+          );
+        }
+      });
+
+      const defaultChats = ConfigLoader.config[ConfigType.EMBEDDINGS].filter(
+        (chat) => chat.default,
+      );
+      if (defaultChats.length > 1) {
+        throw new Error(
+          'Invalid configuration: Multiple default emb configurations found',
+        );
+      }
+    }
+  }
+
+  getConfig(): AppConfig {
+    return ConfigLoader.config;
   }
 }

backend/src/model/__tests__/loadAllChatsModels.spec.ts → backend/src/downloader/__tests__/loadAllChatsModels.spec.ts

@@ -1,9 +1,12 @@
 import path from 'path';
 import * as fs from 'fs';
-import { ConfigLoader } from '../../config/config-loader';
-import { ModelDownloader } from '../model-downloader';
-import { downloadAllModels } from '../utils';
-import { getConfigDir, getConfigPath } from 'src/config/common-path';
+import {
+  ConfigLoader,
+  ModelConfig,
+  EmbeddingConfig,
+} from '../../config/config-loader';
+import { UniversalDownloader } from '../model-downloader';
+import { ConfigType, downloadAll, TaskType } from '../universal-utils';
 
 const originalIsArray = Array.isArray;
 
@@ -36,27 +39,35 @@ Array.isArray = jest.fn((type: any): type is any[] => {
 // });
 
 describe('loadAllChatsModels with real model loading', () => {
-  let configLoader: ConfigLoader;
+  let modelConfigLoader: ConfigLoader;
+  let embConfigLoader: ConfigLoader;
   beforeAll(async () => {
-    const testConfig = [
-      {
-        model: 'Felladrin/onnx-flan-alpaca-base',
-        task: 'text2text-generation',
-      },
-    ];
-    const configPath = getConfigPath('config');
-    fs.writeFileSync(configPath, JSON.stringify(testConfig, null, 2), 'utf8');
+    modelConfigLoader = ConfigLoader.getInstance(ConfigType.CHATS);
+    embConfigLoader = ConfigLoader.getInstance(ConfigType.EMBEDDINGS);
+    const modelConfig: ModelConfig = {
+      model: 'Xenova/flan-t5-small',
+      endpoint: 'http://localhost:11434/v1',
+      token: 'your-token-here',
+      task: 'text2text-generation',
+    };
+    modelConfigLoader.addConfig(modelConfig);
 
-    configLoader = new ConfigLoader();
-    await downloadAllModels();
-  }, 600000);
+    const embConfig: EmbeddingConfig = {
+      model: 'fast-bge-base-en-v1.5',
+      endpoint: 'http://localhost:11434/v1',
+      token: 'your-token-here',
+    };
+    embConfigLoader.addConfig(embConfig);
+    console.log('preload starts');
+    await downloadAll();
+    console.log('preload successfully');
+  }, 60000000);
 
   it('should load real models specified in config', async () => {
-    const downloader = ModelDownloader.getInstance();
-
+    const downloader = UniversalDownloader.getInstance();
     const chat1Model = await downloader.getLocalModel(
-      'text2text-generation',
-      'Felladrin/onnx-flan-alpaca-base',
+      TaskType.CHAT,
+      'Xenova/flan-t5-small',
     );
     expect(chat1Model).toBeDefined();
     console.log('Loaded Model:', chat1Model);
@@ -82,7 +93,7 @@ describe('loadAllChatsModels with real model loading', () => {
     } catch (error) {
       console.error('Error during model inference:', error);
     }
-  }, 600000);
+  }, 6000000);
 });
 
 afterAll(() => {