feat(backend): enhance config loader with embedding support and impro…

backend/src/build-system/__tests__/test.model-provider.spec.ts

@@ -0,0 +1,9 @@
+import { EmbeddingProvider } from "src/common/embedding-provider";
+
+describe('Model Provider Test', () => {
+    let embProvider = EmbeddingProvider.getInstance();
+    it('should generate a response from the model provider', async () => {
+        let res = await embProvider.generateEmbResponse("Your text string goes here", "text-embedding-3-small");
+        console.log(res);
+    });
+});

backend/src/common/embedding-provider/index.ts

@@ -0,0 +1,49 @@
+import { HttpService } from "@nestjs/axios";
+import { Logger } from "@nestjs/common";
+import { rejects } from "assert";
+import { resolve } from "path";
+
+export interface ModelProviderConfig {
+    endpoint: string;
+    defaultModel?: string;
+  }
+
+export class EmbeddingProvider {
+    private static instance: EmbeddingProvider | undefined = undefined;
+    private logger = new Logger(EmbeddingProvider.name);
+    constructor(private readonly httpService: HttpService, private readonly config: ModelProviderConfig) {}
+
+    public static getInstance() {
+        if(!this.instance){
+            this.instance = new EmbeddingProvider(new HttpService(), {
+                endpoint: 'http://localhost:3001',
+            });
+        }
+        return this.instance;
+    }
+
+    // return embedding array
+    public async generateEmbResponse(input: string, model: string): Promise<any[] | null>{
+        try{
+            const res = this.httpService.post(`${this.config.endpoint}/embedding`, {
+                content: input,
+                model: model,
+            });
+            return await new Promise((resolve, rejects) => {
+                res.subscribe({
+                next: (value) => {
+                    console.log(value.data.embedding);
+                    resolve(value.data.embedding);
+                },
+                error: (error) => {
+                    this.logger.error(error);
+                    rejects(null);
+                }
+            })
+            })
+        } catch (error) {
+            this.logger.error(error);
+           return null;
+        }
+    }
+}

backend/src/config/common-path.ts

@@ -1,5 +1,6 @@
 import * as path from 'path';
-import { existsSync, mkdirSync, promises } from 'fs-extra';
+import { existsSync, mkdirSync, promises, writeFileSync } from 'fs-extra';
+import { ConfigType } from '@nestjs/config';
 
 // Constants for base directories
 const APP_NAME = 'codefox';
@@ -27,17 +28,30 @@ const ensureDir = (dirPath: string): string => {
 export const getRootDir = (): string => ensureDir(ROOT_DIR);
 
 // Configuration Paths
-export const getConfigDir = (): string =>
-  ensureDir(path.join(getRootDir(), 'config'));
-export const getConfigPath = (configName: string): string =>
-  path.join(getConfigDir(), `${configName}.json`);
+export const getConfigPath = (): string => {
+  const rootPath = ensureDir(path.join(getRootDir()));
+  return path.join(rootPath, 'config.json');
+};
+
+export const getModelStatusPath = (): string => {
+  const rootPath = ensureDir(getRootDir());
+  const modelStatusPath = path.join(rootPath, `model-status.json`);
+  writeFileSync(modelStatusPath, '{}');
+  return modelStatusPath;
+};
 
 // Models Directory
 export const getModelsDir = (): string =>
   ensureDir(path.join(getRootDir(), 'models'));
 export const getModelPath = (modelName: string): string =>
   path.join(getModelsDir(), modelName);
 
+// Embs Directory
+export const getEmbDir = (): string =>
+  ensureDir(path.join(getRootDir(), 'embeddings'));
+export const getEmbPath = (modelName: string): string =>
+  path.join(getModelsDir(), modelName);
+
 // Project-Specific Paths
 export const getProjectsDir = (): string =>
   ensureDir(path.join(getRootDir(), 'projects'));

backend/src/config/config-loader.ts

@@ -2,54 +2,252 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as _ from 'lodash';
 import { getConfigPath } from './common-path';
-export interface ChatConfig {
+import { ConfigType } from 'src/downloader/universal-utils';
+import { Logger } from '@nestjs/common';
+
+export interface ModelConfig {
   model: string;
   endpoint?: string;
   token?: string;
   default?: boolean;
   task?: string;
 }
 
-export class ConfigLoader {
-  private chatsConfig: ChatConfig[];
+export interface EmbeddingConfig {
+  model: string;
+  endpoint?: string;
+  default?: boolean;
+  token?: string;
+}
 
+export interface AppConfig {
+  models?: ModelConfig[];
+  embeddings?: EmbeddingConfig[];
+}
+
+export const exampleConfigContent = `{
+  // Chat models configuration
+  // You can configure multiple chat models
+  "models": [
+    // Example of OpenAI GPT configuration
+    {
+      "model": "gpt-3.5-turbo",
+      "endpoint": "https://api.openai.com/v1",
+      "token": "your-openai-token",  // Replace with your OpenAI token
+      "default": true                // Set as default chat model
+    },
+
+    // Example of local model configuration
+    {
+      "model": "llama2",
+      "endpoint": "http://localhost:11434/v1"
+    }
+  ],
+
+  // Embedding model configuration (optional)
+  "embeddings": [{
+    "model": "text-embedding-ada-002",
+    "endpoint": "https://api.openai.com/v1",
+    "token": "your-openai-token",     // Replace with your OpenAI token
+    "default": true                // Set as default embedding
+  }]
+}`;
+
+export class ConfigLoader {
+  readonly logger = new Logger(ConfigLoader.name);
+  private type: string;
+  private static instances: Map<ConfigType, ConfigLoader> = new Map();
+  private static config: AppConfig;
   private readonly configPath: string;
 
-  constructor() {
-    this.configPath = getConfigPath('config');
+  private constructor(type: ConfigType) {
+    this.type = type;
+    this.configPath = getConfigPath();
+    this.initConfigFile();
+    this.loadConfig();
+  }
+
+  public static getInstance(type: ConfigType): ConfigLoader {
+    if (!ConfigLoader.instances.has(type)) {
+      ConfigLoader.instances.set(type, new ConfigLoader(type));
+    }
+    return ConfigLoader.instances.get(type)!;
+  }
+
+  public initConfigFile(): void {
+    Logger.log('Creating example config file', 'ConfigLoader');
+
+    const config = getConfigPath();
+    if (fs.existsSync(config)) {
+      return;
+    }
+
+    if (!fs.existsSync(config)) {
+      //make file
+      fs.writeFileSync(config, exampleConfigContent, 'utf-8');
+    }
+    Logger.log('Creating example config file', 'ConfigLoader');
+  }
+
+  public reload(): void {
     this.loadConfig();
   }
 
   private loadConfig() {
-    const file = fs.readFileSync(this.configPath, 'utf-8');
+    try {
+      Logger.log(
+        `Loading configuration from ${this.configPath}`,
+        'ConfigLoader',
+      );
+      const file = fs.readFileSync(this.configPath, 'utf-8');
+      const jsonContent = file.replace(
+        /\\"|"(?:\\"|[^"])*"|(\/\/.*|\/\*[\s\S]*?\*\/)/g,
+        (m, g) => (g ? '' : m),
+      );
+      ConfigLoader.config = JSON.parse(jsonContent);
+      this.validateConfig();
+    } catch (error) {
+      if (
+        error.code === 'ENOENT' ||
+        error.message.includes('Unexpected end of JSON input')
+      ) {
+        ConfigLoader.config = {};
+        this.saveConfig();
+      } else {
+        throw error;
+      }
+    }
 
-    this.chatsConfig = JSON.parse(file);
-    console.log('Raw file content:', this.chatsConfig);
+    this.logger.log(ConfigLoader.config);
   }
 
-  get<T>(path: string) {
+  get<T>(path?: string): T {
     if (!path) {
-      return this.chatsConfig as unknown as T;
+      return ConfigLoader.config as unknown as T;
     }
-    return _.get(this.chatsConfig, path) as T;
+    return _.get(ConfigLoader.config, path) as T;
   }
 
   set(path: string, value: any) {
-    _.set(this.chatsConfig, path, value);
+    _.set(ConfigLoader.config, path, value);
     this.saveConfig();
   }
 
   private saveConfig() {
+    const configDir = path.dirname(this.configPath);
+    if (!fs.existsSync(configDir)) {
+      fs.mkdirSync(configDir, { recursive: true });
+    }
     fs.writeFileSync(
       this.configPath,
-      JSON.stringify(this.chatsConfig, null, 4),
+      JSON.stringify(ConfigLoader.config, null, 2),
       'utf-8',
     );
   }
 
+  addConfig(config: ModelConfig | EmbeddingConfig) {
+    if (!ConfigLoader.config[this.type]) {
+      ConfigLoader.config[this.type] = [];
+    }
+    this.logger.log(ConfigLoader.config);
+    const index = ConfigLoader.config[this.type].findIndex(
+      (chat) => chat.model === config.model,
+    );
+    if (index !== -1) {
+      ConfigLoader.config[this.type].splice(index, 1);
+    }
+
+    if (config.default) {
+      ConfigLoader.config.models.forEach((chat) => {
+        chat.default = false;
+      });
+    }
+
+    ConfigLoader.config[this.type].push(config);
+    this.saveConfig();
+  }
+
+  removeConfig(modelName: string): boolean {
+    if (!ConfigLoader.config[this.type]) {
+      return false;
+    }
+
+    const initialLength = ConfigLoader.config[this.type].length;
+    ConfigLoader.config.models = ConfigLoader.config[this.type].filter(
+      (chat) => chat.model !== modelName,
+    );
+
+    if (ConfigLoader.config[this.type].length !== initialLength) {
+      this.saveConfig();
+      return true;
+    }
+
+    return false;
+  }
+
+  getAllConfigs(): EmbeddingConfig[] | ModelConfig[] | null {
+    const res = ConfigLoader.config[this.type];
+    return Array.isArray(res) ? res : null;
+  }
+
   validateConfig() {
-    if (!this.chatsConfig) {
-      throw new Error("Invalid configuration: 'chats' section is missing.");
+    if (!ConfigLoader.config) {
+      ConfigLoader.config = {};
+    }
+
+    if (typeof ConfigLoader.config !== 'object') {
+      throw new Error('Invalid configuration: Must be an object');
+    }
+
+    if (ConfigLoader.config.models) {
+      if (!Array.isArray(ConfigLoader.config.models)) {
+        throw new Error("Invalid configuration: 'chats' must be an array");
+      }
+
+      ConfigLoader.config.models.forEach((chat, index) => {
+        if (!chat.model) {
+          throw new Error(
+            `Invalid chat configuration at index ${index}: 'model' is required`,
+          );
+        }
+      });
+
+      const defaultChats = ConfigLoader.config.models.filter(
+        (chat) => chat.default,
+      );
+      if (defaultChats.length > 1) {
+        throw new Error(
+          'Invalid configuration: Multiple default chat configurations found',
+        );
+      }
     }
+
+    if (ConfigLoader.config[ConfigType.EMBEDDINGS]) {
+      this.logger.log(ConfigLoader.config[ConfigType.EMBEDDINGS]);
+      if (!Array.isArray(ConfigLoader.config[ConfigType.EMBEDDINGS])) {
+        throw new Error("Invalid configuration: 'embeddings' must be an array");
+      }
+
+      ConfigLoader.config.models.forEach((emb, index) => {
+        if (!emb.model) {
+          throw new Error(
+            `Invalid chat configuration at index ${index}: 'model' is required`,
+          );
+        }
+      });
+
+      const defaultChats = ConfigLoader.config[ConfigType.EMBEDDINGS].filter(
+        (chat) => chat.default,
+      );
+      if (defaultChats.length > 1) {
+        throw new Error(
+          'Invalid configuration: Multiple default emb configurations found',
+        );
+      }
+    }
+  }
+
+  getConfig(): AppConfig {
+    return ConfigLoader.config;
   }
 }