data preprocessing

backend/process_data.ipynb

@@ -0,0 +1,89 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openai import OpenAI\n",
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
+    "client = OpenAI(api_key=OPENAI_API_KEY)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_embedding(text):\n",
+    "    response = client.embeddings.create(\n",
+    "        model=\"text-embedding-3-small\",\n",
+    "        input=text\n",
+    "    )\n",
+    "    return response.data[0].embedding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1206/1206 [04:05<00:00,  4.91it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "with open('202501.json', 'r') as file:\n",
+    "    courses = json.load(file)\n",
+    "\n",
+    "for course in tqdm(courses):\n",
+    "    text_to_embed = f\"{course['short_title']}: {course['description']}\"\n",
+    "    embedding = get_embedding(text_to_embed)    \n",
+    "    course['embedding'] = embedding\n",
+    "\n",
+    "with open('courses_with_embeddings.json', 'w') as file:\n",
+    "    json.dump(courses, file, indent=4)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}