Collaboration service real time (#57)

- Operational Transformations in Collaboration Service
- Add utils/shared-ot.ts
  - next.config.js: Enable experimental feature - external dir
  - Some scary warning appears here
- Fix frontend
  - remove debouncer
- Add cursor support, defaults to 0

Current design:

Client --> Server: OT + Cursor
Server --> Client: Full text + Version number + Cursor

Version number incremented by Server, client should not increment
because version number refers to previous version that is edited upon.

Client waits for: 
- Acknowledgement from server of prev request
- Sync version (and text) from server
before they can send the next update request

This ensures no conflicting versions and mix-ups.

frontend/next.config.js

@@ -1,6 +1,9 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   reactStrictMode: true,
-}
+  experimental: {
+    externalDir: true,
+  },
+};
 
-module.exports = nextConfig
+module.exports = nextConfig;

frontend/package.json

@@ -29,13 +29,15 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.0.0",
     "cmdk": "^0.2.0",
+    "diff-match-patch": "^1.0.5",
     "eslint": "8.49.0",
     "eslint-config-next": "13.4.19",
     "firebase": "^10.4.0",
     "lodash": "^4.17.21",
     "lucide-react": "^0.279.0",
     "monaco-editor": "^0.43.0",
     "next": "13.4.19",
+    "ot-text-unicode": "^4.0.0",
     "postcss": "8.4.29",
     "react": "18.2.0",
     "react-dom": "18.2.0",

frontend/src/components/room/code-editor.tsx

@@ -7,7 +7,7 @@ import {
   Settings,
   Play,
 } from "lucide-react";
-import Editor from "@monaco-editor/react";
+import Editor, { OnMount } from "@monaco-editor/react";
 
 import { cn } from "@/lib/utils";
 import { Button } from "@/components/ui/button";
@@ -25,6 +25,7 @@ import {
 } from "@/components/ui/popover";
 import { Card } from "../ui/card";
 import { TypographyBody, TypographyBodyHeavy } from "../ui/typography";
+import { editor } from "monaco-editor";
 
 type CodeEditorProps = {
   theme?: string;
@@ -33,7 +34,9 @@ type CodeEditorProps = {
   defaultValue?: string;
   className?: string;
   text: string;
+  cursor: number;
   onChange: React.Dispatch<React.SetStateAction<string>>;
+  onCursorChange: React.Dispatch<React.SetStateAction<number>>;
 };
 
 const frameworks = [
@@ -66,17 +69,50 @@ export default function CodeEditor({
   defaultValue = "#Write your solution here",
   className,
   text,
+  cursor,
   onChange,
+  onCursorChange,
 }: CodeEditorProps) {
   const [open, setOpen] = React.useState(false);
   const [value, setValue] = React.useState("");
 
+  const [monacoInstance, setMonacoInstance] =
+    React.useState<editor.IStandaloneCodeEditor | null>(null);
+
+  const editorMount: OnMount = (editorL: editor.IStandaloneCodeEditor) => {
+    setMonacoInstance(editorL);
+  };
+
+  const setCursorPosition = React.useCallback(
+    (cursor: number) => {
+      if (!monacoInstance) return;
+
+      const position = monacoInstance.getModel()!.getPositionAt(cursor);
+      monacoInstance.setPosition(position);
+    },
+    [monacoInstance]
+  );
+
+  React.useEffect(() => {
+    if (cursor !== undefined) {
+      setCursorPosition(cursor);
+    }
+  }, [cursor, setCursorPosition]);
+
   const editorOnChange = React.useCallback(
     (value: string | undefined) => {
+      if (!monacoInstance) return;
       if (value === undefined) return;
+
+      if (monacoInstance.getPosition()) {
+        const cursor = monacoInstance
+          .getModel()!
+          .getOffsetAt(monacoInstance.getPosition()!);
+        onCursorChange(cursor);
+      }
       onChange(value);
     },
-    [onChange]
+    [onChange, onCursorChange, monacoInstance]
   );
 
   return (
@@ -142,6 +178,7 @@ export default function CodeEditor({
         value={text}
         theme={theme}
         onChange={(e) => editorOnChange(e)}
+        onMount={editorMount}
       />
       <Card className="flex-1 p-2 mt-2">
         <div className="h-[9vh] p-2">

frontend/src/hooks/useCollaboration.tsx

@@ -1,30 +1,82 @@
 import { useEffect, useState, useRef } from "react";
-import io from "socket.io-client";
+import { io, Socket } from "socket.io-client";
 import { debounce } from "lodash";
+import {
+  TextOperationSetWithCursor,
+  createTextOpFromTexts,
+} from "../../../utils/shared-ot";
+import { TextOp } from "ot-text-unicode";
 
 type UseCollaborationProps = {
   roomId: string;
   userId: string;
 };
 
+enum SocketEvents {
+  ROOM_JOIN = "api/collaboration-service/room/join",
+  ROOM_UPDATE = "api/collaboration-service/room/update",
+  ROOM_SAVE = "api/collaboration-service/room/save",
+  ROOM_LOAD = "api/collaboration-service/room/load",
+}
+
+var vers = 0;
+
 const useCollaboration = ({ roomId, userId }: UseCollaborationProps) => {
-  const [socket, setSocket] = useState<SocketIOClient.Socket | null>(null);
-  const [text, setText] = useState<string>("");
+  const [socket, setSocket] = useState<Socket | null>(null);
+  const [text, setText] = useState<string>("#Write your solution here");
+  const [cursor, setCursor] = useState<number>(
+    "#Write your solution here".length
+  );
   const textRef = useRef<string>(text);
+  const cursorRef = useRef<number>(cursor);
+  const prevCursorRef = useRef<number>(cursor);
+  const prevTextRef = useRef<string>(text);
+  const awaitingAck = useRef<boolean>(false); // ack from sending update
+  const awaitingSync = useRef<boolean>(false); // synced with server
 
   useEffect(() => {
     const socketConnection = io("http://localhost:5003/");
     setSocket(socketConnection);
 
-    socketConnection.emit("/room/join", roomId, userId);
+    socketConnection.emit(SocketEvents.ROOM_JOIN, roomId, userId);
+
+    socketConnection.on(
+      SocketEvents.ROOM_UPDATE,
+      ({
+        version,
+        text,
+        cursor,
+      }: {
+        version: number;
+        text: string;
+        cursor: number | undefined | null;
+      }) => {
+        prevCursorRef.current = cursorRef.current;
+        console.log("prevCursor: " + prevCursorRef.current);
+
+        console.log("cursor: " + cursor);
 
-    // if is my own socket connection, don't update text
-    if (socket && socket.id === socketConnection.id) {
-      console.log("update");
-      socketConnection.on("/room/update", ({ text }: { text: string }) => {
+        console.log("Update vers to " + version);
+        vers = version;
+
+        if (awaitingAck.current) return;
+
+        textRef.current = text;
+        prevTextRef.current = text;
         setText(text);
-      });
-    }
+        if (cursor && cursor > -1) {
+          console.log("Update cursor to " + cursor);
+          cursorRef.current = cursor;
+          setCursor(cursor);
+        } else {
+          cursorRef.current = prevCursorRef.current;
+          cursor = prevCursorRef.current;
+          console.log("Update cursor to " + prevCursorRef.current);
+          setCursor(prevCursorRef.current);
+        }
+        awaitingSync.current = false;
+      }
+    );
 
     return () => {
       socketConnection.disconnect();
@@ -35,19 +87,43 @@ const useCollaboration = ({ roomId, userId }: UseCollaborationProps) => {
     textRef.current = text;
   }, [text]);
 
+  useEffect(() => {
+    cursorRef.current = cursor;
+  }, [cursor]);
+
   useEffect(() => {
     if (!socket) return;
 
-    const handleTextChange = debounce(() => {
-      socket.emit("/room/update", textRef.current);
-    }, 10);
+    if (prevTextRef.current === textRef.current) return;
+
+    if (awaitingAck.current || awaitingSync.current) return;
+
+    awaitingAck.current = true;
+
+    console.log("prevtext: " + prevTextRef.current);
+    console.log("currenttext: " + textRef.current);
+    console.log("version: " + vers);
+    const textOp: TextOp = createTextOpFromTexts(
+      prevTextRef.current,
+      textRef.current
+    );
 
-    handleTextChange();
+    prevTextRef.current = textRef.current;
+
+    console.log(textOp);
+
+    const textOperationSet: TextOperationSetWithCursor = {
+      version: vers,
+      operations: textOp,
+      cursor: cursorRef.current,
+    };
 
-    return () => handleTextChange.cancel();
+    socket.emit(SocketEvents.ROOM_UPDATE, textOperationSet, () => {
+      awaitingAck.current = false;
+    });
   }, [text, socket]);
 
-  return { text, setText };
+  return { text, setText, cursor, setCursor };
 };
 
 export default useCollaboration;

frontend/src/pages/room/[id].tsx

@@ -12,7 +12,7 @@ export default function Room() {
   const roomId = router.query.id as string;
   const userId = "user1";
 
-  const { text, setText } = useCollaboration({
+  const { text, setText, cursor, setCursor } = useCollaboration({
     roomId: roomId as string,
     userId,
   });
@@ -52,7 +52,12 @@ export default function Room() {
           <TabsContent value="solution">{question.solution}</TabsContent>
         </Tabs>
         <div className="flex-1">
-          <CodeEditor text={text} onChange={setText} />
+          <CodeEditor
+            text={text}
+            cursor={cursor}
+            onChange={setText}
+            onCursorChange={setCursor}
+          />
         </div>
       </div>
     </div>

prisma/migrations/20231012153618_add_room/migration.sql

@@ -0,0 +1,13 @@
+-- CreateEnum
+CREATE TYPE "EnumRoomStatus" AS ENUM ('active', 'inactive');
+
+-- CreateTable
+CREATE TABLE "Room" (
+    "room_id" TEXT NOT NULL,
+    "users" TEXT[],
+    "status" "EnumRoomStatus" NOT NULL,
+    "text" TEXT NOT NULL,
+    "saved_text" TEXT,
+
+    CONSTRAINT "Room_pkey" PRIMARY KEY ("room_id")
+);

prisma/schema.prisma

@@ -12,25 +12,38 @@ datasource db {
 
 // todo rename for colalboration service
 model User {
-  id                String      @id @default(uuid())
+  id                String    @id @default(uuid())
   isLookingForMatch Boolean
-  matchedUserId     String?     @unique
+  matchedUserId     String?   @unique
   lastConnected     DateTime?
 }
 
 model Match {
-  roomId    String      @id @default(uuid())
-  userId1    String
-  userId2 String
-  chosenDifficulty String
+  roomId                    String   @id @default(uuid())
+  userId1                   String
+  userId2                   String
+  chosenDifficulty          String
   chosenProgrammingLanguage String
-  createdAt DateTime @default(now())
+  createdAt                 DateTime @default(now())
 }
 
 model AppUser {
-  uid    String  @id
-  displayName  String?
-  photoUrl String?
-  matchDifficulty Int?
+  uid                      String  @id
+  displayName              String?
+  photoUrl                 String?
+  matchDifficulty          Int?
   matchProgrammingLanguage String?
 }
+
+model Room {
+  room_id    String         @id
+  users      String[] // Array of user_id strings
+  status     EnumRoomStatus
+  text       String
+  saved_text String?
+}
+
+enum EnumRoomStatus {
+  active
+  inactive
+}

services/collaboration-service/package.json

@@ -8,17 +8,22 @@
     "start": "yarn run swagger-autogen && ts-node src/app.ts",
     "dev": "yarn run swagger-autogen && nodemon src/app.ts",
     "start:prod": "yarn run build && node ./dist/app.js",
-    "swagger-autogen": "ts-node swagger.ts"
+    "swagger-autogen": "ts-node swagger.ts",
+    "ts-node": "ts-node"
   },
   "dependencies": {
     "@types/express": "^4.17.17",
     "body-parser": "^1.20.2",
     "cookie-parser": "~1.4.4",
     "debug": "~2.6.9",
+    "diff-match-patch": "^1.0.5",
     "express": "~4.16.1",
     "express-openapi": "^12.1.3",
+    "json0-ot-diff": "^1.1.2",
     "morgan": "~1.9.1",
     "openapi": "^1.0.1",
+    "ot-json1": "^1.0.2",
+    "ot-text-unicode": "^4.0.0",
     "socket.io": "^4.7.2",
     "swagger-autogen": "^2.23.5",
     "swagger-express-ts": "^1.1.0",
@@ -29,9 +34,10 @@
   "devDependencies": {
     "@types/cookie-parser": "^1.4.4",
     "@types/cors": "^2.8.14",
+    "@types/diff-match-patch": "^1.0.34",
     "@types/express": "^4.17.17",
     "@types/morgan": "^1.9.5",
-    "@types/node": "^20.6.2",
+    "@types/node": "^20.8.4",
     "@types/socket.io": "^3.0.2",
     "@types/swagger-ui-express": "^4.1.3",
     "@types/uuid": "^9.0.4",