Add prerequisites data from Horaire-cours PDF (#48)

* Refactor CoursePrerequisite model and migrate unstructuredPrerequisite column to ProgramCourse

* Make credits column nullable in Program model

* Refactor logging missing programs and courses

* Add pdf parsable flags to Program model & create a seeder for all parsable programs

* create worker to get parsed horaire pdf data

* quick cleanup & add endpoint

* refactor variable name (CoursePrerequisite to Prerequisite)

* wip adding prerequisites

* add unstructured prerequisites

* add logging rules for worker thread

* cleanup

* add metrics & enforce logging rules

* move prereq logic to service

* add unit tests for session utils

* doc jobs process

* refactor prerequisite

* refactor prerequisite

* run seeder on prerequisite job

prisma/migrations/20241116063843_swap_unstructered_prerequ_column/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `unstructuredPrerequisite` on the `CoursePrerequisite` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "CoursePrerequisite" DROP COLUMN "unstructuredPrerequisite";
+
+-- AlterTable
+ALTER TABLE "ProgramCourse" ADD COLUMN     "unstructuredPrerequisite" TEXT;
+
+-- CreateIndex
+CREATE INDEX "CoursePrerequisite_courseId_programId_idx" ON "CoursePrerequisite"("courseId", "programId");

prisma/migrations/20241116064515_change_credits_column_nullable/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Program" ALTER COLUMN "credits" DROP NOT NULL;

prisma/migrations/20241116232705_add_pdf_parsable_to_program_table/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Program" ADD COLUMN     "pdfParsable" BOOLEAN NOT NULL DEFAULT false;

prisma/migrations/20241117062747_/migration.sql

@@ -0,0 +1,10 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `pdfParsable` on the `Program` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "Program" DROP COLUMN "pdfParsable",
+ADD COLUMN     "isHorairePdfParsable" BOOLEAN NOT NULL DEFAULT false,
+ADD COLUMN     "isPlanificationPdfParsable" BOOLEAN NOT NULL DEFAULT false;

prisma/migrations/20241119040325_add_unique_constraint_to_session/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[year,trimester]` on the table `Session` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE UNIQUE INDEX "Session_year_trimester_key" ON "Session"("year", "trimester");

prisma/migrations/20241120054503_refactor_prerequisite_name/migration.sql

@@ -0,0 +1,34 @@
+/*
+  Warnings:
+
+  - You are about to drop the `CoursePrerequisite` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropForeignKey
+ALTER TABLE "CoursePrerequisite" DROP CONSTRAINT "CoursePrerequisite_courseId_programId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "CoursePrerequisite" DROP CONSTRAINT "CoursePrerequisite_prerequisiteId_programId_fkey";
+
+-- DropTable
+DROP TABLE "CoursePrerequisite";
+
+-- CreateTable
+CREATE TABLE "Prerequisite" (
+    "courseId" INTEGER NOT NULL,
+    "prerequisiteId" INTEGER NOT NULL,
+    "programId" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "Prerequisite_pkey" PRIMARY KEY ("courseId","programId","prerequisiteId")
+);
+
+-- CreateIndex
+CREATE INDEX "Prerequisite_courseId_programId_idx" ON "Prerequisite"("courseId", "programId");
+
+-- AddForeignKey
+ALTER TABLE "Prerequisite" ADD CONSTRAINT "Prerequisite_courseId_programId_fkey" FOREIGN KEY ("courseId", "programId") REFERENCES "ProgramCourse"("courseId", "programId") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Prerequisite" ADD CONSTRAINT "Prerequisite_prerequisiteId_programId_fkey" FOREIGN KEY ("prerequisiteId", "programId") REFERENCES "ProgramCourse"("courseId", "programId") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/schema.prisma

@@ -24,6 +24,8 @@ model Session {
   updatedAt DateTime  @updatedAt
 
   courseInstances CourseInstance[]
+
+  @@unique([year, trimester])
 }
 
 model CourseInstance {
@@ -57,11 +59,10 @@ model Course {
   @@index([code])
 }
 
-model CoursePrerequisite {
-  courseId                 Int
-  prerequisiteId           Int
-  unstructuredPrerequisite String?
-  programId                Int
+model ProgramCoursePrerequisite {
+  courseId       Int
+  prerequisiteId Int
+  programId      Int
 
   programCourse ProgramCourse @relation("CourseToPrerequisites", fields: [courseId, programId], references: [courseId, programId])
   prerequisite  ProgramCourse @relation("PrerequisiteToCourse", fields: [prerequisiteId, programId], references: [courseId, programId])
@@ -70,19 +71,22 @@ model CoursePrerequisite {
   updatedAt DateTime @updatedAt
 
   @@id([courseId, programId, prerequisiteId])
+  @@index([courseId, programId])
+  @@map("Prerequisite")
 }
 
 model ProgramCourse {
-  courseId            Int
-  programId           Int
-  type                String?
-  typicalSessionIndex Int?
+  courseId                 Int
+  programId                Int
+  type                     String?
+  typicalSessionIndex      Int?
+  unstructuredPrerequisite String?
 
   course  Course  @relation(fields: [courseId], references: [id])
   program Program @relation(fields: [programId], references: [id])
 
-  prerequisites        CoursePrerequisite[] @relation("CourseToPrerequisites")
-  prerequisiteToCourse CoursePrerequisite[] @relation("PrerequisiteToCourse")
+  prerequisites        ProgramCoursePrerequisite[] @relation("CourseToPrerequisites")
+  prerequisiteToCourse ProgramCoursePrerequisite[] @relation("PrerequisiteToCourse")
 
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
@@ -94,14 +98,16 @@ model Program {
   id           Int           @id
   code         String?
   title        String
-  credits      String
+  credits      String?
   cycle        Int
   url          String
   programTypes ProgramType[]
 
-  horaireCoursPdfJson  Json?
-  planificationPdfJson Json?
-  courses              ProgramCourse[]
+  isHorairePdfParsable       Boolean         @default(false)
+  isPlanificationPdfParsable Boolean         @default(false)
+  horaireCoursPdfJson        Json?
+  planificationPdfJson       Json?
+  courses                    ProgramCourse[]
 
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt

prisma/seeds/data/programs-to-seed.json

@@ -0,0 +1,20 @@
+{
+    "horairePdfPrograms": [
+        "5766",
+        "7086",
+        "7625",
+        "7694",
+        "7084",
+        "7684",
+        "6556",
+        "6557",
+        "6646",
+        "4567",
+        "4684",
+        "4563",
+        "0486",
+        "4412",
+        "4288",
+        "4329"
+    ]
+}

prisma/seeds/seed.ts

@@ -0,0 +1,19 @@
+import { Logger } from '@nestjs/common';
+
+import { PrismaService } from '../../src/prisma/prisma.service';
+import { seedProgramPdfParserFlags } from '../../src/prisma/programs.seeder';
+
+const prismaService = new PrismaService();
+
+async function main() {
+  await seedProgramPdfParserFlags();
+}
+
+main()
+  .catch((e) => {
+    Logger.error(`Seeding error: ${e}`);
+    process.exit(1);
+  })
+  .finally(async () => {
+    await prismaService.$disconnect();
+  });

src/app.module.ts

@@ -9,11 +9,9 @@ import { PdfModule } from './common/website-helper/pdf/pdf.module';
 import config from './config/configuration';
 import { CourseModule } from './course/course.module';
 import { CourseInstanceModule } from './course-instance/course-instance.module';
-import { CoursePrerequisiteModule } from './course-prerequisite/course-prerequisite.module';
 import { JobsModule } from './jobs/jobs.module';
 import { JobsService } from './jobs/jobs.service';
-import { CoursesJobService } from './jobs/workers/courses.worker';
-import { ProgramsJobService } from './jobs/workers/programs.worker';
+import { PrerequisiteModule } from './prerequisite/prerequisite.module';
 import { PrismaModule } from './prisma/prisma.module';
 import { ProgramModule } from './program/program.module';
 import { ProgramCourseModule } from './program-course/program-course.module';
@@ -35,12 +33,12 @@ import { SessionModule } from './session/session.module';
 
     CourseModule,
     CourseInstanceModule,
-    CoursePrerequisiteModule,
+    PrerequisiteModule,
     SessionModule,
     ProgramModule,
     ProgramCourseModule,
   ],
-  providers: [ProgramsJobService, CoursesJobService, JobsService],
+  providers: [JobsService],
   controllers: [AppController],
   exports: [HttpModule, JobsService],
 })

src/common/utils/prerequisite/prerequisiteUtil.ts

@@ -0,0 +1,36 @@
+import { CourseCodeValidationPipe } from '../../pipes/models/course/course-code-validation-pipe';
+
+export function parsePrerequisiteString(
+  prerequisiteString: string,
+  courseCodeValidationPipe: CourseCodeValidationPipe,
+): string[] | null {
+  const trimmedPrerequisite = prerequisiteString.trim();
+
+  if (!trimmedPrerequisite) {
+    return null;
+  }
+
+  // Attempt to validate the entire string as a single course code
+  const singleValidation =
+    courseCodeValidationPipe.transform(trimmedPrerequisite);
+  if (singleValidation !== false) {
+    return typeof singleValidation === 'string' ? [singleValidation] : null;
+  }
+
+  // If single validation fails, attempt to split and validate multiple course codes
+  const courseCodes = trimmedPrerequisite.split(',').map((s) => s.trim());
+
+  const validCourseCodes: string[] = [];
+  for (const code of courseCodes) {
+    const validatedCode = courseCodeValidationPipe.transform(code);
+    if (validatedCode === false) {
+      // If any code is invalid, treat the entire prerequisite string as unstructured
+      return null;
+    }
+    if (typeof validatedCode === 'string') {
+      validCourseCodes.push(validatedCode);
+    }
+  }
+
+  return validCourseCodes;
+}

src/common/utils/session/sessionUtil.ts

@@ -0,0 +1,105 @@
+import { Logger } from '@nestjs/common';
+import { Trimester } from '@prisma/client';
+
+const logger = new Logger('SessionUtil');
+
+interface SessionDateRange {
+  trimester: Trimester;
+  index: number;
+  start: { month: number; day: number };
+  end: { month: number; day: number };
+}
+
+const SESSION_DATE_RANGES: SessionDateRange[] = [
+  {
+    trimester: Trimester.HIVER,
+    index: 1,
+    start: { month: 1, day: 6 }, // 6 janvier
+    end: { month: 4, day: 28 }, // 28 avril
+  },
+  {
+    trimester: Trimester.ETE,
+    index: 2,
+    start: { month: 5, day: 6 }, // 6 mai
+    end: { month: 8, day: 17 }, // 17 août
+  },
+  {
+    trimester: Trimester.AUTOMNE,
+    index: 3,
+    start: { month: 9, day: 3 }, // 3 septembre
+    end: { month: 12, day: 19 }, // 19 décembre
+  },
+];
+
+export function isDateInRange(
+  date: { month: number; day: number },
+  start: { month: number; day: number },
+  end: { month: number; day: number },
+): boolean {
+  if (date.month < start.month || date.month > end.month) {
+    return false;
+  }
+
+  if (date.month === start.month && date.day < start.day) {
+    return false;
+  }
+
+  if (date.month === end.month && date.day > end.day) {
+    return false;
+  }
+
+  return true;
+}
+
+export function getCurrentSessionIndex(date: Date = new Date()): string | null {
+  const year = date.getFullYear();
+  const month = date.getMonth() + 1; // JS months are 0-based
+  const day = date.getDate();
+
+  for (const session of SESSION_DATE_RANGES) {
+    if (isDateInRange({ month, day }, session.start, session.end)) {
+      return `${year}${session.index}`;
+    }
+  }
+
+  logger.error(
+    `Unable to determine the current trimester for date: ${date.toISOString()}`,
+  );
+  return null;
+}
+
+export function getTrimesterByIndex(index: number): Trimester | null {
+  switch (index) {
+    case 1:
+      return Trimester.HIVER;
+    case 2:
+      return Trimester.ETE;
+    case 3:
+      return Trimester.AUTOMNE;
+    default:
+      return null;
+  }
+}
+
+export function getTrimesterIndexBySession(trimester: string): number {
+  switch (trimester) {
+    case 'HIVER':
+      return 1;
+    case 'ETE':
+      return 2;
+    case 'AUTOMNE':
+      return 3;
+    default:
+      throw new Error(`Unknown trimester: ${trimester}`);
+  }
+}
+
+export function getCurrentTrimester(date: Date = new Date()): Trimester | null {
+  const yearIndex = getCurrentSessionIndex(date);
+  if (!yearIndex) {
+    return null;
+  }
+
+  const trimesterIndex = parseInt(yearIndex.slice(-1), 10);
+  return getTrimesterByIndex(trimesterIndex);
+}

src/common/website-helper/pdf/pdf-parser/horaire/horaire-cours.service.ts

@@ -18,14 +18,13 @@ export class HoraireCoursService {
 
   private readonly logger = new Logger(HoraireCoursService.name);
 
-  public async parsePdfFromUrl(pdfUrl: string) {
+  public async parsePdfFromUrl(pdfUrl: string): Promise<HoraireCours[]> {
     try {
       const response = await firstValueFrom(
         this.httpService.get(pdfUrl, { responseType: 'arraybuffer' }),
       );
 
       this.logger.debug(`Fetched PDF from URL ${pdfUrl}`);
-      this.logger.debug(`Status code: ${response.status}`);
       if (response.status !== HttpStatus.OK) {
         this.logger.error(
           `Failed to fetch PDF from URL ${pdfUrl}. Status code: ${response.status}`,

src/common/website-helper/pdf/pdf.module.ts

@@ -9,5 +9,6 @@ import { PlanificationCoursService } from './pdf-parser/planification/planificat
   imports: [HttpModule],
   providers: [HoraireCoursService, PlanificationCoursService],
   controllers: [PdfController],
+  exports: [HoraireCoursService, PlanificationCoursService],
 })
 export class PdfModule {}