feat: updater-scripts

apps/datapuller/package.json

@@ -3,6 +3,8 @@
   "private": true,
   "scripts": {
     "build": "tsc --noEmit",
+    "section": "tsx src/section.ts",
+    "class": "tsx src/class.ts",
     "course": "tsx src/course.ts"
   },
   "devDependencies": {

apps/datapuller/src/class.ts

@@ -0,0 +1,67 @@
+import { ClassModel, IClassItem } from "@repo/common";
+import { ClassesAPI } from "@repo/sis-api/classes";
+
+import setup from "./shared";
+import mapClassToNewClass, { CombinedClass } from "./shared/classParser";
+import { Config } from "./shared/config";
+import { fetchActiveTerms, fetchPaginatedData } from "./shared/utils";
+
+async function updateClasses(config: Config) {
+  const log = config.log;
+  const classesAPI = new ClassesAPI();
+
+  log.info("Fetching Active Terms");
+  const activeTerms = await fetchActiveTerms(log, {
+    app_id: config.sis.TERM_APP_ID,
+    app_key: config.sis.TERM_APP_KEY,
+  });
+
+  log.info(activeTerms);
+
+  const classes = await fetchPaginatedData<IClassItem, CombinedClass>(
+    log,
+    classesAPI.v1,
+    activeTerms,
+    "getClassesUsingGet",
+    {
+      app_id: config.sis.CLASS_APP_ID,
+      app_key: config.sis.CLASS_APP_KEY,
+    },
+    (data) => data.apiResponse.response.classes || [],
+    mapClassToNewClass
+  );
+
+  log.info("Example Class:", classes[0]);
+
+  await ClassModel.deleteMany({});
+
+  // Insert classes in batches of 5000
+  const insertBatchSize = 5000;
+
+  for (let i = 0; i < classes.length; i += insertBatchSize) {
+    const batch = classes.slice(i, i + insertBatchSize);
+
+    console.log(`Inserting batch ${i / insertBatchSize + 1}...`);
+
+    await ClassModel.insertMany(batch, { ordered: false });
+  }
+
+  console.log(`Completed updating database with new class data.`);
+
+  log.info(`Updated ${classes.length} classes for active terms`);
+}
+
+const initialize = async () => {
+  const { config } = setup();
+  try {
+    config.log.info("\n=== UPDATE CLASSES ===");
+    await updateClasses(config);
+  } catch (error) {
+    config.log.error(error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+initialize();

apps/datapuller/src/course.ts

@@ -1,43 +1,59 @@
-import { TermModel } from "@repo/common";
-import { ClassesAPI } from "@repo/sis-api/classes";
+import { CourseModel, ICourseItem } from "@repo/common";
 import { CoursesAPI } from "@repo/sis-api/courses";
-import { TermsAPI } from "@repo/sis-api/terms";
 
 import setup from "./shared";
+import { Config } from "./shared/config";
+import mapCourseToNewCourse, { CombinedCourse } from "./shared/courseParser";
+import { fetchPaginatedData } from "./shared/utils";
 
-async function main() {
-  const { log } = setup();
-
-  // Terms API example
-  const termsAPI = new TermsAPI();
+async function updateCourses(config: Config) {
+  const log = config.log;
+  const coursesAPI = new CoursesAPI();
 
-  await termsAPI.v2.getByTermsUsingGet(
+  const courses = await fetchPaginatedData<ICourseItem, CombinedCourse>(
+    log,
+    coursesAPI.v4,
+    null,
+    "findCourseCollectionUsingGet",
     {
-      "temporal-position": "Current",
+      app_id: config.sis.COURSE_APP_ID,
+      app_key: config.sis.COURSE_APP_KEY,
     },
-    {
-      headers: {
-        app_id: "123",
-        app_key: "abc",
-      },
-    }
+    (data) => data.apiResponse.response.courses || [],
+    mapCourseToNewCourse
   );
 
-  // Courses API example
-  const coursesAPI = new CoursesAPI();
+  log.info("Example Course:", courses[0]);
+
+  await CourseModel.deleteMany({});
 
-  await coursesAPI.v4.findCourseCollectionUsingGet({
-    "last-updated-since": "2021-01-01",
-  });
+  // Insert courses in batches of 5000
+  const insertBatchSize = 5000;
 
-  // Classes API example
-  const classesAPI = new ClassesAPI();
+  for (let i = 0; i < courses.length; i += insertBatchSize) {
+    const batch = courses.slice(i, i + insertBatchSize);
 
-  await classesAPI.v1.getClassesUsingGet({
-    "term-id": "123",
-  });
+    console.log(`Inserting batch ${i / insertBatchSize + 1}...`);
 
-  log.info(TermModel);
+    await CourseModel.insertMany(batch, { ordered: false });
+  }
+
+  console.log(`Completed updating database with new course data.`);
+
+  log.info(`Updated ${courses.length} courses for active terms`);
 }
 
-main();
+const initialize = async () => {
+  const { config } = setup();
+  try {
+    config.log.info("\n=== UPDATE COURSES ===");
+    await updateCourses(config);
+  } catch (error) {
+    config.log.error(error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+initialize();

apps/datapuller/src/section.ts

@@ -0,0 +1,67 @@
+import { ISectionItem, SectionModel } from "@repo/common";
+import { ClassSection, ClassesAPI } from "@repo/sis-api/classes";
+
+import setup from "./shared";
+import { Config } from "./shared/config";
+import mapSectionToNewSection from "./shared/sectionParser";
+import { fetchActiveTerms, fetchPaginatedData } from "./shared/utils";
+
+async function updateSections(config: Config) {
+  const log = config.log;
+  const classesAPI = new ClassesAPI();
+
+  log.info("Fetching Active Terms");
+  const activeTerms = await fetchActiveTerms(log, {
+    app_id: config.sis.TERM_APP_ID,
+    app_key: config.sis.TERM_APP_KEY,
+  });
+
+  log.info(activeTerms);
+
+  const sections = await fetchPaginatedData<ISectionItem, ClassSection>(
+    log,
+    classesAPI.v1,
+    activeTerms,
+    "getClassSectionsUsingGet",
+    {
+      app_id: config.sis.CLASS_APP_ID,
+      app_key: config.sis.CLASS_APP_KEY,
+    },
+    (data) => data.apiResponse.response.classSections || [],
+    mapSectionToNewSection
+  );
+
+  log.info("Example Section:", sections[0]);
+
+  log.info(`Updated ${sections.length} sections for Spring 2024`);
+
+  await SectionModel.deleteMany({});
+
+  // Insert sections in batches of 5000
+  const insertBatchSize = 5000;
+
+  for (let i = 0; i < sections.length; i += insertBatchSize) {
+    const batch = sections.slice(i, i + insertBatchSize);
+
+    console.log(`Inserting batch ${i / insertBatchSize + 1}...`);
+
+    await SectionModel.insertMany(batch, { ordered: false });
+  }
+
+  console.log(`Completed updating database with new section data.`);
+}
+
+const initialize = async () => {
+  const { config } = setup();
+  try {
+    config.log.info("\n=== UPDATE SECTIONS ===");
+    await updateSections(config);
+  } catch (error) {
+    config.log.error(error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+initialize();

apps/datapuller/src/shared/classParser.ts

@@ -0,0 +1,106 @@
+import { IClassItem } from "@repo/common";
+import { Class } from "@repo/sis-api/classes";
+
+import { getRequiredField } from "./utils";
+
+// Include other relevant fields missing in the auto-generated Class type
+export type CombinedClass = Class & {
+  requisites: {
+    code: {
+      type: string;
+    };
+    description: {
+      type: string;
+    };
+    formalDescription: string;
+    active: boolean;
+    fromDate: Date;
+    toDate: Date;
+  };
+};
+
+export default function mapClassToNewClass(
+  original: CombinedClass
+): IClassItem {
+  const courseId = getRequiredField(
+    original.course?.identifiers?.find((i) => i.type == "cs-course-id")?.id,
+    "courseId",
+    ""
+  );
+
+  const newClass: IClassItem = {
+    courseId,
+    termId: getRequiredField(original.session?.term?.id, "session.term.id", ""),
+    sessionId: getRequiredField(original.session?.id, "session.id", ""),
+    number: getRequiredField(original.number, "number", ""),
+    offeringNumber: getRequiredField(
+      original.offeringNumber,
+      "offeringNumber",
+      0
+    ),
+    title: getRequiredField(original.classTitle, "classTitle", ""),
+    description: getRequiredField(
+      original.classDescription,
+      "classDescription",
+      ""
+    ),
+    allowedUnits: {
+      minimum: getRequiredField(
+        original.allowedUnits?.minimum,
+        "allowedUnits.minimum",
+        0
+      ),
+      maximum: getRequiredField(
+        original.allowedUnits?.maximum,
+        "allowedUnits.maximum",
+        0
+      ),
+      forAcademicProgress: getRequiredField(
+        original.allowedUnits?.forAcademicProgress,
+        "allowedUnits.forAcademicProgress",
+        0
+      ),
+      forFinancialAid: getRequiredField(
+        original.allowedUnits?.forFinancialAid,
+        "allowedUnits.forFinancialAid",
+        0
+      ),
+    },
+    gradingBasis: getRequiredField(
+      original.gradingBasis?.code,
+      "gradingBasis.code",
+      ""
+    ),
+    status: getRequiredField(original.status?.code, "status.code", ""),
+    finalExam: getRequiredField(original.finalExam?.code, "finalExam.code", ""),
+    instructionMode: getRequiredField(
+      original.instructionMode?.code,
+      "instructionMode.code",
+      ""
+    ),
+    anyPrintInScheduleOfClasses: getRequiredField(
+      original.anyPrintInScheduleOfClasses,
+      "anyPrintInScheduleOfClasses",
+      false
+    ),
+    contactHours: getRequiredField(original.contactHours, "contactHours", 0),
+    blindGrading: getRequiredField(
+      original.blindGrading,
+      "blindGrading",
+      false
+    ),
+    requirementDesignation: getRequiredField(
+      original.requirementDesignation?.code,
+      "requirementDesignation.code",
+      ""
+    ),
+    // TODO: fix with proper requisites
+    requisites: getRequiredField(
+      original.requisites?.formalDescription,
+      "requisites.formalDescription",
+      ""
+    ),
+  };
+
+  return newClass;
+}

apps/datapuller/src/shared/config.ts

@@ -1,4 +1,5 @@
 import dotenv from "dotenv";
+import { Logger } from "tslog";
 
 // Safely get the environment variable in the process
 const env = (name: string): string => {
@@ -12,6 +13,7 @@ const env = (name: string): string => {
 };
 
 export interface Config {
+  log: Logger<unknown>;
   isDev: boolean;
   mongoDB: {
     uri: string;
@@ -21,13 +23,21 @@ export interface Config {
     CLASS_APP_KEY: string;
     COURSE_APP_ID: string;
     COURSE_APP_KEY: string;
+    TERM_APP_ID: string;
+    TERM_APP_KEY: string;
   };
 }
 
 export function loadConfig(): Config {
   dotenv.config();
 
+  const log = new Logger({
+    type: "pretty",
+    prettyLogTimeZone: "local",
+  });
+
   return {
+    log,
     isDev: env("NODE_ENV") === "development",
     mongoDB: {
       uri: env("MONGODB_URI"),
@@ -37,6 +47,8 @@ export function loadConfig(): Config {
       CLASS_APP_KEY: env("SIS_CLASS_APP_KEY"),
       COURSE_APP_ID: env("SIS_COURSE_APP_ID"),
       COURSE_APP_KEY: env("SIS_COURSE_APP_KEY"),
+      TERM_APP_ID: env("SIS_TERM_APP_ID"),
+      TERM_APP_KEY: env("SIS_TERM_APP_KEY"),
     },
   };
 }