feat: updater-scripts

Dockerfile

@@ -15,10 +15,10 @@ COPY --from=datapuller-builder /datapuller/out/package-lock.json ./package-lock.
 RUN ["npm", "install"]
 
 COPY --from=datapuller-builder /datapuller/out/full/ .
-ENTRYPOINT ["turbo", "run", "course", "--filter=datapuller"]
+ENTRYPOINT ["turbo", "run", "section", "--filter=datapuller"]
 
 FROM datapuller-dev AS datapuller-prod
-ENTRYPOINT ["turbo", "run", "course", "--filter=datapuller", "--env-mode=loose"]
+ENTRYPOINT ["turbo", "run", "section", "--filter=datapuller", "--env-mode=loose"]
 
 # backend
 FROM base AS backend-builder

apps/datapuller/package.json

@@ -3,7 +3,7 @@
   "private": true,
   "scripts": {
     "build": "tsc --noEmit",
-    "course": "tsx src/course.ts"
+    "section": "tsx src/section.ts"
   },
   "devDependencies": {
     "@types/node": "^20.14.12",

apps/datapuller/src/section.ts

@@ -0,0 +1,40 @@
+import { ISectionItem } from "@repo/common";
+import { ClassSection, ClassesAPI } from "@repo/sis-api/classes";
+
+import setup from "./shared";
+import mapSectionToNewSection from "./shared/parser";
+import { fetchPaginatedData } from "./shared/utils";
+
+async function updateSections() {
+  const { config, log } = setup();
+  const classesAPI = new ClassesAPI();
+
+  const sections = await fetchPaginatedData<ISectionItem, ClassSection>(
+    classesAPI.v1,
+    "getClassSectionsUsingGet",
+    { "term-id": "2248" },
+    {
+      app_id: config.sis.CLASS_APP_ID,
+      app_key: config.sis.CLASS_APP_KEY,
+    },
+    (data) => data.apiResponse.response.classSections || [],
+    mapSectionToNewSection
+  );
+
+  log.info(`Updated ${sections.length} sections for Spring 2024`);
+}
+
+const initialize = async () => {
+  const { log } = setup();
+  try {
+    log.info("\n=== UPDATE SECTIONS ===");
+    await updateSections();
+  } catch (error) {
+    log.error(error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+initialize();

apps/datapuller/src/shared/config.ts

@@ -21,6 +21,8 @@ export interface Config {
     CLASS_APP_KEY: string;
     COURSE_APP_ID: string;
     COURSE_APP_KEY: string;
+    TERM_APP_ID: string;
+    TERM_APP_KEY: string;
   };
 }
 
@@ -37,6 +39,8 @@ export function loadConfig(): Config {
       CLASS_APP_KEY: env("SIS_CLASS_APP_KEY"),
       COURSE_APP_ID: env("SIS_COURSE_APP_ID"),
       COURSE_APP_KEY: env("SIS_COURSE_APP_KEY"),
+      TERM_APP_ID: env("SIS_TERM_APP_ID"),
+      TERM_APP_KEY: env("SIS_TERM_APP_KEY"),
     },
   };
 }

apps/datapuller/src/shared/index.ts

@@ -3,12 +3,12 @@ import { Logger } from "tslog";
 import { loadConfig } from "./config";
 
 export default function setup() {
-  loadConfig();
+  const config = loadConfig();
 
   const log = new Logger({
     type: "pretty",
     prettyLogTimeZone: "local",
   });
 
-  return { log };
+  return { log, config };
 }

apps/datapuller/src/shared/parser.ts

@@ -0,0 +1,191 @@
+import { ISectionItem } from "@repo/common";
+import { ClassSection } from "@repo/sis-api/classes";
+
+function getRequiredField<T>(
+  value: T | undefined,
+  fieldName: string,
+  defaultValue: T
+): T {
+  if (value === undefined || value === null) {
+    console.warn(`Missing required field: ${fieldName}`);
+    return defaultValue;
+  }
+  return value;
+}
+
+function mapSeatReservation(
+  reservation: any
+): ISectionItem["enrollment"]["reservations"][0] {
+  return {
+    number: getRequiredField(reservation.number, "reservation.number", 0),
+    requirementGroup: getRequiredField(
+      reservation.requirementGroup?.description,
+      "reservation.requirementGroup.description",
+      ""
+    ),
+    maxEnroll: getRequiredField(
+      reservation.maxEnroll,
+      "reservation.maxEnroll",
+      0
+    ),
+    enrolledCount: getRequiredField(
+      reservation.enrolledCount,
+      "reservation.enrolledCount",
+      0
+    ),
+  };
+}
+
+function mapInstructor(
+  instructor: any
+): ISectionItem["meetings"][0]["instructors"][0] {
+  const name =
+    instructor.instructor?.names?.find((n: any) => n.type?.code === "PRF") ||
+    instructor.instructor?.names?.[0];
+
+  return {
+    printInScheduleOfClasses: getRequiredField(
+      instructor.printInScheduleOfClasses,
+      "instructor.printInScheduleOfClasses",
+      false
+    ),
+    familyName: getRequiredField(name?.familyName, "instructor.familyName", ""),
+    givenName: getRequiredField(name?.givenName, "instructor.givenName", ""),
+    role: getRequiredField(instructor.role?.code, "instructor.role.code", ""),
+  };
+}
+
+function mapMeeting(meeting: any): ISectionItem["meetings"][0] {
+  return {
+    number: getRequiredField(meeting.number, "meeting.number", 0),
+    days: [
+      meeting.meetsMonday,
+      meeting.meetsTuesday,
+      meeting.meetsWednesday,
+      meeting.meetsThursday,
+      meeting.meetsFriday,
+      meeting.meetsSaturday,
+      meeting.meetsSunday,
+    ].map((day, index) =>
+      getRequiredField(day, `meeting.day[${index}]`, false)
+    ),
+    startTime: getRequiredField(meeting.startTime, "meeting.startTime", ""),
+    endTime: getRequiredField(meeting.endTime, "meeting.endTime", ""),
+    startDate: getRequiredField(meeting.startDate, "meeting.startDate", ""),
+    endDate: getRequiredField(meeting.endDate, "meeting.endDate", ""),
+    location: getRequiredField(
+      meeting.location?.description,
+      "meeting.location.description",
+      ""
+    ),
+    instructors: meeting.assignedInstructors?.map(mapInstructor) || [],
+  };
+}
+
+export default function mapSectionToNewSection(
+  original: ClassSection
+): ISectionItem {
+  const courseId = getRequiredField(
+    original.class?.course?.identifiers?.find((i) => i.type == "cs-course-id")
+      ?.id,
+    "courseId",
+    ""
+  );
+
+  const newSection: ISectionItem = {
+    courseId,
+    classNumber: getRequiredField(original.class?.number, "classNumber", ""),
+    sessionId: getRequiredField(original.class?.session?.id, "sessionId", ""),
+    termId: getRequiredField(original.class?.session?.term?.id, "termId", ""),
+    sectionId: getRequiredField(original.id, "sectionId", 0),
+    number: getRequiredField(original.number, "number", ""),
+    component: getRequiredField(original.component?.code, "component.code", ""),
+    status: getRequiredField(original.status?.code, "status.code", ""),
+    instructionMode: getRequiredField(
+      original.instructionMode?.code,
+      "instructionMode.code",
+      ""
+    ),
+    printInScheduleOfClasses: getRequiredField(
+      original.printInScheduleOfClasses,
+      "printInScheduleOfClasses",
+      false
+    ),
+    graded: getRequiredField(original.graded, "graded", false),
+    feesExist: getRequiredField(original.feesExist, "feesExist", false),
+    startDate: new Date(getRequiredField(original.startDate, "startDate", "")),
+    endDate: new Date(getRequiredField(original.endDate, "endDate", "")),
+    addConsentRequired: getRequiredField(
+      original.addConsentRequired?.code,
+      "addConsentRequired.code",
+      ""
+    ),
+    dropConsentRequired: getRequiredField(
+      original.dropConsentRequired?.code,
+      "dropConsentRequired.code",
+      ""
+    ),
+    primary: getRequiredField(
+      original.association?.primary,
+      "association.primary",
+      false
+    ),
+    type: getRequiredField(original.type?.code, "type.code", ""),
+    combinedSections: getRequiredField(
+      original.combination?.combinedSections,
+      "combination.combinedSections",
+      []
+    ),
+    enrollment: {
+      status: getRequiredField(
+        original.enrollmentStatus?.status?.code,
+        "enrollmentStatus.status.code",
+        ""
+      ),
+      enrolledCount: getRequiredField(
+        original.enrollmentStatus?.enrolledCount,
+        "enrollmentStatus.enrolledCount",
+        0
+      ),
+      minEnroll: getRequiredField(
+        original.enrollmentStatus?.minEnroll,
+        "enrollmentStatus.minEnroll",
+        0
+      ),
+      maxEnroll: getRequiredField(
+        original.enrollmentStatus?.maxEnroll,
+        "enrollmentStatus.maxEnroll",
+        0
+      ),
+      waitlistedCount: getRequiredField(
+        original.enrollmentStatus?.waitlistedCount,
+        "enrollmentStatus.waitlistedCount",
+        0
+      ),
+      maxWaitlist: getRequiredField(
+        original.enrollmentStatus?.maxWaitlist,
+        "enrollmentStatus.maxWaitlist",
+        0
+      ),
+      reservations:
+        original.enrollmentStatus?.seatReservations?.map(mapSeatReservation) ||
+        [],
+    },
+    exams:
+      original.exams?.map((exam) => ({
+        date: new Date(getRequiredField(exam.date, "exam.date", "")),
+        startTime: getRequiredField(exam.startTime, "exam.startTime", ""),
+        endTime: getRequiredField(exam.endTime, "exam.endTime", ""),
+        location: getRequiredField(
+          exam.location?.description,
+          "exam.location.description",
+          ""
+        ),
+        number: getRequiredField(exam.number, "exam.number", 0),
+        type: getRequiredField(exam.type?.code, "exam.type.code", ""),
+      })) || [],
+    meetings: original.meetings?.map(mapMeeting) || [],
+  };
+
+  return newSection;
+}

apps/datapuller/src/shared/utils.ts

@@ -0,0 +1,60 @@
+export async function fetchPaginatedData<T, R>(
+  api: any,
+  method: string,
+  baseParams: Record<string, any>,
+  headers: Record<string, string>,
+  responseProcessor: (data: any) => R[],
+  itemProcessor: (item: R) => T
+): Promise<T[]> {
+  const results: T[] = [];
+  let page = 1;
+  let retries = 1;
+
+  while (retries > 0) {
+    try {
+      const response = await api[method](
+        {
+          ...baseParams,
+          "page-number": page,
+          "page-size": 100,
+        },
+        { headers }
+      );
+
+      const data = await response.json();
+      const processedData = responseProcessor(data);
+
+      if (processedData.length === 0) {
+        break; // No more data to fetch
+      }
+      console.log("Mapping processedData with itemProcessor...");
+      const transformedData = processedData.map((item, index) => {
+        try {
+          return itemProcessor(item);
+        } catch (error) {
+          console.error(`Error processing item at index ${index}:`, error);
+          console.error("Problematic item:", JSON.stringify(item, null, 2));
+          throw error;
+        }
+      });
+
+      results.push(...transformedData);
+      console.log(`Processed ${processedData.length} items from page ${page}.`);
+      page++;
+      retries = 1; // Reset retries on successful fetch
+    } catch (error) {
+      console.log(`Error fetching page ${page} of data`);
+      console.log(`Unexpected error querying API. Error: "${error}"`);
+
+      if (retries === 0) {
+        console.log(`Too many errors querying API. Terminating update...`);
+        break;
+      }
+
+      retries--;
+      console.log(`Retrying...`);
+    }
+  }
+
+  return results;
+}