feat: updater-scripts

Dockerfile

@@ -15,10 +15,10 @@ COPY --from=datapuller-builder /datapuller/out/package-lock.json ./package-lock.
 RUN ["npm", "install"]
 
 COPY --from=datapuller-builder /datapuller/out/full/ .
-ENTRYPOINT ["turbo", "run", "course", "--filter=datapuller"]
+ENTRYPOINT ["turbo", "run", "runDatapuller", "--filter=datapuller"]
 
 FROM datapuller-dev AS datapuller-prod
-ENTRYPOINT ["turbo", "run", "course", "--filter=datapuller", "--env-mode=loose"]
+ENTRYPOINT ["turbo", "run", "runDatapuller", "--filter=datapuller", "--env-mode=loose"]
 
 # backend
 FROM base AS backend-builder

apps/datapuller/package.json

@@ -3,7 +3,11 @@
   "private": true,
   "scripts": {
     "build": "tsc --noEmit",
-    "course": "tsx src/course.ts"
+    "update:courses": "tsx src/course.ts",
+    "update:sections": "tsx src/section.ts",
+    "update:classes": "tsx src/class.ts",
+    "cleanup:logs": "tsx src/cleanupLogs.ts",
+    "runDatapuller": "tsx src/runDatapuller.ts"
   },
   "devDependencies": {
     "@types/node": "^22.9.1",

apps/datapuller/src/bootstrap/loaders/mongoose.ts

@@ -0,0 +1,34 @@
+import mongoose from "mongoose";
+
+import { Config } from "../../config";
+
+// Close the Mongoose default connection is the event of application termination
+process.on("SIGINT", async () => {
+  await mongoose.connection.close();
+  process.exit(0);
+});
+
+// Your Mongoose setup goes here
+export default async (config: Config): Promise<mongoose.Mongoose> => {
+  // Connect to MongoDB
+  config.log.info("Connecting to MongoDB...");
+  config.log.info("MongoDB URI:", config.mongoDB.uri);
+  const connection = mongoose.connect(config.mongoDB.uri);
+
+  // Log when the connection is established
+  mongoose.connection.on("connected", () => {
+    config.log.info("MongoDB connection established successfully");
+  });
+
+  // Log any errors during the connection
+  mongoose.connection.on("error", (err) => {
+    config.log.error("MongoDB connection error:", err);
+  });
+
+  // Log when the connection is disconnected
+  mongoose.connection.on("disconnected", () => {
+    config.log.info("MongoDB connection disconnected");
+  });
+
+  return connection;
+};

apps/datapuller/src/class.ts

@@ -0,0 +1,69 @@
+import { IClassItem, NewClassModel } from "@repo/common";
+import { ClassesAPI } from "@repo/sis-api/classes";
+
+import { Config } from "./config";
+import setup from "./shared";
+import mapClassToNewClass, { CombinedClass } from "./shared/classParser";
+import { fetchActiveTerms, fetchPaginatedData } from "./shared/utils";
+
+export async function updateClasses(config: Config) {
+  const log = config.log;
+  const classesAPI = new ClassesAPI();
+
+  log.info("Fetching Active Terms");
+  const activeTerms = await fetchActiveTerms(log, {
+    app_id: config.sis.TERM_APP_ID,
+    app_key: config.sis.TERM_APP_KEY,
+  });
+
+  const classes = await fetchPaginatedData<IClassItem, CombinedClass>(
+    log,
+    classesAPI.v1,
+    activeTerms,
+    "getClassesUsingGet",
+    {
+      app_id: config.sis.CLASS_APP_ID,
+      app_key: config.sis.CLASS_APP_KEY,
+    },
+    (data) => data.apiResponse.response.classes || [],
+    mapClassToNewClass,
+    "classes"
+  );
+  log.info(activeTerms);
+
+  log.info("Example Class:", classes[0]);
+
+  await NewClassModel.deleteMany({
+    "session.term.id": { $in: activeTerms },
+  });
+
+  // Insert classes in batches of 5000
+  const insertBatchSize = 5000;
+
+  for (let i = 0; i < classes.length; i += insertBatchSize) {
+    const batch = classes.slice(i, i + insertBatchSize);
+
+    console.log(`Inserting batch ${i / insertBatchSize + 1}...`);
+
+    await NewClassModel.insertMany(batch, { ordered: false });
+  }
+
+  console.log(`Completed updating database with new class data.`);
+
+  log.info(`Updated ${classes.length} classes for active terms`);
+}
+
+const initialize = async () => {
+  const { config } = await setup();
+  try {
+    config.log.info("\n=== UPDATE CLASSES ===");
+    await updateClasses(config);
+  } catch (error) {
+    config.log.error(error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+initialize();

apps/datapuller/src/cleanupLogs.ts

@@ -0,0 +1,43 @@
+import fs from "fs/promises";
+import path from "path";
+import { Logger } from "tslog";
+
+async function cleanupLogs() {
+  const logger = new Logger({ name: "LogCleanup" });
+  const logDir = path.join(__dirname, "logs"); // Adjust the path if necessary
+  const retentionDays = 7; // Number of days to retain logs
+
+  try {
+    logger.info(
+      `Starting log cleanup. Retaining logs from the last ${retentionDays} days.`
+    );
+
+    // Get the current time and calculate the cutoff time
+    const now = Date.now();
+    const cutoffTime = now - retentionDays * 24 * 60 * 60 * 1000;
+
+    // Read the contents of the log directory
+    const files = await fs.readdir(logDir);
+
+    for (const file of files) {
+      // Only process log files that match the naming pattern
+      if (file.startsWith("error_") && file.endsWith(".log")) {
+        const filePath = path.join(logDir, file);
+        const stats = await fs.stat(filePath);
+
+        // Check if the file is older than the retention period
+        if (stats.mtime.getTime() < cutoffTime) {
+          await fs.unlink(filePath);
+          logger.info(`Deleted old log file: ${file}`);
+        }
+      }
+    }
+
+    logger.info("Log cleanup completed successfully.");
+  } catch (error: any) {
+    logger.error(`Log cleanup failed: ${error.message}`);
+    process.exit(1);
+  }
+}
+
+cleanupLogs();

apps/datapuller/src/shared/config.ts → apps/datapuller/src/config.ts

@@ -1,4 +1,5 @@
 import dotenv from "dotenv";
+import { Logger } from "tslog";
 
 // Safely get the environment variable in the process
 const env = (name: string): string => {
@@ -12,6 +13,7 @@ const env = (name: string): string => {
 };
 
 export interface Config {
+  log: Logger<unknown>;
   isDev: boolean;
   mongoDB: {
     uri: string;
@@ -21,13 +23,21 @@ export interface Config {
     CLASS_APP_KEY: string;
     COURSE_APP_ID: string;
     COURSE_APP_KEY: string;
+    TERM_APP_ID: string;
+    TERM_APP_KEY: string;
   };
 }
 
 export function loadConfig(): Config {
   dotenv.config();
 
+  const log = new Logger({
+    type: "pretty",
+    prettyLogTimeZone: "local",
+  });
+
   return {
+    log,
     isDev: env("NODE_ENV") === "development",
     mongoDB: {
       uri: env("MONGODB_URI"),
@@ -37,6 +47,8 @@ export function loadConfig(): Config {
       CLASS_APP_KEY: env("SIS_CLASS_APP_KEY"),
       COURSE_APP_ID: env("SIS_COURSE_APP_ID"),
       COURSE_APP_KEY: env("SIS_COURSE_APP_KEY"),
+      TERM_APP_ID: env("SIS_TERM_APP_ID"),
+      TERM_APP_KEY: env("SIS_TERM_APP_KEY"),
     },
   };
 }

apps/datapuller/src/course.ts

@@ -1,43 +1,60 @@
-import { TermModel } from "@repo/common";
-import { ClassesAPI } from "@repo/sis-api/classes";
+import { ICourseItem, NewCourseModel } from "@repo/common";
 import { CoursesAPI } from "@repo/sis-api/courses";
-import { TermsAPI } from "@repo/sis-api/terms";
 
+import { Config } from "./config";
 import setup from "./shared";
+import mapCourseToNewCourse, { CombinedCourse } from "./shared/courseParser";
+import { fetchPaginatedData } from "./shared/utils";
 
-async function main() {
-  const { log } = setup();
-
-  // Terms API example
-  const termsAPI = new TermsAPI();
+export async function updateCourses(config: Config) {
+  const log = config.log;
+  const coursesAPI = new CoursesAPI();
 
-  await termsAPI.v2.getByTermsUsingGet(
+  const courses = await fetchPaginatedData<ICourseItem, CombinedCourse>(
+    log,
+    coursesAPI.v4,
+    null,
+    "findCourseCollectionUsingGet",
     {
-      "temporal-position": "Current",
+      app_id: config.sis.COURSE_APP_ID,
+      app_key: config.sis.COURSE_APP_KEY,
     },
-    {
-      headers: {
-        app_id: "123",
-        app_key: "abc",
-      },
-    }
+    (data) => data.apiResponse.response.courses || [],
+    mapCourseToNewCourse,
+    "courses"
   );
 
-  // Courses API example
-  const coursesAPI = new CoursesAPI();
+  log.info("Example Course:", courses[0]);
+
+  await NewCourseModel.deleteMany({});
 
-  await coursesAPI.v4.findCourseCollectionUsingGet({
-    "last-updated-since": "2021-01-01",
-  });
+  // Insert courses in batches of 5000
+  const insertBatchSize = 5000;
 
-  // Classes API example
-  const classesAPI = new ClassesAPI();
+  for (let i = 0; i < courses.length; i += insertBatchSize) {
+    const batch = courses.slice(i, i + insertBatchSize);
 
-  await classesAPI.v1.getClassesUsingGet({
-    "term-id": "123",
-  });
+    console.log(`Inserting batch ${i / insertBatchSize + 1}...`);
 
-  log.info(TermModel);
+    await NewCourseModel.insertMany(batch, { ordered: false });
+  }
+
+  console.log(`Completed updating database with new course data.`);
+
+  log.info(`Updated ${courses.length} courses for active terms`);
 }
 
-main();
+const initialize = async () => {
+  const { config } = await setup();
+  try {
+    config.log.info("\n=== UPDATE COURSES ===");
+    await updateCourses(config);
+  } catch (error) {
+    config.log.error(error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+initialize();

apps/datapuller/src/runDatapuller.ts

@@ -0,0 +1,62 @@
+import mongoose from "mongoose";
+
+import { updateClasses } from "./class";
+import { Config } from "./config";
+import { updateCourses } from "./course";
+import { updateSections } from "./section";
+import setup from "./shared";
+
+const testDatabaseWrite = async (config: Config) => {
+  const TestSchema = new mongoose.Schema({
+    testField: String,
+    timestamp: Date,
+  });
+
+  const TestModel = mongoose.model("Test", TestSchema);
+
+  try {
+    const testDocument = new TestModel({
+      testField: "Test write from runDatapuller",
+      timestamp: new Date(),
+    });
+
+    const result = await testDocument.save();
+    config.log.info("Test document written successfully:", result);
+
+    return true;
+  } catch (error) {
+    config.log.error("Error writing to database:", error);
+    return false;
+  }
+};
+
+const runDatapuller = async () => {
+  const { config } = await setup();
+  try {
+    config.log.info("\n=== TESTING DATABASE WRITE ===");
+    const writeSuccessful = await testDatabaseWrite(config);
+    if (!writeSuccessful) {
+      throw new Error(
+        "Failed to write to the database. Please check your connection and permissions."
+      );
+    }
+
+    config.log.info("\n=== UPDATE COURSES ===");
+    await updateCourses(config);
+
+    config.log.info("\n=== UPDATE SECTIONS ===");
+    await updateSections(config);
+
+    config.log.info("\n=== UPDATE CLASSES ===");
+    await updateClasses(config);
+
+    config.log.info("\n=== DATA PULLING COMPLETED ===");
+  } catch (error) {
+    config.log.error(error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+runDatapuller();