fix: scraping

backend/src/bootstrap/loaders/apollo.ts

@@ -1,32 +1,31 @@
 import { ApolloServer } from "@apollo/server";
 import { buildSchema } from "../graphql/buildSchema";
-import { ApolloServerPluginLandingPageLocalDefault } from '@apollo/server/plugin/landingPage/default';
-import { ApolloServerPluginCacheControl } from '@apollo/server/plugin/cacheControl';
-import { KeyValueCache, KeyValueCacheSetOptions } from '@apollo/utils.keyvaluecache';
-import responseCachePlugin from '@apollo/server-plugin-response-cache';
-import { RedisClientType, createClient } from 'redis';
+import { ApolloServerPluginLandingPageLocalDefault } from "@apollo/server/plugin/landingPage/default";
+import { ApolloServerPluginCacheControl } from "@apollo/server/plugin/cacheControl";
+import { KeyValueCache } from "@apollo/utils.keyvaluecache";
+import responseCachePlugin from "@apollo/server-plugin-response-cache";
+import { RedisClientType } from "redis";
 
 class RedisCache implements KeyValueCache {
   client: RedisClientType;
-  prefix: string = 'apollo-cache:';
+  prefix = "apollo-cache:";
 
   constructor(client: RedisClientType) {
     this.client = client;
   }
 
   async get(key: string) {
-    return await this.client.get(this.prefix + key) ?? undefined;
+    return (await this.client.get(this.prefix + key)) ?? undefined;
   }
 
-  async set(key: string, value: string, _?: KeyValueCacheSetOptions | undefined) {
+  async set(key: string, value: string) {
     // ttl options are intentionally ignored because we will invalidate cache in update script
-    await this.client.set(this.prefix + key, value) 
+    await this.client.set(this.prefix + key, value);
   }
 
   async delete(key: string) {
-    return await this.client.del(this.prefix + key) === 1;
+    return (await this.client.del(this.prefix + key)) === 1;
   }
-
 }
 
 export default async (redis: RedisClientType) => {

backend/src/config.ts

@@ -26,6 +26,8 @@ export interface Config {
     CLASS_APP_KEY: string;
     COURSE_APP_ID: string;
     COURSE_APP_KEY: string;
+    TERM_APP_ID: string;
+    TERM_APP_KEY: string;
   };
   SESSION_SECRET: string;
   GOOGLE_CLIENT_ID: string;
@@ -48,6 +50,8 @@ export const config: Config = {
     CLASS_APP_KEY: env("SIS_CLASS_APP_KEY"),
     COURSE_APP_ID: env("SIS_COURSE_APP_ID"),
     COURSE_APP_KEY: env("SIS_COURSE_APP_KEY"),
+    TERM_APP_ID: env("SIS_TERM_APP_ID"),
+    TERM_APP_KEY: env("SIS_TERM_APP_KEY"),
   },
   SESSION_SECRET: env("SESSION_SECRET"),
   GOOGLE_CLIENT_ID: env("GOOGLE_CLIENT_ID"),

backend/src/modules/catalog/controller.ts

@@ -1,5 +1,5 @@
 import { CatalogItem, TermInput } from "../../generated-types/graphql";
-import { ClassModel, ClassType } from "../../models/class";
+import { ClassModel } from "../../models/class";
 import { getTermStartMonth, termToString } from "../../utils/term";
 import { GradeModel, GradeType } from "../../models/grade";
 import { getAverage } from "../grade/controller";

backend/src/scripts/update-catalog.ts

@@ -1,12 +1,12 @@
 import mongooseLoader from "../bootstrap/loaders/mongoose";
 import { CourseModel, CourseType } from "../models/course";
-import { config } from "../config";
 
 import { SISResponse } from "../utils/sis";
 import { ClassModel, ClassType } from "../models/class";
 import { SectionModel, SectionType } from "../models/section";
 import { URLSearchParams } from "url";
 import { TermType, TermModel } from "../models/term";
+import { config } from "../config";
 
 const SIS_COURSE_URL = "https://gateway.api.berkeley.edu/sis/v4/courses";
 
@@ -15,12 +15,9 @@ const SIS_CLASS_URL = "https://gateway.api.berkeley.edu/sis/v1/classes";
 const SIS_SECTION_URL =
   "https://gateway.api.berkeley.edu/sis/v1/classes/sections";
 
-const headers = {
-  app_id: config.sis.COURSE_APP_ID,
-  app_key: config.sis.COURSE_APP_KEY,
-};
-
 const queryPages = async <V>(
+  id: string,
+  key: string,
   url: string,
   field: string,
   params?: Record<string, string>
@@ -43,7 +40,10 @@ const queryPages = async <V>(
       });
 
       const response = await fetch(`${url}${_params}`, {
-        headers,
+        headers: {
+          app_id: id,
+          app_key: key,
+        },
       });
 
       if (response.status === 404) break;
@@ -84,9 +84,15 @@ const queryPages = async <V>(
 const updateCourses = async () => {
   console.log("Updating database with new course data...");
 
-  const courses = await queryPages<CourseType>(SIS_COURSE_URL, "courses", {
-    "status-code": "ACTIVE",
-  });
+  const courses = await queryPages<CourseType>(
+    config.sis.COURSE_APP_ID,
+    config.sis.COURSE_APP_KEY,
+    SIS_COURSE_URL,
+    "courses",
+    {
+      "status-code": "ACTIVE",
+    }
+  );
 
   const operations = courses.map((course) => ({
     replaceOne: {
@@ -111,9 +117,15 @@ const updateClasses = async (currentTerms: TermType[]) => {
   for (const term of currentTerms) {
     console.log(`Updating classses for ${term.name}...`);
 
-    const termClasses = await queryPages<ClassType>(SIS_CLASS_URL, "classes", {
-      "term-id": term.id,
-    });
+    const termClasses = await queryPages<ClassType>(
+      config.sis.CLASS_APP_ID,
+      config.sis.CLASS_APP_KEY,
+      SIS_CLASS_URL,
+      "classes",
+      {
+        "term-id": term.id,
+      }
+    );
 
     classes.push(...termClasses);
   }
@@ -144,6 +156,8 @@ const updateSections = async (currentTerms: TermType[]) => {
     console.log(`Updating sections for ${term.name}...`);
 
     const termClasses = await queryPages<SectionType>(
+      config.sis.CLASS_APP_ID,
+      config.sis.CLASS_APP_KEY,
       SIS_SECTION_URL,
       "classSections",
       { "term-id": term.id }
@@ -174,7 +188,12 @@ const updateSections = async (currentTerms: TermType[]) => {
 const updateTerms = async () => {
   console.log("Updating database with new term data...");
 
-  const terms = await queryPages<TermType>(SIS_COURSE_URL, "terms");
+  const terms = await queryPages<TermType>(
+    config.sis.TERM_APP_ID,
+    config.sis.TERM_APP_KEY,
+    SIS_COURSE_URL,
+    "terms"
+  );
 
   const operations = terms.map((term) => ({
     replaceOne: {

backend/src/utils/term.ts

@@ -23,5 +23,6 @@ export function getTermStartMonth(term: TermInput) {
         "Summer": `${term.year}-05-31`,
     }
 
+    // @ts-expect-error - We know that the key exists
     return startDates[term.semester];
 }