From 4856881e57eac5165f8c0ca6be5902f6b9353c0f Mon Sep 17 00:00:00 2001 From: Matthew Rowland Date: Mon, 24 Jun 2024 16:36:11 -0700 Subject: [PATCH] fix: scraping --- backend/src/bootstrap/loaders/apollo.ts | 21 +++++----- backend/src/config.ts | 4 ++ backend/src/modules/catalog/controller.ts | 2 +- backend/src/scripts/update-catalog.ts | 47 ++++++++++++++++------- backend/src/utils/term.ts | 1 + 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/backend/src/bootstrap/loaders/apollo.ts b/backend/src/bootstrap/loaders/apollo.ts index cc92f8c07..db6927932 100644 --- a/backend/src/bootstrap/loaders/apollo.ts +++ b/backend/src/bootstrap/loaders/apollo.ts @@ -1,32 +1,31 @@ import { ApolloServer } from "@apollo/server"; import { buildSchema } from "../graphql/buildSchema"; -import { ApolloServerPluginLandingPageLocalDefault } from '@apollo/server/plugin/landingPage/default'; -import { ApolloServerPluginCacheControl } from '@apollo/server/plugin/cacheControl'; -import { KeyValueCache, KeyValueCacheSetOptions } from '@apollo/utils.keyvaluecache'; -import responseCachePlugin from '@apollo/server-plugin-response-cache'; -import { RedisClientType, createClient } from 'redis'; +import { ApolloServerPluginLandingPageLocalDefault } from "@apollo/server/plugin/landingPage/default"; +import { ApolloServerPluginCacheControl } from "@apollo/server/plugin/cacheControl"; +import { KeyValueCache } from "@apollo/utils.keyvaluecache"; +import responseCachePlugin from "@apollo/server-plugin-response-cache"; +import { RedisClientType } from "redis"; class RedisCache implements KeyValueCache { client: RedisClientType; - prefix: string = 'apollo-cache:'; + prefix = "apollo-cache:"; constructor(client: RedisClientType) { this.client = client; } async get(key: string) { - return await this.client.get(this.prefix + key) ?? undefined; + return (await this.client.get(this.prefix + key)) ?? undefined; } - async set(key: string, value: string, _?: KeyValueCacheSetOptions | undefined) { + async set(key: string, value: string) { // ttl options are intentionally ignored because we will invalidate cache in update script - await this.client.set(this.prefix + key, value) + await this.client.set(this.prefix + key, value); } async delete(key: string) { - return await this.client.del(this.prefix + key) === 1; + return (await this.client.del(this.prefix + key)) === 1; } - } export default async (redis: RedisClientType) => { diff --git a/backend/src/config.ts b/backend/src/config.ts index 92a936763..20222d839 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -26,6 +26,8 @@ export interface Config { CLASS_APP_KEY: string; COURSE_APP_ID: string; COURSE_APP_KEY: string; + TERM_APP_ID: string; + TERM_APP_KEY: string; }; SESSION_SECRET: string; GOOGLE_CLIENT_ID: string; @@ -48,6 +50,8 @@ export const config: Config = { CLASS_APP_KEY: env("SIS_CLASS_APP_KEY"), COURSE_APP_ID: env("SIS_COURSE_APP_ID"), COURSE_APP_KEY: env("SIS_COURSE_APP_KEY"), + TERM_APP_ID: env("SIS_TERM_APP_ID"), + TERM_APP_KEY: env("SIS_TERM_APP_KEY"), }, SESSION_SECRET: env("SESSION_SECRET"), GOOGLE_CLIENT_ID: env("GOOGLE_CLIENT_ID"), diff --git a/backend/src/modules/catalog/controller.ts b/backend/src/modules/catalog/controller.ts index e9247a9d6..eda866fe4 100644 --- a/backend/src/modules/catalog/controller.ts +++ b/backend/src/modules/catalog/controller.ts @@ -1,5 +1,5 @@ import { CatalogItem, TermInput } from "../../generated-types/graphql"; -import { ClassModel, ClassType } from "../../models/class"; +import { ClassModel } from "../../models/class"; import { getTermStartMonth, termToString } from "../../utils/term"; import { GradeModel, GradeType } from "../../models/grade"; import { getAverage } from "../grade/controller"; diff --git a/backend/src/scripts/update-catalog.ts b/backend/src/scripts/update-catalog.ts index 2f46a1499..b1eb38993 100644 --- a/backend/src/scripts/update-catalog.ts +++ b/backend/src/scripts/update-catalog.ts @@ -1,12 +1,12 @@ import mongooseLoader from "../bootstrap/loaders/mongoose"; import { CourseModel, CourseType } from "../models/course"; -import { config } from "../config"; import { SISResponse } from "../utils/sis"; import { ClassModel, ClassType } from "../models/class"; import { SectionModel, SectionType } from "../models/section"; import { URLSearchParams } from "url"; import { TermType, TermModel } from "../models/term"; +import { config } from "../config"; const SIS_COURSE_URL = "https://gateway.api.berkeley.edu/sis/v4/courses"; @@ -15,12 +15,9 @@ const SIS_CLASS_URL = "https://gateway.api.berkeley.edu/sis/v1/classes"; const SIS_SECTION_URL = "https://gateway.api.berkeley.edu/sis/v1/classes/sections"; -const headers = { - app_id: config.sis.COURSE_APP_ID, - app_key: config.sis.COURSE_APP_KEY, -}; - const queryPages = async ( + id: string, + key: string, url: string, field: string, params?: Record @@ -43,7 +40,10 @@ const queryPages = async ( }); const response = await fetch(`${url}${_params}`, { - headers, + headers: { + app_id: id, + app_key: key, + }, }); if (response.status === 404) break; @@ -84,9 +84,15 @@ const queryPages = async ( const updateCourses = async () => { console.log("Updating database with new course data..."); - const courses = await queryPages(SIS_COURSE_URL, "courses", { - "status-code": "ACTIVE", - }); + const courses = await queryPages( + config.sis.COURSE_APP_ID, + config.sis.COURSE_APP_KEY, + SIS_COURSE_URL, + "courses", + { + "status-code": "ACTIVE", + } + ); const operations = courses.map((course) => ({ replaceOne: { @@ -111,9 +117,15 @@ const updateClasses = async (currentTerms: TermType[]) => { for (const term of currentTerms) { console.log(`Updating classses for ${term.name}...`); - const termClasses = await queryPages(SIS_CLASS_URL, "classes", { - "term-id": term.id, - }); + const termClasses = await queryPages( + config.sis.CLASS_APP_ID, + config.sis.CLASS_APP_KEY, + SIS_CLASS_URL, + "classes", + { + "term-id": term.id, + } + ); classes.push(...termClasses); } @@ -144,6 +156,8 @@ const updateSections = async (currentTerms: TermType[]) => { console.log(`Updating sections for ${term.name}...`); const termClasses = await queryPages( + config.sis.CLASS_APP_ID, + config.sis.CLASS_APP_KEY, SIS_SECTION_URL, "classSections", { "term-id": term.id } @@ -174,7 +188,12 @@ const updateSections = async (currentTerms: TermType[]) => { const updateTerms = async () => { console.log("Updating database with new term data..."); - const terms = await queryPages(SIS_COURSE_URL, "terms"); + const terms = await queryPages( + config.sis.TERM_APP_ID, + config.sis.TERM_APP_KEY, + SIS_COURSE_URL, + "terms" + ); const operations = terms.map((term) => ({ replaceOne: { diff --git a/backend/src/utils/term.ts b/backend/src/utils/term.ts index 0e07459ef..dcbeb7721 100644 --- a/backend/src/utils/term.ts +++ b/backend/src/utils/term.ts @@ -23,5 +23,6 @@ export function getTermStartMonth(term: TermInput) { "Summer": `${term.year}-05-31`, } + // @ts-expect-error - We know that the key exists return startDates[term.semester]; }