Skip to content

Commit

Permalink
Formatter scripts for CalAnswers grade data + new grade data (WIP) (#652
Browse files Browse the repository at this point in the history 
)

* Added old semester grade data + FA22, SP23, SU23

* Removed invalid grade data files, added FA22, SP23, and SU23

* created base code for update grades

* grades bug fixes

* Fixed MongooseBulkLoader type error + update-catalog.ts formatting

* lint

---------

Co-authored-by: ARtheboss <[email protected]>
Co-authored-by: Max Wang <[email protected]>
Co-authored-by: maxmwang <[email protected]>
  • Loading branch information
@KartavyaSharma @maxmwang
4 people authored Jul 22, 2024
1 parent 48acfe9 commit fa04f11
Show file tree
Hide file tree
Showing 3 changed files with 246 additions and 143 deletions.
3 changes: 2 additions & 1 deletion backend/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
"coverage": "jest --coverage",
"generate": "graphql-codegen --config codegen.ts",
"lint": "tsc --noEmit && eslint --ext .ts,.tsx .",
"update:catalog": "node ./build/scripts/update-catalog.js"
"update:catalog": "node ./build/scripts/update-catalog.js",
"update:grades": "node ./build/scripts/update-grades.js"
},
"repository": {
"type": "git",
Expand Down
284 changes: 142 additions & 142 deletions backend/src/scripts/update-catalog.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@ import { SectionModel, SectionType } from "../models/section";

const SIS_COURSE_URL = "https://gateway.api.berkeley.edu/sis/v4/courses";
const SIS_CLASS_URL = "https://gateway.api.berkeley.edu/sis/v1/classes";
const SIS_SECTION_URL =
"https://gateway.api.berkeley.edu/sis/v1/classes/sections";
const SIS_SECTION_URL = "https://gateway.api.berkeley.edu/sis/v1/classes/sections";

type StrictOption = boolean | "throw";

const bulkWriteOptions: { strict?: StrictOption } = { strict: "throw" };

const semToTermId = (s: SemesterType) => {
// term-id is computed by dropping the century digit of the year, then adding the term code
Expand All @@ -26,182 +29,179 @@ const semToTermId = (s: SemesterType) => {
};

const queryPages = async <T>(
url: string,
params: any,
headers: any,
field: string,
retries: number = 3
url: string,
params: any,
headers: any,
field: string,
retries = 3,
) => {
let page = 1;
const values: T[] = [];

console.log("Querying SIS API pages...");
console.log(`URL: ${url}`);
console.log(`Params: ${JSON.stringify(params)}`);
console.log(`Headers: ${JSON.stringify(headers)}`);
while (true) {
let resp: AxiosResponse<SISResponse<T>>;

try {
resp = await axios.get(url, {
params: { "page-number": page, ...params },
headers,
});
} catch (err) {
if (axios.isAxiosError(err) && err.response?.status === 404) {
break;
} else {
console.log(`Unexpected err querying SIS API. Error: ${err}.`);

if (retries > 0) {
retries--;
console.log(`Retrying...`);
continue;
} else {
console.log(
`Too many errors querying SIS API for courses. Terminating update...`
);
throw err;
let page = 1;
const values: T[] = [];

console.log("Querying SIS API pages...");
console.log(`URL: ${url}`);
console.log(`Params: ${JSON.stringify(params)}`);
console.log(`Headers: ${JSON.stringify(headers)}`);
while (true) {
let resp: AxiosResponse<SISResponse<T>>;

try {
resp = await axios.get(url, {
params: { "page-number": page, ...params },
headers,
});
} catch (err) {
if (axios.isAxiosError(err) && err.response?.status === 404) {
break;
} else {
console.log(`Unexpected err querying SIS API. Error: ${err}.`);

if (retries > 0) {
retries--;
console.log(`Retrying...`);
continue;
} else {
console.log(
`Too many errors querying SIS API for courses. Terminating update...`,
);
throw err;
}
}
}
}
}

values.push(...resp.data.apiResponse.response[field]);
page++;
}
values.push(...resp.data.apiResponse.response[field]);
page++;
}

console.log(
`Completed querying SIS API. Received ${values.length} objects in ${
page - 1
} pages.`
);
console.log(
`Completed querying SIS API. Received ${values.length} objects in ${page - 1} pages.`,
);

return values;
return values;
};

const updateCourses = async () => {
const headers = {
app_id: config.sis.COURSE_APP_ID,
app_key: config.sis.COURSE_APP_KEY,
};

const params = {
"status-code": "ACTIVE",
"page-size": 100,
};
const headers = {
app_id: config.sis.COURSE_APP_ID,
app_key: config.sis.COURSE_APP_KEY,
};
const params = {
"status-code": "ACTIVE",
"page-size": 100,
};

const courses = await queryPages<CourseType>(
SIS_COURSE_URL,
params,
headers,
"courses"
);
const courses = await queryPages<CourseType>(
SIS_COURSE_URL,
params,
headers,
"courses",
);

console.log("Updating database with new course data...");
console.log("Updating database with new course data...");

const bulkOps = courses.map((c) => ({
replaceOne: {
filter: { classDisplayName: c.classDisplayName },
replacement: c,
upsert: true,
},
}));
const bulkOps = courses.map((c) => ({
replaceOne: {
filter: { classDisplayName: c.classDisplayName },
replacement: c,
upsert: true,
},
}));

const options = {} as MongooseBulkWriteOptions;
const options = bulkWriteOptions as MongooseBulkWriteOptions;

const res = await CourseModel.bulkWrite(bulkOps, options);
const res = await CourseModel.bulkWrite(bulkOps, options);

console.log(
`Completed updating database with new course data. Created ${res.upsertedCount} and updated ${res.modifiedCount} course objects.`
);
console.log(
`Completed updating database with new course data. Created ${res.upsertedCount} and updated ${res.modifiedCount} course objects.`,
);
};

const updateClasses = async () => {
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};

const activeSemesters = await SemesterModel.find({ active: true }).lean();
const classes: ClassType[] = [];
const activeSemesters = await SemesterModel.find({ active: true }).lean();
const classes: ClassType[] = [];

for (const s of activeSemesters) {
console.log(`Updating classses for ${s.term} ${s.year}...`);
for (const s of activeSemesters) {
console.log(`Updating classses for ${s.term} ${s.year}...`);

const params = {
"term-id": semToTermId(s),
"page-size": 100,
};
const params = {
"term-id": semToTermId(s),
"page-size": 100,
};

const semesterClasses = await queryPages<ClassType>(
SIS_CLASS_URL,
params,
headers,
"classes"
);
classes.push(...semesterClasses);
}
const semesterClasses = await queryPages<ClassType>(
SIS_CLASS_URL,
params,
headers,
"classes",
);
classes.push(...semesterClasses);
}

console.log("Updating database with new class data...");
const bulkOps = classes.map((c) => ({
replaceOne: {
filter: { displayName: c.displayName },
replacement: c,
upsert: true,
},
}));
console.log("Updating database with new class data...");
const bulkOps = classes.map((c) => ({
replaceOne: {
filter: { displayName: c.displayName },
replacement: c,
upsert: true,
},
}));

const options = {} as MongooseBulkWriteOptions;
const options = bulkWriteOptions as MongooseBulkWriteOptions;

const res = await ClassModel.bulkWrite(bulkOps, options);
const res = await ClassModel.bulkWrite(bulkOps, options);

console.log(
`Completed updating database with new class data. Created ${res.upsertedCount} and updated ${res.modifiedCount} class objects.`
);
console.log(
`Completed updating database with new class data. Created ${res.upsertedCount} and updated ${res.modifiedCount} class objects.`,
);
};

const updateSections = async () => {
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};

const activeSemesters = await SemesterModel.find({ active: true }).lean();
const sections: SectionType[] = [];
const activeSemesters = await SemesterModel.find({ active: true }).lean();
const sections: SectionType[] = [];

for (const s of activeSemesters) {
console.log(`Updating sections for ${s.term} ${s.year}...`);
for (const s of activeSemesters) {
console.log(`Updating sections for ${s.term} ${s.year}...`);

const params = {
"term-id": semToTermId(s),
"page-size": 100,
};
const params = {
"term-id": semToTermId(s),
"page-size": 100,
};

const semesterClasses = await queryPages<SectionType>(
SIS_SECTION_URL,
params,
headers,
"classSections"
);
sections.push(...semesterClasses);
}
const semesterClasses = await queryPages<SectionType>(
SIS_SECTION_URL,
params,
headers,
"classSections",
);
sections.push(...semesterClasses);
}

console.log("Updating database with new section data...");
const bulkOps = sections.map((s) => ({
replaceOne: {
filter: { displayName: s.displayName },
replacement: s,
upsert: true,
},
}));
console.log("Updating database with new section data...");
const bulkOps = sections.map((s) => ({
replaceOne: {
filter: { displayName: s.displayName },
replacement: s,
upsert: true,
},
}));

const options = {} as MongooseBulkWriteOptions;
const options = bulkWriteOptions as MongooseBulkWriteOptions;

const res = await SectionModel.bulkWrite(bulkOps, options);
const res = await SectionModel.bulkWrite(bulkOps, options);

console.log(
`Completed updating database with new section data. Created ${res.upsertedCount} and updated ${res.modifiedCount} section objects.`
);
console.log(
`Completed updating database with new section data. Created ${res.upsertedCount} and updated ${res.modifiedCount} section objects.`,
);
};

(async () => {
Expand Down
Loading

0 comments on commit fa04f11

Please sign in to comment.