Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Formatter scripts for CalAnswers grade data + new grade data (WIP) #652

Merged
merged 7 commits into from
Jul 22, 2024
Merged

Formatter scripts for CalAnswers grade data + new grade data (WIP) #652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
afe8acc
Added old semester grade data + FA22, SP23, SU23
KartavyaSharma Nov 11, 2023
0877d50
Removed invalid grade data files, added FA22, SP23, and SU23
KartavyaSharma Nov 11, 2023
abc9045
created base code for update grades
Nov 12, 2023
2eebead
grades bug fixes
Apr 24, 2024
849661d
Fixed MongooseBulkLoader type error + update-catalog.ts formatting
KartavyaSharma Jun 5, 2024
5c34c32
Merge branch 'gql' into kurt-gql-grade-data-scripts
maxmwang Jul 22, 2024
e86413b
lint
maxmwang Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion backend/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
"coverage": "jest --coverage",
"generate": "graphql-codegen --config codegen.ts",
"lint": "tsc --noEmit && eslint --ext .ts,.tsx .",
"update:catalog": "node ./build/scripts/update-catalog.js"
"update:catalog": "node ./build/scripts/update-catalog.js",
"update:grades": "node ./build/scripts/update-grades.js"
},
"repository": {
"type": "git",
Expand Down
284 changes: 142 additions & 142 deletions backend/src/scripts/update-catalog.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@ import { SectionModel, SectionType } from "../models/section";

const SIS_COURSE_URL = "https://gateway.api.berkeley.edu/sis/v4/courses";
const SIS_CLASS_URL = "https://gateway.api.berkeley.edu/sis/v1/classes";
const SIS_SECTION_URL =
"https://gateway.api.berkeley.edu/sis/v1/classes/sections";
const SIS_SECTION_URL = "https://gateway.api.berkeley.edu/sis/v1/classes/sections";

type StrictOption = boolean | "throw";

const bulkWriteOptions: { strict?: StrictOption } = { strict: "throw" };

const semToTermId = (s: SemesterType) => {
// term-id is computed by dropping the century digit of the year, then adding the term code
Expand All @@ -26,182 +29,179 @@ const semToTermId = (s: SemesterType) => {
};

const queryPages = async <T>(
url: string,
params: any,
headers: any,
field: string,
retries: number = 3
url: string,
params: any,
headers: any,
field: string,
retries = 3,
) => {
let page = 1;
const values: T[] = [];

console.log("Querying SIS API pages...");
console.log(`URL: ${url}`);
console.log(`Params: ${JSON.stringify(params)}`);
console.log(`Headers: ${JSON.stringify(headers)}`);
while (true) {
let resp: AxiosResponse<SISResponse<T>>;

try {
resp = await axios.get(url, {
params: { "page-number": page, ...params },
headers,
});
} catch (err) {
if (axios.isAxiosError(err) && err.response?.status === 404) {
break;
} else {
console.log(`Unexpected err querying SIS API. Error: ${err}.`);

if (retries > 0) {
retries--;
console.log(`Retrying...`);
continue;
} else {
console.log(
`Too many errors querying SIS API for courses. Terminating update...`
);
throw err;
let page = 1;
const values: T[] = [];

console.log("Querying SIS API pages...");
console.log(`URL: ${url}`);
console.log(`Params: ${JSON.stringify(params)}`);
console.log(`Headers: ${JSON.stringify(headers)}`);
while (true) {
let resp: AxiosResponse<SISResponse<T>>;

try {
resp = await axios.get(url, {
params: { "page-number": page, ...params },
headers,
});
} catch (err) {
if (axios.isAxiosError(err) && err.response?.status === 404) {
break;
} else {
console.log(`Unexpected err querying SIS API. Error: ${err}.`);

if (retries > 0) {
retries--;
console.log(`Retrying...`);
continue;
} else {
console.log(
`Too many errors querying SIS API for courses. Terminating update...`,
);
throw err;
}
}
}
}
}

values.push(...resp.data.apiResponse.response[field]);
page++;
}
values.push(...resp.data.apiResponse.response[field]);
page++;
}

console.log(
`Completed querying SIS API. Received ${values.length} objects in ${
page - 1
} pages.`
);
console.log(
`Completed querying SIS API. Received ${values.length} objects in ${page - 1} pages.`,
);

return values;
return values;
};

const updateCourses = async () => {
const headers = {
app_id: config.sis.COURSE_APP_ID,
app_key: config.sis.COURSE_APP_KEY,
};

const params = {
"status-code": "ACTIVE",
"page-size": 100,
};
const headers = {
app_id: config.sis.COURSE_APP_ID,
app_key: config.sis.COURSE_APP_KEY,
};
const params = {
"status-code": "ACTIVE",
"page-size": 100,
};

const courses = await queryPages<CourseType>(
SIS_COURSE_URL,
params,
headers,
"courses"
);
const courses = await queryPages<CourseType>(
SIS_COURSE_URL,
params,
headers,
"courses",
);

console.log("Updating database with new course data...");
console.log("Updating database with new course data...");

const bulkOps = courses.map((c) => ({
replaceOne: {
filter: { classDisplayName: c.classDisplayName },
replacement: c,
upsert: true,
},
}));
const bulkOps = courses.map((c) => ({
replaceOne: {
filter: { classDisplayName: c.classDisplayName },
replacement: c,
upsert: true,
},
}));

const options = {} as MongooseBulkWriteOptions;
const options = bulkWriteOptions as MongooseBulkWriteOptions;

const res = await CourseModel.bulkWrite(bulkOps, options);
const res = await CourseModel.bulkWrite(bulkOps, options);

console.log(
`Completed updating database with new course data. Created ${res.upsertedCount} and updated ${res.modifiedCount} course objects.`
);
console.log(
`Completed updating database with new course data. Created ${res.upsertedCount} and updated ${res.modifiedCount} course objects.`,
);
};

const updateClasses = async () => {
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};

const activeSemesters = await SemesterModel.find({ active: true }).lean();
const classes: ClassType[] = [];
const activeSemesters = await SemesterModel.find({ active: true }).lean();
const classes: ClassType[] = [];

for (const s of activeSemesters) {
console.log(`Updating classses for ${s.term} ${s.year}...`);
for (const s of activeSemesters) {
console.log(`Updating classses for ${s.term} ${s.year}...`);

const params = {
"term-id": semToTermId(s),
"page-size": 100,
};
const params = {
"term-id": semToTermId(s),
"page-size": 100,
};

const semesterClasses = await queryPages<ClassType>(
SIS_CLASS_URL,
params,
headers,
"classes"
);
classes.push(...semesterClasses);
}
const semesterClasses = await queryPages<ClassType>(
SIS_CLASS_URL,
params,
headers,
"classes",
);
classes.push(...semesterClasses);
}

console.log("Updating database with new class data...");
const bulkOps = classes.map((c) => ({
replaceOne: {
filter: { displayName: c.displayName },
replacement: c,
upsert: true,
},
}));
console.log("Updating database with new class data...");
const bulkOps = classes.map((c) => ({
replaceOne: {
filter: { displayName: c.displayName },
replacement: c,
upsert: true,
},
}));

const options = {} as MongooseBulkWriteOptions;
const options = bulkWriteOptions as MongooseBulkWriteOptions;

const res = await ClassModel.bulkWrite(bulkOps, options);
const res = await ClassModel.bulkWrite(bulkOps, options);

console.log(
`Completed updating database with new class data. Created ${res.upsertedCount} and updated ${res.modifiedCount} class objects.`
);
console.log(
`Completed updating database with new class data. Created ${res.upsertedCount} and updated ${res.modifiedCount} class objects.`,
);
};

const updateSections = async () => {
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};
const headers = {
app_id: config.sis.CLASS_APP_ID,
app_key: config.sis.CLASS_APP_KEY,
};

const activeSemesters = await SemesterModel.find({ active: true }).lean();
const sections: SectionType[] = [];
const activeSemesters = await SemesterModel.find({ active: true }).lean();
const sections: SectionType[] = [];

for (const s of activeSemesters) {
console.log(`Updating sections for ${s.term} ${s.year}...`);
for (const s of activeSemesters) {
console.log(`Updating sections for ${s.term} ${s.year}...`);

const params = {
"term-id": semToTermId(s),
"page-size": 100,
};
const params = {
"term-id": semToTermId(s),
"page-size": 100,
};

const semesterClasses = await queryPages<SectionType>(
SIS_SECTION_URL,
params,
headers,
"classSections"
);
sections.push(...semesterClasses);
}
const semesterClasses = await queryPages<SectionType>(
SIS_SECTION_URL,
params,
headers,
"classSections",
);
sections.push(...semesterClasses);
}

console.log("Updating database with new section data...");
const bulkOps = sections.map((s) => ({
replaceOne: {
filter: { displayName: s.displayName },
replacement: s,
upsert: true,
},
}));
console.log("Updating database with new section data...");
const bulkOps = sections.map((s) => ({
replaceOne: {
filter: { displayName: s.displayName },
replacement: s,
upsert: true,
},
}));

const options = {} as MongooseBulkWriteOptions;
const options = bulkWriteOptions as MongooseBulkWriteOptions;

const res = await SectionModel.bulkWrite(bulkOps, options);
const res = await SectionModel.bulkWrite(bulkOps, options);

console.log(
`Completed updating database with new section data. Created ${res.upsertedCount} and updated ${res.modifiedCount} section objects.`
);
console.log(
`Completed updating database with new section data. Created ${res.upsertedCount} and updated ${res.modifiedCount} section objects.`,
);
};

(async () => {
Expand Down
Loading
Loading