diff --git a/.eslintrc.js b/.eslintrc.js index b4eb1b7d6..d1cd8c7ff 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -48,6 +48,7 @@ module.exports = { "react-hooks/rules-of-hooks": "error", "@typescript-eslint/prefer-nullish-coalescing": "warn", "@typescript-eslint/prefer-optional-chain": "warn", + // complexity: "warn", // Place to specify ESLint rules. Can be used to overwrite rules specified from the extended configs // e.g. "@typescript-eslint/explicit-function-return-type": "off", }, diff --git a/.prettierrc.js b/.prettierrc.js index 64d2380c2..afdc64e2a 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -11,7 +11,9 @@ module.exports = { "", "^[@]", "", - "^[./]", + "^[./](?!graphql)", + "", + "^/graphql", ], plugins: [require.resolve("@ianvs/prettier-plugin-sort-imports")], importOrderBuiltinModulesToTop: true, diff --git a/README.md b/README.md index 5c5c3dd63..50bd70a0e 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,58 @@ -## Development environment +# Development environment -Note: Before starting development, please run the following command on the repo root: +## Requirements -```bash -npm ci -``` +Create `.env` files for backend and frontend. See examples and ask your local boffin for details. -frontend: +Install `docker-compose`, if not already installed. -```bash -cd frontend -npm ci -npm run dev -``` +## Development workflow -backend: +Run `npm ci` in the each of the root, backend and frontend directories to install dependencies. + +Create separate shells for the database container, backend and frontend: ```bash cd backend docker-compose up ``` +```bash +cd frontend +npm run dev +``` + ```bash cd backend -npm ci npm run migrate npm run dev ``` -## Generating GraphQL types for frontend +If the database doesn't seem to do anything, ie. no messages after the initial ones after running `docker-compose up` and the database queries are not getting through, run `docker-compose down` and try again. You can always run the database container in detached mode (`-d`) but then you won't see the logs live. -If you make changes to the GraphQL schema or the queries in the frontend, you probably need to regenerate the Typescript types. Run `npm run generate-graphql-types` in the frontend folder. +Run `npm run prettier` in the root directory before committing. The commit runs hooks to check this as well as some linters, type checks etc. -If you're getting errors about mismatching GraphQL versions, then try `npm i --legacy-peer-deps --save-dev apollo` to force it. This is because `apollo-tooling` keeps on packaging some old versions of dependencies instead of using the newer ones available. -## Using installed `librdkafka` to speed up backend development +
+Using pre-built librdkafka to speed up backend development -By default, `node-rdkafka` builds `librdkafka` from the source. This can take minutes on a bad day and can slow development down quite considerably. However, there's an option to use the version installed locally. +By default, `node-rdkafka` builds `librdkafka` from the source. This can take minutes on a bad day and can slow development down quite considerably, especially when you're working with different branches with different dependencies and need to run `npm ci` often. However, there's an option to use the version installed locally. Do this in some other directory than the project one: -```bash -wget https://github.com/edenhill/librdkafka/archive/v1.4.0.tar.gz -O - | tar -xz -cd librdkafka-1.4.0 +```bash +wget https://github.com/edenhill/librdkafka/archive/v1.8.2.tar.gz -O - | tar -xz +cd librdkafka-1.8.2 ./configure --prefix=/usr make && make install ``` -(Ubuntu 20.04 and later seem to require v1.5.0, so change accordingly.) - -You may have to do some of that as root. Alternatively, you can install a prebuilt package - see [here](https://github.com/edenhill/librdkafka) for more information. Just be sure to install version >1.4.0. +You may have to do some of that as root. Alternatively, you can install a prebuilt package - see [here](https://github.com/edenhill/librdkafka) for more information. Set the env `BUILD_LIBRDKAFKA=0` when doing `npm ci` or similar on the backend to skip the build. -## Documentation +
+ +## More documentation -[Kafka](backend/docs/kafka.md) +- [Kafka](docs/kafka.md) +- [GraphQL](docs/graphql.md) diff --git a/backend/.env.example b/backend/.env.example index c2d8a5a24..35ab79155 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -20,4 +20,5 @@ GOOGLE_CLOUD_STORAGE_BUCKET=x GOOGLE_CLOUD_STORAGE_PROJECT=x GOOGLE_CLOUD_STORAGE_KEYFILE=x -UPDATE_USER_SECRET=secret \ No newline at end of file +UPDATE_USER_SECRET=secret +BACKEND_URL=http://localhost:4000 \ No newline at end of file diff --git a/backend/api/completions.ts b/backend/api/completions.ts index 11e158d09..e19f31926 100644 --- a/backend/api/completions.ts +++ b/backend/api/completions.ts @@ -152,6 +152,7 @@ export class CompletionController { const { user } = getUserResult.value const { slug } = req.params + // TODO: typing let tierData: any = [] const course = ( diff --git a/backend/bin/kafkaConsumer/common/createKafkaConsumer.ts b/backend/bin/kafkaConsumer/common/createKafkaConsumer.ts index 5c872253f..87c21a128 100644 --- a/backend/bin/kafkaConsumer/common/createKafkaConsumer.ts +++ b/backend/bin/kafkaConsumer/common/createKafkaConsumer.ts @@ -16,7 +16,8 @@ import { KafkaError } from "../../lib/errors" import checkConnectionInInterval from "./connectedChecker" const logCommit = - (logger: winston.Logger) => (err: any, topicPartitions: any) => { + (logger: winston.Logger) => + (err: any, topicPartitions: Kafka.TopicPartition[]) => { if (err) { logger.error(new KafkaError("Error in commit", err)) } else { diff --git a/backend/bin/kafkaConsumer/common/userCourseProgress/interfaces.ts b/backend/bin/kafkaConsumer/common/userCourseProgress/interfaces.ts index 7f1d8cd64..3ea034c21 100644 --- a/backend/bin/kafkaConsumer/common/userCourseProgress/interfaces.ts +++ b/backend/bin/kafkaConsumer/common/userCourseProgress/interfaces.ts @@ -3,7 +3,7 @@ export interface Message { user_id: number course_id: string service_id: string - progress: [PointsByGroup] + progress: PointsByGroup[] message_format_version: Number } diff --git a/backend/bin/kafkaConsumer/common/userCourseProgress/validate.ts b/backend/bin/kafkaConsumer/common/userCourseProgress/validate.ts index d4ca523fe..c4bc42a40 100644 --- a/backend/bin/kafkaConsumer/common/userCourseProgress/validate.ts +++ b/backend/bin/kafkaConsumer/common/userCourseProgress/validate.ts @@ -1,6 +1,8 @@ import { Message as KafkaMessage } from "node-rdkafka" import * as yup from "yup" +import { Message } from "./interfaces" + const CURRENT_MESSAGE_FORMAT_VERSION = 1 const PointsByGroupYupSchema = yup.object().shape({ @@ -32,14 +34,14 @@ export const MessageYupSchema = yup.object().shape({ .required(), }) -const handleNullProgressImpl = (value: any) => ({ +const handleNullProgressImpl = (value: Message) => ({ ...value, - progress: value?.progress?.map((progress: any) => ({ - ...progress, + progress: value?.progress?.map((pointsByGroup) => ({ + ...pointsByGroup, progress: - progress.progress === null || isNaN(progress.progress) + pointsByGroup.progress === null || isNaN(pointsByGroup.progress) ? 0 - : progress.progress, + : pointsByGroup.progress, })), }) diff --git a/backend/bin/kafkaConsumer/common/userFunctions.ts b/backend/bin/kafkaConsumer/common/userFunctions.ts index 770a0c132..7bb101ae1 100644 --- a/backend/bin/kafkaConsumer/common/userFunctions.ts +++ b/backend/bin/kafkaConsumer/common/userFunctions.ts @@ -21,15 +21,20 @@ import { ServiceProgressType, } from "./userCourseProgress/interfaces" +interface WithKafkaContext { + context: KafkaContext +} + +interface GetCombinedUserCourseProgressArgs extends WithKafkaContext { + user: User + course: Course +} + export const getCombinedUserCourseProgress = async ({ user, course, context: { prisma }, -}: { - user: User - course: Course - context: KafkaContext -}): Promise => { +}: GetCombinedUserCourseProgressArgs): Promise => { const userCourseServiceProgresses = await prisma.user .findUnique({ where: { id: user.id } }) .user_course_service_progresses({ @@ -61,15 +66,16 @@ export const getCombinedUserCourseProgress = async ({ return combined } +interface CheckRequiredExerciseCompletionsArgs extends WithKafkaContext { + user: User + course: Course +} + export const checkRequiredExerciseCompletions = async ({ user, course, context: { knex }, -}: { - user: User - course: Course - context: KafkaContext -}): Promise => { +}: CheckRequiredExerciseCompletionsArgs): Promise => { if (course.exercise_completions_needed) { const exercise_completions = await knex("exercise_completion") .countDistinct("exercise_completion.exercise_id") @@ -85,15 +91,16 @@ export const checkRequiredExerciseCompletions = async ({ return true } +interface GetExerciseCompletionsForCoursesArgs extends WithKafkaContext { + user: User + courseIds: string[] +} + export const getExerciseCompletionsForCourses = async ({ user, courseIds, context: { knex }, -}: { - user: User - courseIds: string[] - context: KafkaContext -}) => { +}: GetExerciseCompletionsForCoursesArgs) => { // picks only one exercise completion per exercise/user: // the one with the latest timestamp and latest updated_at const exercise_completions: ExerciseCompletionPart[] = await knex( @@ -118,15 +125,16 @@ export const getExerciseCompletionsForCourses = async ({ return exercise_completions // ?.rows ?? [] } +interface PruneDuplicateExerciseCompletionsArgs extends WithKafkaContext { + user_id: string + course_id: string +} + export const pruneDuplicateExerciseCompletions = async ({ user_id, course_id, context: { knex }, -}: { - user_id: string - course_id: string - context: KafkaContext -}) => { +}: PruneDuplicateExerciseCompletionsArgs) => { // variation: only prune those with the latest timestamp but older updated_at /*const deleted: Array> = await knex( "exercise_completion", @@ -198,9 +206,7 @@ export const pruneDuplicateExerciseCompletions = async ({ export const pruneOrphanedExerciseCompletionRequiredActions = async ({ context: { knex }, -}: { - context: KafkaContext -}) => { +}: WithKafkaContext) => { const deleted: Array> = await knex("exercise_completion_required_actions") .whereNull("exercise_completion_id") @@ -210,15 +216,16 @@ export const pruneOrphanedExerciseCompletionRequiredActions = async ({ return deleted } +interface GetUserCourseSettingsArgs extends WithKafkaContext { + user_id: string + course_id: string +} + export const getUserCourseSettings = async ({ user_id, course_id, context: { prisma }, -}: { - user_id: string - course_id: string - context: KafkaContext -}): Promise => { +}: GetUserCourseSettingsArgs): Promise => { // - if the course inherits user course settings from some course, get settings from that one // - if not, get from the course itself or null if none exists const result = await prisma.course.findUnique({ @@ -256,12 +263,11 @@ export const getUserCourseSettings = async ({ ) } -interface CheckCompletionArgs { +interface CheckCompletionArgs extends WithKafkaContext { user: User course: Course handler?: Course | null combinedProgress?: CombinedUserCourseProgress - context: KafkaContext } export const checkCompletion = async ({ @@ -301,12 +307,11 @@ export const checkCompletion = async ({ } } -interface CreateCompletionArgs { +interface CreateCompletionArgs extends WithKafkaContext { user: User course: Course handler?: Course | null tier?: number - context: KafkaContext } export const createCompletion = async ({ diff --git a/backend/bin/kafkaConsumer/exerciseConsumer/interfaces.ts b/backend/bin/kafkaConsumer/exerciseConsumer/interfaces.ts index 53356a802..92a82601d 100644 --- a/backend/bin/kafkaConsumer/exerciseConsumer/interfaces.ts +++ b/backend/bin/kafkaConsumer/exerciseConsumer/interfaces.ts @@ -2,7 +2,7 @@ export interface Message { timestamp: string course_id: string service_id: string - data: ExerciseData[] //[ExerciseData] + data: ExerciseData[] message_format_version: number } diff --git a/backend/bin/seedPoints.ts b/backend/bin/seedPoints.ts index 12171a068..c3d13ffdb 100644 --- a/backend/bin/seedPoints.ts +++ b/backend/bin/seedPoints.ts @@ -5,21 +5,20 @@ import { Prisma } from "@prisma/client" import prisma from "../prisma" //Generate integer id which is not already taken -function generateUniqueUpstreamId({ ExistingIds }: { ExistingIds: number[] }) { +function generateUniqueUpstreamId(existingIds: number[]) { //take the largest possible integer - const LargestPossibleUpstreamId = 2147483647 - let UniqueIntId = 0 + const MAX_INTEGER = 2147483647 + let uniqueIntId = 0 //Go down from the largest possible integer //until value not already in use is found - let i: number - for (i = LargestPossibleUpstreamId; i > 0; i--) { - if (ExistingIds.indexOf(i) === -1) { - UniqueIntId = i - return UniqueIntId + for (let i = MAX_INTEGER; i > 0; i--) { + if (existingIds.indexOf(i) === -1) { + uniqueIntId = i + return uniqueIntId } } - return UniqueIntId + return uniqueIntId } function generateRandomString() { @@ -30,10 +29,12 @@ function generateRandomString() { } const addUsers = async () => { - //get existing users from database - const UsersInDatabase = await prisma.user.findMany() - //create a list of upstream ids already in use - let UpstreamIdsInUse = UsersInDatabase.map((user) => user.upstream_id) + //get existing upstream_ids + const upstreamIdsInUse = ( + await prisma.user.findMany({ + select: { upstream_id: true }, + }) + ).map((user) => user.upstream_id) //Generate random data for 100 users //and add them to the database let i = 0 @@ -42,7 +43,7 @@ const addUsers = async () => { const last_name = faker.name.lastName() const newUser = { - upstream_id: generateUniqueUpstreamId({ ExistingIds: UpstreamIdsInUse }), + upstream_id: generateUniqueUpstreamId(upstreamIdsInUse), first_name, last_name, username: faker.internet.userName(first_name, last_name), @@ -52,7 +53,7 @@ const addUsers = async () => { real_student_number: generateRandomString(), } //add new upstreamId to ids already in use - UpstreamIdsInUse = UpstreamIdsInUse.concat(newUser.upstream_id) + upstreamIdsInUse.push(newUser.upstream_id) await prisma.user.create({ data: newUser }) i += 1 @@ -71,10 +72,11 @@ const addServices = async () => { } } -const addUserCourseProgressess = async ({ courseId }: { courseId: string }) => { - const UsersInDb = await prisma.user.findMany({ take: 100 }) +const addUserCourseProgressess = async (courseId: string) => { + const usersInDb = await prisma.user.findMany({ take: 100 }) + return await Promise.all( - UsersInDb.map(async (user) => { + usersInDb.map(async (user) => { const progress = [ { group: "week1", @@ -129,7 +131,7 @@ const addUserCourseProgressess = async ({ courseId }: { courseId: string }) => { ) } -const addUserCourseSettingses = async ({ courseId }: { courseId: string }) => { +const addUserCourseSettingses = async (courseId: string) => { const UsersInDb = await prisma.user.findMany({ take: 100 }) return await Promise.all( UsersInDb.map(async (user) => { @@ -160,11 +162,14 @@ const seedPointsData = async () => { const course = await prisma.course.findUnique({ where: { slug: "elements-of-ai" }, }) - console.log("course", course) + await addUsers() await addServices() - course && (await addUserCourseProgressess({ courseId: course.id })) - course && (await addUserCourseSettingses({ courseId: course.id })) + + if (course) { + await addUserCourseProgressess(course.id) + await addUserCourseSettingses(course.id) + } } seedPointsData().finally(() => process.exit(0)) diff --git a/backend/graphql/Completion/index.ts b/backend/graphql/Completion/index.ts index 8a66e27d7..e617b1941 100644 --- a/backend/graphql/Completion/index.ts +++ b/backend/graphql/Completion/index.ts @@ -1,4 +1,4 @@ -// generated Mon Jul 11 2022 17:59:37 GMT+0300 (Itä-Euroopan kesäaika) +// generated Thu Aug 11 2022 17:12:07 GMT+0300 (Itä-Euroopan kesäaika) export * from "./input" export * from "./model" diff --git a/backend/graphql/Completion/model.ts b/backend/graphql/Completion/model.ts index b9dd8ecce..61d7d6554 100644 --- a/backend/graphql/Completion/model.ts +++ b/backend/graphql/Completion/model.ts @@ -14,6 +14,7 @@ export const Completion = objectType({ t.model.completion_language() t.model.email() t.model.student_number() + t.model.user_id() t.model.user_upstream_id() t.model.completions_registered() t.model.course_id() diff --git a/backend/graphql/CompletionRegistered.ts b/backend/graphql/CompletionRegistered.ts index 77bab901a..fd2167871 100644 --- a/backend/graphql/CompletionRegistered.ts +++ b/backend/graphql/CompletionRegistered.ts @@ -1,6 +1,15 @@ import { ForbiddenError } from "apollo-server-express" import { chunk } from "lodash" -import { arg, extendType, intArg, list, objectType, stringArg } from "nexus" +import { + arg, + extendType, + intArg, + list, + nonNull, + objectType, + stringArg, +} from "nexus" +import { type NexusGenInputs } from "nexus-typegen" import { Prisma } from "@prisma/client" @@ -109,11 +118,11 @@ export const CompletionRegisteredMutations = extendType({ t.field("registerCompletion", { type: "String", args: { - completions: list(arg({ type: "CompletionArg" })), + completions: nonNull(list(nonNull(arg({ type: "CompletionArg" })))), }, authorize: isOrganization, resolve: async (_, args, ctx: Context) => { - let queue = chunk(args.completions, 500) + const queue = chunk(args.completions, 500) for (let i = 0; i < queue.length; i++) { const promises = buildPromises(queue[i], ctx) @@ -125,7 +134,10 @@ export const CompletionRegisteredMutations = extendType({ }, }) -const buildPromises = (array: any[], ctx: Context) => { +const buildPromises = ( + array: Array, + ctx: Context, +) => { return array.map(async (entry) => { const { user_id, course_id } = (await ctx.prisma.completion.findUnique({ diff --git a/backend/graphql/Course/index.ts b/backend/graphql/Course/index.ts index 8a66e27d7..e617b1941 100644 --- a/backend/graphql/Course/index.ts +++ b/backend/graphql/Course/index.ts @@ -1,4 +1,4 @@ -// generated Mon Jul 11 2022 17:59:37 GMT+0300 (Itä-Euroopan kesäaika) +// generated Thu Aug 11 2022 17:12:07 GMT+0300 (Itä-Euroopan kesäaika) export * from "./input" export * from "./model" diff --git a/backend/graphql/Course/mutations.ts b/backend/graphql/Course/mutations.ts index 8c0a3d1a4..8307073ca 100644 --- a/backend/graphql/Course/mutations.ts +++ b/backend/graphql/Course/mutations.ts @@ -10,11 +10,15 @@ import { Context } from "../../context" import KafkaProducer, { ProducerMessage } from "../../services/kafkaProducer" import { invalidate } from "../../services/redis" import { convertUpdate } from "../../util/db-functions" +import { notEmpty } from "../../util/notEmpty" import { deleteImage, uploadImage } from "../Image" const isNotNull = (value: T | null | undefined): value is T => value !== null && value !== undefined +const nullToUndefined = (value: T | null | undefined): T | undefined => + value ?? undefined + export const CourseMutations = extendType({ type: "Mutation", definition(t) { @@ -77,7 +81,7 @@ export const CourseMutations = extendType({ study_modules: !!study_modules ? { connect: study_modules.map((s) => ({ - id: s?.id ?? undefined, + id: nullToUndefined(s?.id), })), } : undefined, @@ -105,7 +109,7 @@ export const CourseMutations = extendType({ }, }) - const kafkaProducer = await new KafkaProducer() + const kafkaProducer = new KafkaProducer() const producerMessage: ProducerMessage = { message: JSON.stringify(newCourse), partition: null, @@ -246,14 +250,15 @@ export const CourseMutations = extendType({ .findUnique({ where: { slug } }) .study_modules() //const addedModules: StudyModuleWhereUniqueInput[] = pullAll(study_modules, existingStudyModules.map(module => module.id)) - const removedModuleIds = (existingStudyModules || []) - .filter((module) => !getIds(study_modules ?? []).includes(module.id)) - .map((module) => ({ id: module.id } as { id: string })) + const removedModuleIds = + existingStudyModules + ?.filter((module) => !getIds(study_modules).includes(module.id)) + .map((module) => ({ id: module.id })) ?? [] const connectModules = study_modules?.map((s) => ({ ...s, - id: s?.id ?? undefined, - slug: s?.slug ?? undefined, + id: nullToUndefined(s?.id), + slug: nullToUndefined(s?.slug), })) ?? [] const studyModuleMutation: @@ -294,7 +299,7 @@ export const CourseMutations = extendType({ const updatedCourse = await ctx.prisma.course.update({ where: { - id: id ?? undefined, + id: nullToUndefined(id), slug, }, data: convertUpdate({ @@ -308,7 +313,7 @@ export const CourseMutations = extendType({ "completion_email_id", "course_stats_email_id", ]), - slug: new_slug ? new_slug : slug, + slug: new_slug ?? slug, end_date, // FIXME: disconnect removed photos? photo: !!photo ? { connect: { id: photo } } : undefined, @@ -374,18 +379,16 @@ export const CourseMutations = extendType({ }, }) -const getIds = (arr: any[]) => (arr || []).map((t) => t.id) -const filterNotIncluded = (arr1: any[], arr2: any[], mapToId = true) => { +type WithIdOrNull = { id?: string | null; [key: string]: any } | null +const getIds = (arr?: WithIdOrNull[] | null) => arr?.map((t) => t?.id) ?? [] + +function filterNotIncluded(arr1: WithIdOrNull[], arr2: WithIdOrNull[]) { const ids1 = getIds(arr1) const ids2 = getIds(arr2) - const filtered = ids1.filter((id) => !ids2.includes(id)) - - if (mapToId) { - return filtered.map((id) => ({ id })) - } + const filtered = ids1.filter((id) => !ids2.includes(id)).filter(notEmpty) - return filtered + return filtered.map((id) => ({ id })) } interface ICreateMutation { @@ -395,7 +398,7 @@ interface ICreateMutation { field: keyof Prisma.Prisma__CourseClient } -const createMutation = async ({ +const createMutation = async ({ ctx, slug, data, @@ -420,7 +423,7 @@ const createMutation = async ({ const updated = (data || []) .filter(hasId) // (t) => !!t.id) .map((t) => ({ - where: { id: t.id } as { id: string }, + where: { id: t.id }, data: t, //{ ...t, id: undefined }, })) const removed = filterNotIncluded(existing!, data) @@ -432,8 +435,6 @@ const createMutation = async ({ } } -const hasId = ( - data: T, -): data is any & { id: string | null } => Boolean(data?.id) -const hasNotId = (data: T) => - !hasId(data) +const hasId = (data: T): data is T & { id: string } => + Boolean(data?.id) +const hasNotId = (data: T) => !hasId(data) diff --git a/backend/graphql/Image.ts b/backend/graphql/Image.ts index 5789178b7..1285b2b33 100644 --- a/backend/graphql/Image.ts +++ b/backend/graphql/Image.ts @@ -1,10 +1,13 @@ +import { ReadStream } from "fs" + +import { FileUpload } from "graphql-upload" import { arg, booleanArg, extendType, idArg, nonNull, objectType } from "nexus" import { isAdmin } from "../accessControl" import { Context } from "../context" import { - deleteImage as deleteStorageImage, - uploadImage as uploadStorageImage, + deleteStorageImage, + uploadStorageImage, } from "../services/google-cloud" const sharp = require("sharp") @@ -24,7 +27,7 @@ export const Image = objectType({ t.model.original_mimetype() t.model.uncompressed() t.model.uncompressed_mimetype() - t.model.courses() + // t.model.courses() }, }) @@ -58,35 +61,29 @@ export const ImageMutations = extendType({ }, }) -const readFS = (stream: NodeJS.ReadStream): Promise => { - let chunkList: any[] | Uint8Array[] = [] +const readFS = (stream: ReadStream): Promise => { + const chunkList: Uint8Array[] = [] return new Promise((resolve, reject) => stream - .on("data", (data) => chunkList.push(data)) + .on("data", (data: Buffer) => chunkList.push(data)) .on("error", (err) => reject(err)) .on("end", () => resolve(Buffer.concat(chunkList))), ) } +interface UploadImageArgs { + ctx: Context + file: Promise + base64: boolean +} + export const uploadImage = async ({ ctx, file, base64 = false, -}: { - ctx: Context - file: any - base64: boolean -}) => { - const { - createReadStream, - mimetype, - filename, - }: { - createReadStream: Function - mimetype: string - filename: string - } = await file +}: UploadImageArgs) => { + const { createReadStream, mimetype, filename } = await file const image: Buffer = await readFS(createReadStream()) const filenameWithoutExtension = /(.+?)(\.[^.]*$|$)$/.exec(filename)?.[1] @@ -145,13 +142,15 @@ export const uploadImage = async ({ return newImage } +interface DeleteImageArgs { + ctx: Context + id: string +} + export const deleteImage = async ({ ctx, id, -}: { - ctx: Context - id: string -}): Promise => { +}: DeleteImageArgs): Promise => { const image = await ctx.prisma.image.findUnique({ where: { id } }) if (!image) { diff --git a/backend/graphql/StudyModule/index.ts b/backend/graphql/StudyModule/index.ts index 8a66e27d7..e617b1941 100644 --- a/backend/graphql/StudyModule/index.ts +++ b/backend/graphql/StudyModule/index.ts @@ -1,4 +1,4 @@ -// generated Mon Jul 11 2022 17:59:37 GMT+0300 (Itä-Euroopan kesäaika) +// generated Thu Aug 11 2022 17:12:07 GMT+0300 (Itä-Euroopan kesäaika) export * from "./input" export * from "./model" diff --git a/backend/graphql/Upload.ts b/backend/graphql/Upload.ts index dd6cf724c..66389b22f 100644 --- a/backend/graphql/Upload.ts +++ b/backend/graphql/Upload.ts @@ -5,5 +5,6 @@ export type UploadRoot = Promise export const Upload = scalarType({ ...GraphQLUpload!, + name: "Upload", rootTyping: "UploadRoot", }) diff --git a/backend/graphql/User/index.ts b/backend/graphql/User/index.ts index 8a66e27d7..e617b1941 100644 --- a/backend/graphql/User/index.ts +++ b/backend/graphql/User/index.ts @@ -1,4 +1,4 @@ -// generated Mon Jul 11 2022 17:59:37 GMT+0300 (Itä-Euroopan kesäaika) +// generated Thu Aug 11 2022 17:12:07 GMT+0300 (Itä-Euroopan kesäaika) export * from "./input" export * from "./model" diff --git a/backend/graphql/UserCourseProgress.ts b/backend/graphql/UserCourseProgress.ts index c1b6bfaa7..c41dfa307 100644 --- a/backend/graphql/UserCourseProgress.ts +++ b/backend/graphql/UserCourseProgress.ts @@ -93,7 +93,7 @@ export const UserCourseProgress = objectType({ throw new Error("no course or user found") } - const courseProgress: any = normalizeProgress(progress) + const courseProgress = normalizeProgress(progress) // TODO: this should probably also only count completed exercises! const exercises = await ctx.prisma.course diff --git a/backend/graphql/index.ts b/backend/graphql/index.ts index 177e86aea..4e792c5e6 100644 --- a/backend/graphql/index.ts +++ b/backend/graphql/index.ts @@ -1,4 +1,4 @@ -// generated Mon Jul 11 2022 17:59:37 GMT+0300 (Itä-Euroopan kesäaika) +// generated Thu Aug 11 2022 17:12:07 GMT+0300 (Itä-Euroopan kesäaika) export * from "./ABEnrollment" export * from "./ABStudy" diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 8231fc2d9..a54d9f3c9 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -1,6 +1,5 @@ generator client { provider = "prisma-client-js" - previewFeatures = ["aggregateApi"] } datasource db { diff --git a/backend/schema.ts b/backend/schema.ts index da141cbde..1eb26408d 100644 --- a/backend/schema.ts +++ b/backend/schema.ts @@ -69,7 +69,11 @@ export default makeSchema({ module: require.resolve(".prisma/client/index.d.ts"), alias: "prisma", }, + { module: "@types/graphql-upload/index.d.ts", alias: "upload" }, ], + mapping: { + Upload: "upload.Upload['promise']", + }, }, plugins: createPlugins(), outputs: { diff --git a/backend/services/google-cloud.ts b/backend/services/google-cloud.ts index bfe20a05f..4ce9b167d 100644 --- a/backend/services/google-cloud.ts +++ b/backend/services/google-cloud.ts @@ -38,19 +38,21 @@ const storage = const bucket = storage.bucket(GOOGLE_CLOUD_STORAGE_BUCKET ?? "") // this shouldn't ever happen in production -export const uploadImage = async ({ - imageBuffer, - mimeType, - name = "", - directory = "", - base64 = false, -}: { +interface UploadStorageImageArgs { imageBuffer: Buffer mimeType: string name?: string directory?: string base64?: boolean -}): Promise => { +} + +export const uploadStorageImage = async ({ + imageBuffer, + mimeType, + name = "", + directory = "", + base64 = false, +}: UploadStorageImageArgs): Promise => { const filename = `${directory ? directory + "/" : ""}${shortid.generate()}${ name && name !== "" ? "-" + name : "" }.${mime.extension(mimeType)}` @@ -83,7 +85,9 @@ export const uploadImage = async ({ }) } -export const deleteImage = async (filename: string): Promise => { +export const deleteStorageImage = async ( + filename: string, +): Promise => { if (!filename || filename === "") { return Promise.resolve(false) } diff --git a/backend/util/server-functions.ts b/backend/util/server-functions.ts index 9cb857a01..c470ee044 100644 --- a/backend/util/server-functions.ts +++ b/backend/util/server-functions.ts @@ -13,13 +13,15 @@ interface GetUserReturn { details: UserInfo } +interface RequireCourseOwnershipArgs { + course_id: string + ctx: ApiContext +} + export function requireCourseOwnership({ course_id, ctx, -}: { - course_id: string - ctx: ApiContext -}) { +}: RequireCourseOwnershipArgs) { return async function ( req: Request, res: Response, diff --git a/docs/graphql.md b/docs/graphql.md new file mode 100644 index 000000000..8b71b3cca --- /dev/null +++ b/docs/graphql.md @@ -0,0 +1,93 @@ +# Working with GraphQL + +## Frontend types + +If you make changes to the GraphQL schema, resolvers etc. in the backend and/or the operations/definitions in the frontend, you probably need to regenerate the Typescript types for the frontend. + +### In brief: + +Ensure you have a fresh GraphQL schema by running `npm run generate` in the backend folder. + +Then run `npm run graphql-codegen` in the frontend folder. You can also run `npm run graphql-codegen:watch` to watch for changes and regenerate the frontend types automatically. + +## Detailed example from backend to frontend: + +You've created a model called `StaffMember` in the Prisma schema: + +```prisma +model StaffMember { + id String @id @default(uuid()) + user_id String? + user User? @relation(fields: [user_id], references: [id]) + work_email String? + work_phone String? + created_at DateTime? @default(now()) + updated_at DateTime? @updatedAt + + @@map("staff_member) +} +``` + +You've also created the appropriate database migration and created a resolver for a query called `staffMember`, taking a single required parameter `id` of type `ID` and returing a `StaffMember` or `null` if none found. + +First, ensure that you have a fresh GraphQL schema by running `npm run generate` in the backend folder. The generated schema is automatically linked to the frontend folder. + +You then might define the query in the frontend as follows in a `.graphql` file situated somewhere in the `graphql` folder: + +```graphql +query StaffMemberDetails($id: ID!) { + staffMember(id: $id) { + id + user { + id + email + name + } + work_email + work_phone + } +} +``` + +Note that even if you could leave the query unnamed, it would then be assigned a random type name which would be less helpful. + +Run `npm run graphql-codegen` in the frontend folder. This will generate a bucketload of useful typings and other code. + +The most useful is the `StaffMemberDetailsDocument`, which is a correctly typed document node for the GraphQL operation, also providing the result type automatically. The result type is available separately as `StaffMemberDetailsQuery` and the operation variables as `StaffMemberDetailsQueryVariables`. Also, the model definition is available as `StaffMember` -- do note that it includes all the fields exposed in the GraphQL schema, not just the ones that are actually used in the query. If you're using only a subselection of fields for a query and passing the result to a component, you might want to create a fragment for the fields you're using and use the generated type. + +You can now use the query in the frontend as follows: + +```typescript +import { StaffMemberDetailsDocument } from "/graphql/generated" + +// in React components using the Apollo hooks: +function SayHello(id: string) { + const { data, loading, error } = useQuery(StaffMemberDetailsDocument, { + variables: { + id, + }, + }) + + return ( +
+ {loading &&

Loading...

} + {error &&

Error: {error.message}

} + {data(

Hello, {data.staffMember?.user.name ?? "stranger"}

} +
+ ) +} + +// or, if you're outside React components, assuming client is an ApolloClient instance +async function getStaffMemberDetails(client: ApolloClient, id: string) { + const { data } = await client.query({ + query: StaffMemberDetailsDocument, + variables: { id }, + }) + + return data?.staffMember +} +``` + +In both cases, the results of the query in `data` are typed correctly, so you would get `data.staffMember.work_email` for example -- provided that there is a user with the id that given as parameter. + +The flow is similar with mutations and fragments. Mutation result and operation variable types are named similarly, only with `Mutation` instead of `Query`. Fragments do not produce result types; the type of the fragment is the defined name + `Fragment`. diff --git a/backend/docs/kafka.md b/docs/kafka.md similarity index 100% rename from backend/docs/kafka.md rename to docs/kafka.md diff --git a/frontend/.env.example b/frontend/.env.example new file mode 100644 index 000000000..78b10c9b3 --- /dev/null +++ b/frontend/.env.example @@ -0,0 +1,3 @@ +BACKEND_URL=http://localhost:4000 +FRONTEND_URL=http://localhost:3000 +PORT=3000 diff --git a/frontend/apollo.config.js b/frontend/apollo.config.js index 55315592a..ceb8e5925 100644 --- a/frontend/apollo.config.js +++ b/frontend/apollo.config.js @@ -1,16 +1,7 @@ const path = require("path") module.exports = { client: { - tagName: "gql", - includes: [ - "./components/**/*.tsx", - "./pages/**/*.tsx", - "./components/**/*.ts", - "./pages/**/*.ts", - "./graphql/**/*.ts", - "./lib/**/*.tsx", - "./lib/**/*.ts", - ], + includes: ["./graphql/**/*.graphql"], excludes: ["node_modules"], service: { name: "backend", diff --git a/frontend/codegen.yml b/frontend/codegen.yml new file mode 100644 index 000000000..3770903ef --- /dev/null +++ b/frontend/codegen.yml @@ -0,0 +1,31 @@ +schema: schema.graphql +documents: "graphql/**/*.{ts,tsx,graphql}" +config: + preResolveTypes: true + namingConvention: keep + avoidOptionals: + field: true + nonOptionalTypeName: true + dedupeFragments: true +hooks: + afterAllFileWrite: + - prettier --write +generates: + ./graphql/generated/index.ts: + pluckConfig: + modules: + - name: "@apollo/client" + identifier: gql + plugins: + - add: + placement: "prepend" + content: + - "/**" + - " * This is an automatically generated file." + - " * Run `npm run graphql-codegen` to regenerate." + - " **/" + - time + - typescript + - typescript-operations + - fragment-matcher + - typed-document-node diff --git a/frontend/components/CertificateButton.tsx b/frontend/components/CertificateButton.tsx index ce528ed1d..976112339 100644 --- a/frontend/components/CertificateButton.tsx +++ b/frontend/components/CertificateButton.tsx @@ -1,6 +1,6 @@ import { useContext, useEffect, useReducer, useState } from "react" -import { gql, useMutation } from "@apollo/client" +import { useMutation } from "@apollo/client" import styled from "@emotion/styled" import { CircularProgress, @@ -15,16 +15,20 @@ import DialogTitle from "@mui/material/DialogTitle" import AlertContext from "/contexts/AlertContext" import LoginStateContext from "/contexts/LoginStateContext" -import { UserOverViewQuery as CompletionsUserOverViewQuery } from "/graphql/queries/currentUser" import { updateAccount } from "/lib/account" import { checkCertificate, createCertificate } from "/lib/certificates" -import { UserDetailQuery } from "/lib/with-apollo-client/fetch-user-details" -import { UserOverViewQuery } from "/pages/profile" -import { ProfileUserOverView_currentUser_completions_course } from "/static/types/generated/ProfileUserOverView" -import { UserOverView_currentUser } from "/static/types/generated/UserOverView" import CompletionsTranslations from "/translations/completions" import { useTranslator } from "/util/useTranslator" +import { + CourseCoreFieldsFragment, + CurrentUserDetailedDocument, + CurrentUserDocument, + CurrentUserOverviewDocument, + UpdateUserNameDocument, + UserOverviewFieldsFragment, +} from "/graphql/generated" + const StyledButton = styled(Button)` margin: auto; background-color: #005361; @@ -37,18 +41,8 @@ const StyledTextField = styled(TextField)` margin-bottom: 1rem; ` -const updateUserNameMutation = gql` - mutation updateUserName($first_name: String, $last_name: String) { - updateUserName(first_name: $first_name, last_name: $last_name) { - id - first_name - last_name - } - } -` - interface CertificateProps { - course: ProfileUserOverView_currentUser_completions_course + course: CourseCoreFieldsFragment } type Status = @@ -132,9 +126,9 @@ const reducer = (state: CertificateState, action: Action): CertificateState => { status: "ERROR", error: action.payload, } + default: + return state } - - return state } const CertificateButton = ({ course }: CertificateProps) => { @@ -146,11 +140,11 @@ const CertificateButton = ({ course }: CertificateProps) => { const [firstName, setFirstName] = useState(currentUser?.first_name ?? "") const [lastName, setLastName] = useState(currentUser?.last_name ?? "") - const [updateUserName] = useMutation(updateUserNameMutation, { + const [updateUserName] = useMutation(UpdateUserNameDocument, { refetchQueries: [ - { query: UserDetailQuery }, - { query: UserOverViewQuery }, - { query: CompletionsUserOverViewQuery }, + { query: CurrentUserDocument }, + { query: CurrentUserDetailedDocument }, + { query: CurrentUserOverviewDocument }, ], }) @@ -204,7 +198,7 @@ const CertificateButton = ({ course }: CertificateProps) => { ...(currentUser || { email: "", id: "" }), first_name: firstName, last_name: lastName, - } as UserOverView_currentUser) + } as UserOverviewFieldsFragment) dispatch({ type: "UPDATED_NAME", payload: res }) } diff --git a/frontend/components/CompletedCourseCard.tsx b/frontend/components/CompletedCourseCard.tsx index cbab6c318..3d18d3678 100644 --- a/frontend/components/CompletedCourseCard.tsx +++ b/frontend/components/CompletedCourseCard.tsx @@ -6,11 +6,15 @@ import Typography from "@mui/material/Typography" import { mapLangToLanguage } from "/components/DataFormatFunctions" import { ClickableDiv } from "/components/Surfaces/ClickableCard" -import { ProfileUserOverView_currentUser_completions } from "/static/types/generated/ProfileUserOverView" import CompletionsTranslations from "/translations/completions" import ProfileTranslations from "/translations/profile" import { useTranslator } from "/util/useTranslator" +import { + CompletionDetailedFieldsFragment, + CourseWithPhotoCoreFieldsFragment, +} from "/graphql/generated" + const Background = styled(ClickableDiv)` display: flex; flex-direction: column; @@ -38,7 +42,9 @@ const RegistrationDetails = styled.div` ` interface CourseCardProps { - completion: ProfileUserOverView_currentUser_completions + completion: CompletionDetailedFieldsFragment & { + course: CourseWithPhotoCoreFieldsFragment + } } function formatDateTime(date: string) { diff --git a/frontend/components/CourseImage.tsx b/frontend/components/CourseImage.tsx index 446773e42..e2c21beff 100644 --- a/frontend/components/CourseImage.tsx +++ b/frontend/components/CourseImage.tsx @@ -3,9 +3,10 @@ import { memo } from "react" import styled from "@emotion/styled" import { Typography } from "@mui/material" -import { AllCourses_courses_photo } from "/static/types/generated/AllCourses" import { addDomain } from "/util/imageUtils" +import { ImageCoreFieldsFragment } from "/graphql/generated" + const ComponentStyle = ` width: 100%; height: 100%; @@ -23,7 +24,7 @@ const PlaceholderComponent = styled.div` align-items: center; ` interface CourseImageProps { - photo?: AllCourses_courses_photo | null + photo?: ImageCoreFieldsFragment | null [k: string]: any } diff --git a/frontend/components/CreateEmailTemplateDialog.tsx b/frontend/components/CreateEmailTemplateDialog.tsx index 107831fe0..a49458e18 100644 --- a/frontend/components/CreateEmailTemplateDialog.tsx +++ b/frontend/components/CreateEmailTemplateDialog.tsx @@ -3,12 +3,7 @@ import { useState } from "react" import { omit } from "lodash" import Router from "next/router" -import { - gql, - OperationVariables, - useApolloClient, - useQuery, -} from "@apollo/client" +import { useApolloClient, useQuery } from "@apollo/client" import { Button, Dialog, @@ -23,34 +18,17 @@ import { import CustomSnackbar from "/components/CustomSnackbar" import Spinner from "/components/Spinner" -import { UpdateCourseMutation } from "/graphql/mutations/courses" -import { AddEmailTemplateMutation } from "/graphql/mutations/email-templates" -import { AddEmailTemplate } from "/static/types/generated/AddEmailTemplate" -import { CourseDetailsFromSlugQuery_course as CourseDetailsData } from "/static/types/generated/CourseDetailsFromSlugQuery" -import { updateCourse } from "/static/types/generated/updateCourse" - -export const AllCoursesDetails = gql` - query AllCoursesDetails { - courses { - id - slug - name - teacher_in_charge_name - teacher_in_charge_email - start_date - completion_email { - name - id - } - course_stats_email { - id - } - } - } -` + +import { + AddEmailTemplateDocument, + CourseCoreFieldsFragment, + CourseUpsertArg, + EmailTemplateEditorCoursesDocument, + UpdateCourseDocument, +} from "/graphql/generated" interface CreateEmailTemplateDialogParams { - course?: CourseDetailsData + course?: CourseCoreFieldsFragment buttonText: string type?: string } @@ -63,13 +41,10 @@ const CreateEmailTemplateDialog = ({ const [openDialog, setOpenDialog] = useState(false) const [nameInput, setNameInput] = useState("") const [templateType, setTemplateType] = useState(type) - const [selectedCourse, setSelectedCourse] = useState< - CourseDetailsData | undefined - >(undefined) + const [selectedCourse, setSelectedCourse] = + useState(null) const [isErrorSnackbarOpen, setIsErrorSnackbarOpen] = useState(false) - const { loading, error, data } = useQuery<{ courses: CourseDetailsData[] }>( - AllCoursesDetails, - ) + const { loading, error, data } = useQuery(EmailTemplateEditorCoursesDocument) const client = useApolloClient() if (loading) { @@ -90,8 +65,8 @@ const CreateEmailTemplateDialog = ({ const courseOptions = templateType === "completion" - ? data.courses - .filter((c) => c?.completion_email === null) + ? data!.courses + ?.filter((c) => c?.completion_email === null) .map((c, i) => { return ( ) }) - : data.courses.map((c, i) => { + : data!.courses?.map((c, i) => { return (