diff --git a/packages/api/src/controllers/api-token.js b/packages/api/src/controllers/api-token.js index 1ee47a05ad..5caeb1547a 100644 --- a/packages/api/src/controllers/api-token.js +++ b/packages/api/src/controllers/api-token.js @@ -48,6 +48,7 @@ const fieldsMap = { name: { val: `api_token.data->>'name'`, type: "full-text" }, lastSeen: `api_token.data->'lastSeen'`, userId: `api_token.data->>'userId'`, + projectId: `api_token.data->>'projectId'`, "user.email": { val: `users.data->>'email'`, type: "full-text" }, }; @@ -66,6 +67,9 @@ app.get("/", async (req, res) => { if (!userId) { const query = parseFilters(fieldsMap, filters); + query.push( + sql`coalesce(api_token.data->>'projectId', '') = ${req.project?.id || ""}` + ); let fields = " api_token.id as id, api_token.data as data, users.id as usersId, users.data as usersdata"; @@ -102,9 +106,13 @@ app.get("/", async (req, res) => { errors: ["user can only request information on their own tokens"], }); } - const query = parseFilters(fieldsMap, filters); query.push(sql`api_token.data->>'userId' = ${userId}`); + query.push( + req.project?.id + ? sql`api_token.data->>'projectId' = ${req.project.id}` + : sql`api_token.data->>'projectId' IS NULL OR api_token.data->>'projectId' = ''` + ); let fields = " api_token.id as id, api_token.data as data"; if (count) { @@ -155,6 +163,7 @@ app.post("/", validatePost("api-token"), async (req, res) => { await req.store.create({ id: id, userId: userId, + projectId: req.query.projectId?.toString(), kind: "api-token", name: req.body.name, access: req.body.access, diff --git a/packages/api/src/controllers/asset.test.ts b/packages/api/src/controllers/asset.test.ts index 3cf02c1a90..af0f77be79 100644 --- a/packages/api/src/controllers/asset.test.ts +++ b/packages/api/src/controllers/asset.test.ts @@ -6,7 +6,13 @@ import { createMockFile, } from "../test-helpers"; import { v4 as uuid } from "uuid"; -import { Asset, AssetPatchPayload, Task, User } from "../schema/types"; +import { + ApiToken, + Asset, + AssetPatchPayload, + Task, + User, +} from "../schema/types"; import { db } from "../store"; import { WithID } from "../store/types"; import Table from "../store/table"; @@ -76,6 +82,39 @@ describe("controllers/asset", () => { let nonAdminUser: User; let nonAdminToken: string; + const createProject = async () => { + let res = await client.post(`/project`); + expect(res.status).toBe(201); + const project = await res.json(); + expect(project).toBeDefined(); + return project; + }; + + const allowedOrigins = [ + "http://localhost:3000", + "https://staging.wetube.com", + "http://blockflix.io:69", + ]; + + const createApiToken = async ( + cors: ApiToken["access"]["cors"], + projectId: string + ) => { + client.jwtAuth = nonAdminToken; + let res = await client.post(`/api-token/?projectId=${projectId}`, { + name: "test", + access: { cors }, + }); + client.jwtAuth = null; + expect(res.status).toBe(201); + const apiKeyObj = await res.json(); + expect(apiKeyObj).toMatchObject({ + id: expect.any(String), + access: { cors }, + }); + return apiKeyObj.id; + }; + beforeEach(async () => { await db.objectStore.create({ id: "mock_vod_store", @@ -103,6 +142,7 @@ describe("controllers/asset", () => { type: "url", url: spec.url, }, + projectId: "", //should be blank when using jwt and projectId not specified as query-param status: { phase: "waiting" }, }); @@ -123,6 +163,7 @@ describe("controllers/asset", () => { client.jwtAuth = null; client.apiKey = adminApiKey; + res = await client.post(`/task/${taskId}/status`, { status: { phase: "running", @@ -148,6 +189,103 @@ describe("controllers/asset", () => { }); }); + it.only("should import asset (using jwt) for existing project (created with jwt)", async () => { + const spec = { + name: "test", + url: "https://example.com/test.mp4", + }; + const projectId = await createProject(); + + let res = await client.post( + `/asset/upload/url/?projectId=${projectId}`, + spec + ); + expect(res.status).toBe(201); + const { asset, task } = await res.json(); + expect(asset).toMatchObject({ + id: expect.any(String), + name: "test", + source: { + type: "url", + url: spec.url, + }, + projectId: `${projectId}`, + status: { phase: "waiting" }, + }); + + client.jwtAuth = null; + client.apiKey = adminApiKey; + + res = await client.get(`/project/${projectId}`); + const project = await res.json(); + expect(res.status).toBe(200); + expect(project).toBeDefined(); //api-key be retrieve if adminApiKey is used.. + }); + + it.only("should import asset (using api-token) for existing project (created with jwt)", async () => { + const spec = { + name: "test", + url: "https://example.com/test.mp4", + }; + const projectId = await createProject(); + + client.jwtAuth = null; + client.apiKey = await createApiToken({ allowedOrigins }, projectId); + + let res = await client.post(`/asset/upload/url/`, spec); + expect(res.status).toBe(201); + const { asset, task } = await res.json(); + expect(asset).toMatchObject({ + id: expect.any(String), + name: "test", + source: { + type: "url", + url: spec.url, + }, + projectId: `${projectId}`, + status: { phase: "waiting" }, + }); + + client.apiKey = adminApiKey; + res = await client.get(`/project/${projectId}`); + const project = await res.json(); + expect(res.status).toBe(200); + expect(project.id).toBeDefined(); + }); + + it("should NOT import asset (using api-key) when projectId passed as ouery-param", async () => { + const spec = { + name: "test", + url: "https://example.com/test.mp4", + }; + + client.jwtAuth = null; + client.apiKey = adminApiKey; + + const projectId = await createProject(); + + // BadRequest is expected if projectId is passed in as query-param + let res = await client.post( + `/asset/upload/url/?projectId=${projectId}`, + spec + ); + expect(res.status).toBe(400); + + // Let's try again without query-param + res = await client.post(`/asset/upload/url/`, spec); + const { asset, task } = await res.json(); + expect(asset).toMatchObject({ + id: expect.any(String), + name: "test", + source: { + type: "url", + url: spec.url, + }, + projectId: "", //should be blank when using an existing api-key and new project was created + status: { phase: "waiting" }, + }); + }); + it("should detect duplicate assets", async () => { const spec = { name: "test", diff --git a/packages/api/src/controllers/asset.ts b/packages/api/src/controllers/asset.ts index 2d307ef159..b1c7076eee 100644 --- a/packages/api/src/controllers/asset.ts +++ b/packages/api/src/controllers/asset.ts @@ -45,6 +45,7 @@ import { NewAssetPayload, ObjectStore, PlaybackPolicy, + Project, Task, } from "../schema/types"; import { WithID } from "../store/types"; @@ -183,7 +184,7 @@ function parseUrlToDStorageUrl( } export async function validateAssetPayload( - req: Pick, + req: Pick, id: string, playbackId: string, createdAt: number, @@ -235,6 +236,7 @@ export async function validateAssetPayload( name: payload.name, source, staticMp4: payload.staticMp4, + projectId: req.project?.id ?? "", creatorId: mapInputCreatorId(payload.creatorId), playbackPolicy, objectStoreId: payload.objectStoreId || (await defaultObjectStoreId(req)), @@ -614,6 +616,7 @@ const fieldsMap = { creatorId: `asset.data->'creatorId'->>'value'`, playbackId: `asset.data->>'playbackId'`, playbackRecordingId: `asset.data->>'playbackRecordingId'`, + projectId: `asset.data->>'projectId'`, phase: `asset.data->'status'->>'phase'`, "user.email": { val: `users.data->>'email'`, type: "full-text" }, cid: `asset.data->'storage'->'ipfs'->>'cid'`, @@ -655,6 +658,12 @@ app.get("/", authorizer({}), async (req, res) => { query.push(sql`asset.data->>'deleted' IS NULL`); } + query.push( + req.project?.id + ? sql`asset.data->>'projectId' = ${req.project.id}` + : sql`asset.data->>'projectId' IS NULL OR asset.data->>'projectId' = ''` + ); + let output: WithID[]; let newCursor: string; if (req.user.admin && allUsers && allUsers !== "false") { @@ -799,7 +808,11 @@ const uploadWithUrlHandler: RequestHandler = async (req, res) => { url, encryption: assetEncryptionWithoutKey(encryption), }); - const dupAsset = await db.asset.findDuplicateUrlUpload(url, req.user.id); + const dupAsset = await db.asset.findDuplicateUrlUpload( + url, + req.user.id, + req.project?.id + ); if (dupAsset) { const [task] = await db.task.find({ outputAssetId: dupAsset.id }); if (!task.length) { diff --git a/packages/api/src/controllers/index.ts b/packages/api/src/controllers/index.ts index 8351af8ba9..b791f5eac6 100644 --- a/packages/api/src/controllers/index.ts +++ b/packages/api/src/controllers/index.ts @@ -24,6 +24,7 @@ import session from "./session"; import playback from "./playback"; import did from "./did"; import room from "./room"; +import project from "./project"; // Annoying but necessary to get the routing correct export default { @@ -53,4 +54,5 @@ export default { did, room, clip, + project, }; diff --git a/packages/api/src/controllers/project.ts b/packages/api/src/controllers/project.ts new file mode 100644 index 0000000000..e4e3930cf2 --- /dev/null +++ b/packages/api/src/controllers/project.ts @@ -0,0 +1,145 @@ +import { Request, Router } from "express"; +import { authorizer, validatePost } from "../middleware"; +import { db } from "../store"; +import { v4 as uuid } from "uuid"; +import { + makeNextHREF, + parseFilters, + parseOrder, + toStringValues, +} from "./helpers"; +import { + NotFoundError, + ForbiddenError, + BadRequestError, + BadGatewayError, + InternalServerError, +} from "../store/errors"; +import sql from "sql-template-strings"; +import { WithID } from "../store/types"; +import { Project } from "../schema/types"; + +const app = Router(); + +const fieldsMap = { + id: `project.ID`, + name: { val: `project.data->>'name'`, type: "full-text" }, + createdAt: { val: `project.data->'createdAt'`, type: "int" }, + userId: `project.data->>'userId'`, +} as const; + +app.get("/", authorizer({}), async (req, res) => { + let { limit, cursor, order, all, filters, count } = toStringValues(req.query); + + if (isNaN(parseInt(limit))) { + limit = undefined; + } + if (!order) { + order = "updatedAt-true,createdAt-true"; + } + + const query = [...parseFilters(fieldsMap, filters)]; + + if (!req.user.admin || !all || all === "false") { + query.push(sql`project.data->>'deleted' IS NULL`); + } + + let output: WithID[]; + let newCursor: string; + if (req.user.admin) { + let fields = + " project.id as id, project.data as data, users.id as usersId, users.data as usersdata"; + if (count) { + fields = fields + ", count(*) OVER() AS count"; + } + const from = `project left join users on project.data->>'userId' = users.id`; + [output, newCursor] = await db.project.find(query, { + limit, + cursor, + fields, + from, + order: parseOrder(fieldsMap, order), + process: ({ data, usersdata, count: c }) => { + if (count) { + res.set("X-Total-Count", c); + } + return { + ...data, + user: db.user.cleanWriteOnlyResponse(usersdata), + }; + }, + }); + } else { + query.push(sql`project.data->>'userId' = ${req.user.id}`); + + let fields = " project.id as id, project.data as data"; + if (count) { + fields = fields + ", count(*) OVER() AS count"; + } + [output, newCursor] = await db.project.find(query, { + limit, + cursor, + fields, + order: parseOrder(fieldsMap, order), + process: ({ data, count: c }) => { + if (count) { + res.set("X-Total-Count", c); + } + return data; + }, + }); + } + + res.status(200); + if (output.length > 0 && newCursor) { + res.links({ next: makeNextHREF(req, newCursor) }); + } + + return res.json(output); +}); + +app.get("/:id", authorizer({}), async (req, res) => { + const project = await db.project.get(req.params.id, { + useReplica: false, + }); + console.log("YYY :", project, req.params.id); + + if (!project || project.deleted) { + res.status(403); + return res.json({ errors: ["project not found"] }); + } + console.log("YYY here:", project, req.params.id); + + if (req.user.admin !== true && req.user.id !== project.userId) { + throw new ForbiddenError( + "user can only request information on their own projects" + ); + } + console.log("YYY here here:", project, req.params.id); + + res.json(project); +}); + +app.post("/", authorizer({}), async (req, res) => { + const { name } = req.body; + + const id = uuid(); + await db.project.create({ + id: id, + name: name, + userId: req.user.id, + createdAt: Date.now(), + }); + + const project = await db.project.get(id, { useReplica: false }); + + if (!project) { + res.status(403); + return res.json({ errors: ["project not created"] }); + } + + res.status(201); + res.json(id); +}); + +export default app; diff --git a/packages/api/src/middleware/auth.ts b/packages/api/src/middleware/auth.ts index 0dc2006aac..2d97289d49 100644 --- a/packages/api/src/middleware/auth.ts +++ b/packages/api/src/middleware/auth.ts @@ -3,11 +3,14 @@ import basicAuth from "basic-auth"; import corsLib, { CorsOptions } from "cors"; import { Request, RequestHandler, Response } from "express"; import jwt, { JwtPayload, TokenExpiredError } from "jsonwebtoken"; - import { pathJoin2, trimPathPrefix } from "../controllers/helpers"; -import { ApiToken, User } from "../schema/types"; +import { ApiToken, User, Project } from "../schema/types"; import { db } from "../store"; -import { ForbiddenError, UnauthorizedError } from "../store/errors"; +import { + ForbiddenError, + BadRequestError, + UnauthorizedError, +} from "../store/errors"; import { WithID } from "../store/types"; import { AuthRule, AuthPolicy } from "./authPolicy"; import tracking from "./tracking"; @@ -56,6 +59,17 @@ function isAuthorized( } } +export async function getProject(req: Request, projectId: string) { + const project = projectId + ? await db.project.get(projectId) + : { id: "", name: "default", userId: req.user.id }; + if (!req.user.admin && req.user.id !== project.userId) { + throw new ForbiddenError(`invalid user`); + } + + return project; +} + /** * Creates a middleware that parses and verifies the authentication method from * the request and populates the `express.Request` object. @@ -84,8 +98,10 @@ function authenticator(): RequestHandler { parseAuthHeader(authHeader); const basicUser = basicAuth.parse(authHeader); let user: User; + let project: Project; let tokenObject: WithID; let userId: string; + let projectId: string; if (!authScheme) { return next(); @@ -101,7 +117,12 @@ function authenticator(): RequestHandler { throw new UnauthorizedError(`no token ${tokenId} found`); } + if (req.query.projectId) { + throw new BadRequestError(`projectId as query param not supported`); + } + userId = tokenObject.userId; + projectId = tokenObject.projectId; // track last seen tracking.recordToken(tokenObject); } else if (authScheme === "jwt") { @@ -110,6 +131,7 @@ function authenticator(): RequestHandler { audience: req.config.jwtAudience, }) as JwtPayload; userId = verified.sub; + projectId = req.query.projectId?.toString(); // jwt lib will already validate the exp in case its present, so we just // need to check for the never-expiring JWTs. @@ -149,6 +171,11 @@ function authenticator(): RequestHandler { req.token = tokenObject; req.user = user; + if (projectId) { + project = await getProject(req, projectId); + req.project = project; + } + // UI admins must have a JWT req.isUIAdmin = user.admin && authScheme === "jwt"; diff --git a/packages/api/src/schema/api-schema.yaml b/packages/api/src/schema/api-schema.yaml index d0d2f6af2f..2687819b62 100644 --- a/packages/api/src/schema/api-schema.yaml +++ b/packages/api/src/schema/api-schema.yaml @@ -351,6 +351,25 @@ components: type: string url: $ref: "#/components/schemas/multistream-target/properties/url" + project: + type: object + required: + - name + additionalProperties: false + properties: + id: + type: string + readOnly: true + example: de7818e7-610a-4057-8f6f-b785dc1e6f88 + name: + type: string + example: test_project + createdAt: + type: number + readOnly: true + description: + Timestamp (in milliseconds) at which stream object was created + example: 1587667174725 stream: type: object required: @@ -1038,6 +1057,10 @@ components: The name of the asset. This is not necessarily the filename - it can be a custom name or title. example: filename.mp4 + projectId: + type: string + description: The ID of the project + example: aac12556-4d65-4d34-9fb6-d1f0985eb0a9 createdAt: readOnly: true type: number @@ -1186,6 +1209,8 @@ components: The name of the asset. This is not necessarily the filename - it can be a custom name or title. example: filename.mp4 + projectId: + $ref: "#/components/schemas/asset" staticMp4: type: boolean description: Whether to generate MP4s for the asset. diff --git a/packages/api/src/schema/db-schema.yaml b/packages/api/src/schema/db-schema.yaml index 7845b03f9a..c7dc3b468e 100644 --- a/packages/api/src/schema/db-schema.yaml +++ b/packages/api/src/schema/db-schema.yaml @@ -402,6 +402,10 @@ components: type: string index: true example: 66E2161C-7670-4D05-B71D-DA2D6979556F + projectId: + type: string + index: true + example: 66E2161C-7670-4D05-B71D-DA2D6979556F name: type: string example: Example Token @@ -653,6 +657,20 @@ components: type: string probability: type: number + project: + table: project + properties: + kind: + type: string + example: project + readOnly: true + userId: + index: true + type: string + example: 66E2161C-7670-4D05-B71D-DA2D6979556F + deleted: + type: boolean + description: Set to true when the project is deleted stream: table: stream properties: diff --git a/packages/api/src/store/asset-table.ts b/packages/api/src/store/asset-table.ts index ebf48d89f9..dbb08454ed 100644 --- a/packages/api/src/store/asset-table.ts +++ b/packages/api/src/store/asset-table.ts @@ -121,7 +121,8 @@ export default class AssetTable extends Table> { async findDuplicateUrlUpload( url: string, - userId: string + userId: string, + projectId: string ): Promise> { const createdAfter = Date.now() - DUPLICATE_ASSETS_THRESHOLD; const query = [ @@ -132,6 +133,14 @@ export default class AssetTable extends Table> { sql`asset.data->'status'->>'phase' IN ('waiting', 'processing')`, sql`coalesce((asset.data->>'createdAt')::bigint, 0) > ${createdAfter}`, ]; + if (projectId) { + query.push(sql`asset.data->>'projectId' = ${projectId}`); + } else { + query.push( + sql`(asset.data->>'projectId' IS NULL OR asset.data->>'projectId' = '')` + ); + } + const [assets] = await this.find(query, { limit: 1 }); return assets?.length > 0 ? assets[0] : null; } diff --git a/packages/api/src/store/db.ts b/packages/api/src/store/db.ts index a136e1c30b..a72f13ea58 100644 --- a/packages/api/src/store/db.ts +++ b/packages/api/src/store/db.ts @@ -17,6 +17,7 @@ import { Attestation, JwtRefreshToken, WebhookLog, + Project, } from "../schema/types"; import BaseTable, { TableOptions } from "./table"; import StreamTable from "./stream-table"; @@ -65,6 +66,7 @@ export class DB { region: Table; session: SessionTable; room: Table; + project: Table; postgresUrl: string; replicaUrl: string; @@ -175,6 +177,7 @@ export class DB { }); this.session = new SessionTable({ db: this, schema: schemas["session"] }); this.room = makeTable({ db: this, schema: schemas["room"] }); + this.project = makeTable({ db: this, schema: schemas["project"] }); const tables = Object.entries(schema.components.schemas).filter( ([name, schema]) => "table" in schema && schema.table diff --git a/packages/api/src/types/common.d.ts b/packages/api/src/types/common.d.ts index 10c0e9afd1..d21ddf4ebc 100644 --- a/packages/api/src/types/common.d.ts +++ b/packages/api/src/types/common.d.ts @@ -1,5 +1,5 @@ import { Ingest, Price } from "../middleware/hardcoded-nodes"; -import { Stream, User, ApiToken } from "../schema/types"; +import { Stream, User, ApiToken, Project } from "../schema/types"; import { WithID } from "../store/types"; import Queue from "../store/queue"; import { TaskScheduler } from "../task/scheduler"; @@ -42,6 +42,7 @@ declare global { frontendDomain: string; catalystBaseUrl: string; user?: User; + project?: Project; isUIAdmin?: boolean; isNeverExpiringJWT?: boolean; token?: WithID;