Skip to content

Commit

Permalink
Start on webhook log API (#2015)
Browse files Browse the repository at this point in the history
* Start on webhook log API

* remove webhook status api

* rename to requests

* fix webhook list sorting

* Review comments, schema updates etc

* schema updates
  • Loading branch information
mjh1 authored Jan 23, 2024
1 parent cf4ba6f commit 3803bea
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 115 deletions.
2 changes: 1 addition & 1 deletion packages/api/src/controllers/task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ app.get("/", authorizer({}), async (req, res) => {
const query = parseFilters(fieldsMap, filters);
query.push(sql`task.data->>'userId' = ${req.user.id}`);

if (!all || all === "false") {
if (!all || all === "false" || !req.user.admin) {
query.push(sql`task.data->>'deleted' IS NULL`);
}

Expand Down
41 changes: 24 additions & 17 deletions packages/api/src/controllers/webhook.test.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import serverPromise from "../test-server";
import { TestClient, clearDatabase } from "../test-helpers";
import { sleep } from "../util";

let server;
let mockAdminUser;
Expand Down Expand Up @@ -190,23 +191,6 @@ describe("controllers/webhook", () => {
expect(updated.name).toEqual("modified_name");
});

it("update the status of a webhook", async () => {
const { id } = generatedWebhook;
const payload = {
response: {
webhookId: id,
statusCode: 400,
response: {
body: "",
status: 400,
},
},
errorMessage: "Error test",
};
const res = await client.post(`/webhook/${id}/status`, payload);
expect(res.status).toBe(204);
});

it("disallows setting webhook for another user", async () => {
const { id } = generatedWebhook;
const modifiedHook = { ...generatedWebhook, userId: nonAdminUser.id };
Expand Down Expand Up @@ -306,6 +290,29 @@ describe("controllers/webhook", () => {
expect(setActiveRes.status).toBe(204);
// const setActiveResJson = await setActiveRes.json()
// expect(setActiveResJson).toBeDefined()

// a log entry for the webhook should appear after a short wait
let requestsJson;
for (let i = 0; i < 5; i++) {
await sleep(500);
const requestsRes = await client.get(
`/webhook/${generatedWebhook.id}/requests`
);
expect(requestsRes.status).toBe(200);
requestsJson = await requestsRes.json();
if (requestsJson.length > 0) {
break;
}
}

expect(requestsJson.length).toBeGreaterThan(0);
expect(requestsJson[0].webhookId).toBe(generatedWebhook.id);
expect(requestsJson[0].event).toBe("stream.started");
expect(requestsJson[0].request).toBeDefined();
expect(requestsJson[0].request.body).toBeDefined();
expect(requestsJson[0].request.headers).toBeDefined();
expect(requestsJson[0].response).toBeDefined();
expect(requestsJson[0].response.body).toBeDefined();
}, 20000);

it("trigger webhook with localIP", async () => {
Expand Down
159 changes: 91 additions & 68 deletions packages/api/src/controllers/webhook.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import { makeNextHREF, parseFilters, parseOrder, FieldsMap } from "./helpers";
import { db } from "../store";
import sql from "sql-template-strings";
import { UnprocessableEntityError, NotFoundError } from "../store/errors";
import { WebhookStatusPayload } from "../schema/types";
import { DBWebhook } from "../store/webhook-table";

function validateWebhookPayload(id, userId, createdAt, payload) {
try {
Expand Down Expand Up @@ -46,7 +44,7 @@ const fieldsMap: FieldsMap = {
url: `webhook.data->>'url'`,
blocking: `webhook.data->'blocking'`,
deleted: `webhook.data->'deleted'`,
createdAt: `webhook.data->'createdAt'`,
createdAt: { val: `webhook.data->'createdAt'`, type: "int" },
userId: `webhook.data->>'userId'`,
"user.email": { val: `users.data->>'email'`, type: "full-text" },
sharedSecret: `webhook.data->>'sharedSecret'`,
Expand Down Expand Up @@ -96,7 +94,7 @@ app.get("/", authorizer({}), async (req, res) => {
const query = parseFilters(fieldsMap, filters);
query.push(sql`webhook.data->>'userId' = ${req.user.id}`);

if (!all || all === "false") {
if (!all || all === "false" || !req.user.admin) {
query.push(sql`webhook.data->>'deleted' IS NULL`);
}

Expand Down Expand Up @@ -234,70 +232,6 @@ app.patch(
}
);

app.post(
"/:id/status",
authorizer({ anyAdmin: true }),
validatePost("webhook-status-payload"),
async (req, res) => {
const { id } = req.params;
const webhook = await db.webhook.get(id);
if (!webhook || webhook.deleted) {
return res.status(404).json({ errors: ["webhook not found or deleted"] });
}

const { response, errorMessage } = req.body as WebhookStatusPayload;

if (!response || !response.response) {
return res.status(422).json({ errors: ["missing response in payload"] });
}
if (!response.response.body && response.response.body !== "") {
return res
.status(400)
.json({ errors: ["missing body in payload response"] });
}

try {
const triggerTime = response.createdAt ?? Date.now();
let status: DBWebhook["status"] = { lastTriggeredAt: triggerTime };

if (
response.statusCode >= 300 ||
!response.statusCode ||
response.statusCode === 0
) {
status = {
...status,
lastFailure: {
error: errorMessage,
timestamp: triggerTime,
statusCode: response.statusCode,
response: response.response.body,
},
};
}
await db.webhook.updateStatus(webhook.id, status);
} catch (e) {
console.log(
`Unable to store status of webhook ${webhook.id} url: ${webhook.url}`
);
}

// TODO : Change the response type and save the response making sure it's compatible object
/*await db.webhookResponse.create({
id: uuid(),
webhookId: webhook.id,
createdAt: Date.now(),
statusCode: response.statusCode,
response: {
body: response.response.body,
status: response.statusCode,
},
});*/

res.status(204).end();
}
);

app.delete("/:id", authorizer({}), async (req, res) => {
// delete a specific webhook
const webhook = await db.webhook.get(req.params.id);
Expand Down Expand Up @@ -345,4 +279,93 @@ app.delete("/", authorizer({}), async (req, res) => {
res.end();
});

const requestsFieldsMap: FieldsMap = {
id: `webhook_response.ID`,
createdAt: { val: `webhook_response.data->'createdAt'`, type: "int" },
userId: `webhook_response.data->>'userId'`,
event: `webhook_response.data->>'event'`,
statusCode: `webhook_response.data->'response'->>'status'`,
};

app.get("/:id/requests", authorizer({}), async (req, res) => {
let { limit, cursor, all, event, allUsers, order, filters, count } =
req.query;
if (isNaN(parseInt(limit))) {
limit = undefined;
}
if (!order) {
order = "createdAt-true";
}

if (req.user.admin && allUsers && allUsers !== "false") {
const query = parseFilters(requestsFieldsMap, filters);
query.push(sql`webhook_response.data->>'webhookId' = ${req.params.id}`);
if (!all || all === "false") {
query.push(sql`webhook_response.data->>'deleted' IS NULL`);
}

let fields =
" webhook_response.id as id, webhook_response.data as data, users.id as usersId, users.data as usersdata";
if (count) {
fields = fields + ", count(*) OVER() AS count";
}
const from = `webhook_response left join users on webhook_response.data->>'userId' = users.id`;
const [output, newCursor] = await db.webhookResponse.find(query, {
limit,
cursor,
fields,
from,
order: parseOrder(requestsFieldsMap, order),
process: ({ data, usersdata, count: c }) => {
if (count) {
res.set("X-Total-Count", c);
}
return { ...data, user: db.user.cleanWriteOnlyResponse(usersdata) };
},
});

res.status(200);

if (output.length > 0 && newCursor) {
res.links({ next: makeNextHREF(req, newCursor) });
}
return res.json(output);
}

const query = parseFilters(requestsFieldsMap, filters);
query.push(sql`webhook_response.data->>'userId' = ${req.user.id}`);
query.push(sql`webhook_response.data->>'webhookId' = ${req.params.id}`);

if (!all || all === "false" || !req.user.admin) {
query.push(sql`webhook_response.data->>'deleted' IS NULL`);
}

let fields = " webhook_response.id as id, webhook_response.data as data";
if (count) {
fields = fields + ", count(*) OVER() AS count";
}
const from = `webhook_response`;
const [output, newCursor] = await db.webhookResponse.find(query, {
limit,
cursor,
fields,
from,
order: parseOrder(requestsFieldsMap, order),
process: ({ data, count: c }) => {
if (count) {
res.set("X-Total-Count", c);
}
return { ...data };
},
});

res.status(200);

if (output.length > 0) {
res.links({ next: makeNextHREF(req, newCursor) });
}

return res.json(output);
});

export default app;
46 changes: 23 additions & 23 deletions packages/api/src/schema/api-schema.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,6 @@ components:
type: object
required:
- webhookId
- statusCode
additionalProperties: false
properties:
id:
Expand All @@ -229,39 +228,49 @@ components:
webhookId:
readOnly: true
type: string
eventId:
description: ID of the webhook this request was made for
event:
readOnly: true
type: string
description: The event type that triggered the webhook request
createdAt:
readOnly: true
type: number
description: |
Timestamp (in milliseconds) at which webhook response object was
Timestamp (in milliseconds) at which webhook request object was
created
example: 1587667174725
duration:
type: number
statusCode:
type: number
default: 0
description: The time taken (in seconds) to make the webhook request
request:
type: object
properties:
url:
type: string
description: URL used for the request
method:
type: string
description: HTTP request method
headers:
type: object
description: HTTP request headers
body:
type: string
description: request body
response:
type: object
additionalProperties: false
properties:
body:
type: string
headers:
type: object
additionalProperties:
type: array
items:
type: string
redirected:
type: boolean
description: response body
status:
type: number
description: HTTP status code
statusText:
type: string
description: response status text
clip-payload:
type: object
additionalProperties: false
Expand Down Expand Up @@ -614,15 +623,6 @@ components:
$ref: "#/components/schemas/webhook/properties/sharedSecret"
streamId:
$ref: "#/components/schemas/webhook/properties/streamId"
webhook-status-payload:
type: object
additionalProperties: false
properties:
errorMessage:
type: string
description: Error message if the webhook failed to process the event
response:
$ref: "#/components/schemas/webhook-response"
session:
type: object
required:
Expand Down
18 changes: 18 additions & 0 deletions packages/api/src/schema/db-schema.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -593,10 +593,28 @@ components:
readOnly: true
type: string
example: webhookResponse
userId:
readOnly: true
type: string
index: true
webhookId:
index: true
eventId:
type: string
index: true
writeOnly: true
event:
index: true
createdAt:
index: true
response:
type: object
properties:
status:
index: true
redirected:
type: boolean
writeOnly: true
detection-webhook-payload:
type: object
required:
Expand Down
Loading

0 comments on commit 3803bea

Please sign in to comment.