From a1c9dc6b523ad6ca6bd2b524255baab6c82d2030 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 13 Jan 2025 20:24:56 +0100 Subject: [PATCH 1/8] Assert headers for fetch helper --- library/helpers/fetch.test.ts | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/library/helpers/fetch.test.ts b/library/helpers/fetch.test.ts index fa1d0782..874ab152 100644 --- a/library/helpers/fetch.test.ts +++ b/library/helpers/fetch.test.ts @@ -2,6 +2,7 @@ import * as t from "tap"; import { createServer, Server } from "http"; import { fetch } from "./fetch"; import { gzip } from "zlib"; +import { getMajorNodeVersion } from "./getNodeVersion"; let server: Server; @@ -13,6 +14,7 @@ t.beforeEach(async () => { req.on("end", () => { const body = JSON.stringify({ method: req.method, + headers: req.headers, body: Buffer.concat(chunks).toString(), }); if (req.headers["accept-encoding"] === "gzip") { @@ -50,7 +52,14 @@ t.test("should make a GET request", async (t) => { const response = await fetch({ url }); t.equal(response.statusCode, 200); - t.same(JSON.parse(response.body), { method: "GET", body: "" }); + t.same(JSON.parse(response.body), { + method: "GET", + body: "", + headers: { + host: `localhost:${(server.address() as any).port}`, + connection: getMajorNodeVersion() >= 19 ? "keep-alive" : "close", + }, + }); }); t.test("should make a GET request with gzip", async (t) => { @@ -63,7 +72,15 @@ t.test("should make a GET request with gzip", async (t) => { }); t.equal(response.statusCode, 200); - t.same(JSON.parse(response.body), { method: "GET", body: "" }); + t.same(JSON.parse(response.body), { + method: "GET", + body: "", + headers: { + host: `localhost:${(server.address() as any).port}`, + connection: getMajorNodeVersion() >= 19 ? "keep-alive" : "close", + "accept-encoding": "gzip", + }, + }); }); t.test("should make a POST request with body", async (t) => { @@ -79,6 +96,12 @@ t.test("should make a POST request with body", async (t) => { t.same(JSON.parse(response.body), { method: "POST", body: '{"key":"value"}', + headers: { + host: `localhost:${(server.address() as any).port}`, + connection: getMajorNodeVersion() >= 19 ? "keep-alive" : "close", + "content-type": "application/json", + "content-length": "15", + }, }); }); @@ -98,5 +121,12 @@ t.test("should make a POST request with body and gzip", async (t) => { t.same(JSON.parse(response.body), { method: "POST", body: '{"key":"value"}', + headers: { + host: `localhost:${(server.address() as any).port}`, + connection: getMajorNodeVersion() >= 19 ? "keep-alive" : "close", + "content-type": "application/json", + "content-length": "15", + "accept-encoding": "gzip", + }, }); }); From d647c4a7e15e4812db6e5f60728a9b461fe9125f Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 13 Jan 2025 20:25:55 +0100 Subject: [PATCH 2/8] Pass body to `req.end(...)` instead of `req.write(...)` To avoid sending a chunked response we cannot use `req.write(...)` > Sends a chunk of the body. This method can be called multiple times. If no Content-Length is set, data will automatically be encoded in HTTP Chunked transfer encoding, so that server knows when the data ends. The Transfer-Encoding: chunked header is added. Calling request.end() is necessary to finish sending the request. Instead call `req.end(...)` with the body. This will set `Content-length` automatically. --- library/helpers/fetch.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/library/helpers/fetch.ts b/library/helpers/fetch.ts index 2be14934..62f6ff16 100644 --- a/library/helpers/fetch.ts +++ b/library/helpers/fetch.ts @@ -54,8 +54,7 @@ async function request({ reject(error); }); - req.write(body); - req.end(); + req.end(body); }); } From 9416f344b57b2513edd875e1aef926b810060a97 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 14 Jan 2025 00:11:24 +0100 Subject: [PATCH 3/8] Use t.fail(error) Better error formatting --- library/sources/HTTPServer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/sources/HTTPServer.test.ts b/library/sources/HTTPServer.test.ts index 992859c2..56354c61 100644 --- a/library/sources/HTTPServer.test.ts +++ b/library/sources/HTTPServer.test.ts @@ -455,7 +455,7 @@ t.test("it uses limit from AIKIDO_MAX_BODY_SIZE_MB", async (t) => { t.equal(response2.statusCode, 413); }) .catch((error) => { - t.fail(`Unexpected error: ${error.message} ${error.stack}`); + t.fail(error); }) .finally(() => { server.close(); From 0a87279a8616b7c8d974fdadf3c8c7b7fa9d21c7 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 14 Jan 2025 00:17:44 +0100 Subject: [PATCH 4/8] Use req.destroy() --- library/sources/http-server/readBodyStream.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/library/sources/http-server/readBodyStream.ts b/library/sources/http-server/readBodyStream.ts index 19b1eeb5..993045f0 100644 --- a/library/sources/http-server/readBodyStream.ts +++ b/library/sources/http-server/readBodyStream.ts @@ -28,7 +28,10 @@ export async function readBodyStream( if (bodySize + chunk.length > maxBodySize) { res.statusCode = 413; res.end( - "This request was aborted by Aikido firewall because the body size exceeded the maximum allowed size. Use AIKIDO_MAX_BODY_SIZE_MB to increase the limit." + "This request was aborted by Aikido firewall because the body size exceeded the maximum allowed size. Use AIKIDO_MAX_BODY_SIZE_MB to increase the limit.", + () => { + req.destroy(); + } ); agent.getInspectionStatistics().onAbortedRequest(); @@ -44,7 +47,10 @@ export async function readBodyStream( } catch { res.statusCode = 500; res.end( - "Aikido firewall encountered an error while reading the request body." + "Aikido firewall encountered an error while reading the request body.", + () => { + req.destroy(); + } ); return { From 428656c68e67302d47b7132cc66c797ef27399d8 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 14 Jan 2025 10:58:13 +0100 Subject: [PATCH 5/8] Use cURL to send requests to server --- library/sources/HTTPServer.test.ts | 77 +++++++++++++++---- library/sources/http-server/readBodyStream.ts | 2 +- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/library/sources/HTTPServer.test.ts b/library/sources/HTTPServer.test.ts index 56354c61..74372801 100644 --- a/library/sources/HTTPServer.test.ts +++ b/library/sources/HTTPServer.test.ts @@ -6,9 +6,14 @@ import { getContext } from "../agent/Context"; import { fetch } from "../helpers/fetch"; import { wrap } from "../helpers/wrap"; import { HTTPServer } from "./HTTPServer"; +import { join } from "path"; import { createTestAgent } from "../helpers/createTestAgent"; import type { Blocklist } from "../agent/api/fetchBlockedLists"; import * as fetchBlockedLists from "../agent/api/fetchBlockedLists"; +import { mkdtemp, writeFile, unlink } from "fs/promises"; +import { exec } from "child_process"; +import { promisify } from "util"; +const execAsync = promisify(exec); // Before require("http") const api = new ReportingAPIForTesting({ @@ -359,17 +364,51 @@ function generateJsonPayload(sizeInMb: number) { return JSON.stringify("a".repeat(sizeInBytes)); } -t.test("it sends 413 when body is larger than 20 Mb", async (t) => { +// Send request using curl +// Returns the response message + status code in stdout +// We need this because http.request throws EPIPE write error when the server closes the connection abruptly +// We cannot read the status code or the response message when that happens +async function sendUsingCurl({ + url, + method, + headers, + body, + timeoutInMS, +}: { + url: URL; + method: string; + headers: Record; + body: string; + timeoutInMS: number; +}) { + const headersString = Object.entries(headers) + .map(([key, value]) => `-H "${key}: ${value}"`) + .join(" "); + + const tmpDir = await mkdtemp("/tmp/aikido-"); + const tmpFile = join(tmpDir, "/body.json"); + await writeFile(tmpFile, body, { encoding: "utf-8" }); + + const { stdout } = await execAsync( + `curl -X ${method} ${headersString} -d @${tmpFile} ${url} --max-time ${timeoutInMS / 1000} --write-out "%{http_code}"` + ); + await unlink(tmpFile); + + return stdout; +} + +t.only("it sends 413 when body is larger than 20 Mb", async (t) => { // Enables body parsing process.env.NEXT_DEPLOYMENT_ID = ""; const server = http.createServer((req, res) => { t.fail(); + res.end(); }); await new Promise((resolve) => { server.listen(3320, () => { - fetch({ + sendUsingCurl({ url: new URL("http://localhost:3320"), method: "POST", headers: { @@ -377,15 +416,20 @@ t.test("it sends 413 when body is larger than 20 Mb", async (t) => { }, body: generateJsonPayload(21), timeoutInMS: 2000, - }).then(({ body, statusCode }) => { - t.equal( - body, - "This request was aborted by Aikido firewall because the body size exceeded the maximum allowed size. Use AIKIDO_MAX_BODY_SIZE_MB to increase the limit." - ); - t.equal(statusCode, 413); - server.close(); - resolve(); - }); + }) + .then((stdout) => { + t.equal( + stdout, + "This request was aborted by Aikido firewall because the body size exceeded the maximum allowed size. Use AIKIDO_MAX_BODY_SIZE_MB to increase the limit.413" + ); + }) + .catch((error) => { + t.fail(error); + }) + .finally(() => { + server.close(); + resolve(); + }); }); }); }); @@ -431,7 +475,7 @@ t.test("it uses limit from AIKIDO_MAX_BODY_SIZE_MB", async (t) => { server.listen(3322, () => { process.env.AIKIDO_MAX_BODY_SIZE_MB = "1"; Promise.all([ - fetch({ + sendUsingCurl({ url: new URL("http://localhost:3322"), method: "POST", headers: { @@ -440,7 +484,7 @@ t.test("it uses limit from AIKIDO_MAX_BODY_SIZE_MB", async (t) => { body: generateJsonPayload(1), timeoutInMS: 2000, }), - fetch({ + sendUsingCurl({ url: new URL("http://localhost:3322"), method: "POST", headers: { @@ -451,8 +495,11 @@ t.test("it uses limit from AIKIDO_MAX_BODY_SIZE_MB", async (t) => { }), ]) .then(([response1, response2]) => { - t.equal(response1.statusCode, 200); - t.equal(response2.statusCode, 413); + t.equal(response1, "200"); + t.same( + response2, + "This request was aborted by Aikido firewall because the body size exceeded the maximum allowed size. Use AIKIDO_MAX_BODY_SIZE_MB to increase the limit.413" + ); }) .catch((error) => { t.fail(error); diff --git a/library/sources/http-server/readBodyStream.ts b/library/sources/http-server/readBodyStream.ts index 993045f0..e59024dc 100644 --- a/library/sources/http-server/readBodyStream.ts +++ b/library/sources/http-server/readBodyStream.ts @@ -15,7 +15,7 @@ type BodyReadResult = export async function readBodyStream( req: IncomingMessage, - res: ServerResponse, + res: ServerResponse, agent: Agent ): Promise { let body = ""; From bd8662e3dc519d098b05e403d9c9351be554f95f Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 14 Jan 2025 11:59:21 +0100 Subject: [PATCH 6/8] Upgrade Zen internals to v0.1.35 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1f9a2d37..6254480b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -INTERNALS_VERSION = v0.1.34 +INTERNALS_VERSION = v0.1.35 INTERNALS_URL = https://github.com/AikidoSec/zen-internals/releases/download/$(INTERNALS_VERSION) TARBALL = zen_internals.tgz CHECKSUM_FILE = zen_internals.tgz.sha256sum From 53848eff065d305b30b2c3bf2279fabf6c248f48 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 14 Jan 2025 13:11:02 +0100 Subject: [PATCH 7/8] Use separate table for postgres sink tests Postgresjs.test.ts and Postgres.test.ts use the same `cats` table They sometimes fail when they run at the same time --- library/sinks/Postgresjs.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/library/sinks/Postgresjs.test.ts b/library/sinks/Postgresjs.test.ts index 395def97..56ce945a 100644 --- a/library/sinks/Postgresjs.test.ts +++ b/library/sinks/Postgresjs.test.ts @@ -27,11 +27,11 @@ t.test("it inspects query method calls and blocks if needed", async (t) => { try { await sql` - CREATE TABLE IF NOT EXISTS cats ( + CREATE TABLE IF NOT EXISTS cats_2 ( petname varchar(255) ); `; - await sql`TRUNCATE cats`; + await sql`TRUNCATE cats_2`; const cats = [ { @@ -42,23 +42,23 @@ t.test("it inspects query method calls and blocks if needed", async (t) => { }, ]; - await sql`insert into cats ${sql(cats, "petname")}`; + await sql`insert into cats_2 ${sql(cats, "petname")}`; const transactionResult = await sql.begin((sql) => [ - sql`SELECT * FROM cats`, + sql`SELECT * FROM cats_2`, ]); t.same(transactionResult[0], cats); - t.same(await sql`select * from ${sql("cats")}`, cats); - t.same(await sql.unsafe("SELECT * FROM cats"), cats); + t.same(await sql`select * from ${sql("cats_2")}`, cats); + t.same(await sql.unsafe("SELECT * FROM cats_2"), cats); await runWithContext(context, async () => { - t.same(await sql`select * from ${sql("cats")}`, cats); - t.same(await sql.unsafe("SELECT * FROM cats"), cats); + t.same(await sql`select * from ${sql("cats_2")}`, cats); + t.same(await sql.unsafe("SELECT * FROM cats_2"), cats); const error = await t.rejects(async () => { await sql.unsafe( - `SELECT * FROM cats WHERE petname = test; -- should be blocked` + `SELECT * FROM cats_2 WHERE petname = test; -- should be blocked` ); }); From 41b770ecb875fd2470cf6d83c8c88d5f93c5e139 Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 15 Jan 2025 12:10:11 +0100 Subject: [PATCH 8/8] chore: added reference to env vars --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4739d7ff..afdebd1f 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,10 @@ To block requests, set the `AIKIDO_BLOCK` environment variable to `true`. See [Reporting to Aikido](#reporting-to-your-aikido-security-dashboard) to learn how to send events to Aikido. +## Additional configuration + +[Configure Zen using environment variables for authentication, mode settings, debugging, and more.](https://help.aikido.dev/doc/configuration-via-env-vars/docrSItUkeR9) + ## License This program is offered under a commercial and under the AGPL license.