diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a075075..d3e68c2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,11 +24,17 @@ jobs: # Install packages yarn - # Create the namespace if it doesn't exist + # KV + ## Create the namespace if it doesn't exist wrangler2 kv:namespace list | grep -q GmodExpress || wrangler2 kv:namespace create GmodExpress - # Get the namespace ID + ## Get the namespace ID namespaceid=$(wrangler2 kv:namespace list | jq '.[] | select(.title=="gmod-express-GmodExpress") | .id') - # Add the binding to the wrangler.toml + ## Add the binding to the wrangler.toml echo "kv_namespaces = [ { binding = 'GmodExpress', id = $namespaceid } ]" >> wrangler.toml + + + # R2 + wrangler2 r2 bucket list | grep -q '"express-v1"' || wrangler2 r2 bucket create express-v1 + echo "r2_buckets = [ { binding = 'ExpressV1Bucket', bucket_name = 'express-v1' } ]" >> wrangler.toml diff --git a/.gitignore b/.gitignore index 239ecff..5ed9508 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +.env node_modules yarn.lock +wrangler.*.toml diff --git a/README.md b/README.md index e8e6edf..547d5ee 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,22 @@ It should only take a couple of minutes, just click this button! (more instructi ![chrome_5xgZ8Z0zRg](https://user-images.githubusercontent.com/7936439/202330307-8756142d-42e5-4e85-919a-1e4c335afff3.png) +

Configuring Bucket Lifecycle (important!)

+Due to some technical weirdness with Cloudflare's KV, we also use Cloudflare's R2 storage system as a backup. +In the event that a client can't read a KV value, it will read it from R2 instead (slower, but R2 is "strongly consistent" so it's a reliable way to get the data). + +However, R2 doesn't have a built-in way to expire documents. + +We can use the S3 API to configure an "Object Lifecycle" on our backup bucket which will expire keys at the same cadence as KV. + +If you don't do this, your R2 bucket will fill up indefinitely! + +Here's how you do it: +1. Wait for the app to deploy at least once (it will create the Bucket for you) +2. Visit the Cloudflare Dashboard, click on "R2" +3. Follow along with these images to set up an Expiration lifecycle: + +
diff --git a/docker/selfhost.js b/docker/selfhost.js index 32abf3d..bb45196 100644 --- a/docker/selfhost.js +++ b/docker/selfhost.js @@ -29,11 +29,23 @@ const expressStub = { } } +async function noop() { + return new Promise((resolve, reject) => { + resolve(null) + }) +} + +const bucketStub = { + get: noop, + put: noop +} + const app = new Hono() app.use("*", logger()) app.use("*", async (c, next) => { c.env.GmodExpress = expressStub + c.env.ExpressV1Bucket = bucketStub await next() }) diff --git a/index.js b/index.js index a705547..5807d47 100644 --- a/index.js +++ b/index.js @@ -37,7 +37,6 @@ const parseRange = (total, range) => { }); } - async function validateRequest(c, token) { // TODO: Do a sanity check on expiration time too const expected = await c.env.GmodExpress.get(`token:${token}` ) @@ -46,12 +45,13 @@ async function validateRequest(c, token) { async function putData(c, data) { const id = makeUUID() - const metadata = makeMetadata(c) try { + const size = data.byteLength.toString() + await Promise.all([ - c.env.GmodExpress.put(`size:${id}`, data.byteLength.toString(), metadata), - c.env.GmodExpress.put(`data:${id}`, data, { ...metadata, type: "arrayBuffer" }) + c.env.ExpressV1Bucket.put(`data:${id}`, data), + c.env.ExpressV1Bucket.put(`size:${id}`, size) ]) } catch (e) { console.log("Failed to put data", e) @@ -67,11 +67,23 @@ async function putToken(c, token) { } async function getData(c, id) { - return await c.env.GmodExpress.get(`data:${id}`, { type: "arrayBuffer" }) + const data = await c.env.ExpressV1Bucket.get(`data:${id}`) + + if (data === null) { + return null + } + + return await data.arrayBuffer() } async function getSize(c, id) { - return await c.env.GmodExpress.get(`size:${id}`) + const size = await c.env.ExpressV1Bucket.get(`size:${id}`) + + if (size === null) { + return null + } + + return parseInt(await size.text()) } async function registerRequest(c) { @@ -100,11 +112,13 @@ async function readRequest(c) { } const id = c.req.param("id") - let data = await getData(c, id) - if (data === null) { - console.log("No data found for id", id) + const data = await getData(c, id) + if (data == null) { + console.log("No data found for id in Bucket", id) return c.notFound() } + + console.log("Found data in bucket when KV missed", id) const fullSize = data.byteLength @@ -141,10 +155,10 @@ async function readSizeRequest(c) { } const id = c.req.param("id") - const size = await getSize(c, id) + let size = await getSize(c, id) if (size === null) { - console.log("No size found for id", id) - return c.text("Size not found", 404) + console.log("No size found for id in Bucket", id) + return c.notFound() } return c.json({size: size}) @@ -171,6 +185,7 @@ async function writeRequest(c) { } app.get("/", async () => Response.redirect("https://github.com/CFC-Servers/gm_express", 302)); +app.get("/discord", async () => Response.redirect("https://discord.gg/5JUqZjzmYJ", 302)); // V1 Routes app.get("/v1/register", registerRequest) diff --git a/wrangler.toml b/wrangler.toml index e34ec3e..4d590f7 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,7 +2,7 @@ name = "gmod-express" main = "./index.js" workers_dev = true -compatibility_date = "2022-11-05" +compatibility_date = "2023-06-22" rules = [ { type = "ESModule", globs = ["setup_app.js"] } ]