Add R2 fallback for strange 404s

.github/workflows/deploy.yml

@@ -24,11 +24,17 @@ jobs:
             # Install packages
             yarn
 
-            # Create the namespace if it doesn't exist
+            # KV
+            ## Create the namespace if it doesn't exist
             wrangler2 kv:namespace list | grep -q GmodExpress || wrangler2 kv:namespace create GmodExpress
 
-            # Get the namespace ID
+            ## Get the namespace ID
             namespaceid=$(wrangler2 kv:namespace list | jq '.[] | select(.title=="gmod-express-GmodExpress") | .id')
 
-            # Add the binding to the wrangler.toml
+            ## Add the binding to the wrangler.toml
             echo "kv_namespaces = [ { binding = 'GmodExpress', id = $namespaceid } ]" >> wrangler.toml
+
+
+            # R2
+            wrangler2 r2 bucket list | grep -q '"express-v1"' || wrangler2 r2 bucket create express-v1
+            echo "r2_buckets = [ { binding = 'ExpressV1Bucket', bucket_name = 'express-v1' } ]" >> wrangler.toml

.gitignore

@@ -1,2 +1,4 @@
+.env
 node_modules
 yarn.lock
+wrangler.*.toml

README.md

@@ -31,6 +31,22 @@ It should only take a couple of minutes, just click this button! (more instructi
 
 ![chrome_5xgZ8Z0zRg](https://user-images.githubusercontent.com/7936439/202330307-8756142d-42e5-4e85-919a-1e4c335afff3.png)
 
+<h4>Configuring Bucket Lifecycle (important!)</h4>
+Due to some technical weirdness with Cloudflare's KV, we also use Cloudflare's R2 storage system as a backup.
+In the event that a client can't read a KV value, it will read it from R2 instead (slower, but R2 is "strongly consistent" so it's a reliable way to get the data).
+
+However, R2 doesn't have a built-in way to expire documents.
+
+We can use the S3 API to configure an "Object Lifecycle" on our backup bucket which will expire keys at the same cadence as KV.
+
+<strong>If you don't do this, your R2 bucket will fill up indefinitely!</strong>
+
+Here's how you do it:
+1. Wait for the app to deploy at least once (it will create the Bucket for you)
+2. Visit the Cloudflare Dashboard, click on "R2"
+3. Follow along with these images to set up an Expiration lifecycle:
+
+
 </details>
 
 <details>

docker/selfhost.js

@@ -29,11 +29,23 @@ const expressStub = {
   }
 }
 
+async function noop() {
+  return new Promise((resolve, reject) => {
+    resolve(null)
+  })
+}
+
+const bucketStub = {
+  get: noop,
+  put: noop
+}
+
 const app = new Hono()
 
 app.use("*", logger())
 app.use("*", async (c, next) => {
   c.env.GmodExpress = expressStub
+  c.env.ExpressV1Bucket = bucketStub
   await next()
 })
 

index.js

@@ -37,7 +37,6 @@ const parseRange = (total, range) => {
     });
 }
 
-
 async function validateRequest(c, token) {
   // TODO: Do a sanity check on expiration time too
   const expected = await c.env.GmodExpress.get(`token:${token}` )
@@ -46,12 +45,13 @@ async function validateRequest(c, token) {
 
 async function putData(c, data) {
   const id = makeUUID()
-  const metadata = makeMetadata(c)
 
   try {
+    const size = data.byteLength.toString()
+
     await Promise.all([
-      c.env.GmodExpress.put(`size:${id}`, data.byteLength.toString(), metadata),
-      c.env.GmodExpress.put(`data:${id}`, data, { ...metadata, type: "arrayBuffer" })
+      c.env.ExpressV1Bucket.put(`data:${id}`, data),
+      c.env.ExpressV1Bucket.put(`size:${id}`, size)
     ])
   } catch (e) {
     console.log("Failed to put data", e)
@@ -67,11 +67,23 @@ async function putToken(c, token) {
 }
 
 async function getData(c, id) {
-  return await c.env.GmodExpress.get(`data:${id}`, { type: "arrayBuffer" })
+  const data = await c.env.ExpressV1Bucket.get(`data:${id}`)
+
+  if (data === null) {
+    return null
+  }
+
+  return await data.arrayBuffer()
 }
 
 async function getSize(c, id) {
-  return await c.env.GmodExpress.get(`size:${id}`)
+  const size = await c.env.ExpressV1Bucket.get(`size:${id}`)
+
+  if (size === null) {
+    return null
+  }
+
+  return parseInt(await size.text())
 }
 
 async function registerRequest(c) {
@@ -100,11 +112,13 @@ async function readRequest(c) {
   }
 
   const id = c.req.param("id")
-  let data = await getData(c, id)
-  if (data === null) {
-    console.log("No data found for id", id)
+  const data = await getData(c, id)
+  if (data == null) {
+    console.log("No data found for id in Bucket", id)
     return c.notFound()
   }
+
+  console.log("Found data in bucket when KV missed", id)
 
   const fullSize = data.byteLength
 
@@ -141,10 +155,10 @@ async function readSizeRequest(c) {
   }
 
   const id = c.req.param("id")
-  const size = await getSize(c, id)
+  let size = await getSize(c, id)
   if (size === null) {
-    console.log("No size found for id", id)
-    return c.text("Size not found", 404)
+    console.log("No size found for id in Bucket", id)
+    return c.notFound()
   }
 
   return c.json({size: size})
@@ -171,6 +185,7 @@ async function writeRequest(c) {
 }
 
 app.get("/", async () => Response.redirect("https://github.com/CFC-Servers/gm_express", 302));
+app.get("/discord", async () => Response.redirect("https://discord.gg/5JUqZjzmYJ", 302));
 
 // V1 Routes
 app.get("/v1/register", registerRequest)

wrangler.toml

@@ -2,7 +2,7 @@ name = "gmod-express"
 main = "./index.js"
 workers_dev = true
 
-compatibility_date = "2022-11-05"
+compatibility_date = "2023-06-22"
 rules = [
   { type = "ESModule", globs = ["setup_app.js"] }
 ]