diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index a075075..d3e68c2 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -24,11 +24,17 @@ jobs:
# Install packages
yarn
- # Create the namespace if it doesn't exist
+ # KV
+ ## Create the namespace if it doesn't exist
wrangler2 kv:namespace list | grep -q GmodExpress || wrangler2 kv:namespace create GmodExpress
- # Get the namespace ID
+ ## Get the namespace ID
namespaceid=$(wrangler2 kv:namespace list | jq '.[] | select(.title=="gmod-express-GmodExpress") | .id')
- # Add the binding to the wrangler.toml
+ ## Add the binding to the wrangler.toml
echo "kv_namespaces = [ { binding = 'GmodExpress', id = $namespaceid } ]" >> wrangler.toml
+
+
+ # R2
+ wrangler2 r2 bucket list | grep -q '"express-v1"' || wrangler2 r2 bucket create express-v1
+ echo "r2_buckets = [ { binding = 'ExpressV1Bucket', bucket_name = 'express-v1' } ]" >> wrangler.toml
diff --git a/.gitignore b/.gitignore
index 239ecff..5ed9508 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
+.env
node_modules
yarn.lock
+wrangler.*.toml
diff --git a/README.md b/README.md
index e8e6edf..547d5ee 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,22 @@ It should only take a couple of minutes, just click this button! (more instructi
![chrome_5xgZ8Z0zRg](https://user-images.githubusercontent.com/7936439/202330307-8756142d-42e5-4e85-919a-1e4c335afff3.png)
+
Configuring Bucket Lifecycle (important!)
+Due to some technical weirdness with Cloudflare's KV, we also use Cloudflare's R2 storage system as a backup.
+In the event that a client can't read a KV value, it will read it from R2 instead (slower, but R2 is "strongly consistent" so it's a reliable way to get the data).
+
+However, R2 doesn't have a built-in way to expire documents.
+
+We can use the S3 API to configure an "Object Lifecycle" on our backup bucket which will expire keys at the same cadence as KV.
+
+If you don't do this, your R2 bucket will fill up indefinitely!
+
+Here's how you do it:
+1. Wait for the app to deploy at least once (it will create the Bucket for you)
+2. Visit the Cloudflare Dashboard, click on "R2"
+3. Follow along with these images to set up an Expiration lifecycle:
+
+
diff --git a/docker/selfhost.js b/docker/selfhost.js
index 32abf3d..bb45196 100644
--- a/docker/selfhost.js
+++ b/docker/selfhost.js
@@ -29,11 +29,23 @@ const expressStub = {
}
}
+async function noop() {
+ return new Promise((resolve, reject) => {
+ resolve(null)
+ })
+}
+
+const bucketStub = {
+ get: noop,
+ put: noop
+}
+
const app = new Hono()
app.use("*", logger())
app.use("*", async (c, next) => {
c.env.GmodExpress = expressStub
+ c.env.ExpressV1Bucket = bucketStub
await next()
})
diff --git a/index.js b/index.js
index a705547..5807d47 100644
--- a/index.js
+++ b/index.js
@@ -37,7 +37,6 @@ const parseRange = (total, range) => {
});
}
-
async function validateRequest(c, token) {
// TODO: Do a sanity check on expiration time too
const expected = await c.env.GmodExpress.get(`token:${token}` )
@@ -46,12 +45,13 @@ async function validateRequest(c, token) {
async function putData(c, data) {
const id = makeUUID()
- const metadata = makeMetadata(c)
try {
+ const size = data.byteLength.toString()
+
await Promise.all([
- c.env.GmodExpress.put(`size:${id}`, data.byteLength.toString(), metadata),
- c.env.GmodExpress.put(`data:${id}`, data, { ...metadata, type: "arrayBuffer" })
+ c.env.ExpressV1Bucket.put(`data:${id}`, data),
+ c.env.ExpressV1Bucket.put(`size:${id}`, size)
])
} catch (e) {
console.log("Failed to put data", e)
@@ -67,11 +67,23 @@ async function putToken(c, token) {
}
async function getData(c, id) {
- return await c.env.GmodExpress.get(`data:${id}`, { type: "arrayBuffer" })
+ const data = await c.env.ExpressV1Bucket.get(`data:${id}`)
+
+ if (data === null) {
+ return null
+ }
+
+ return await data.arrayBuffer()
}
async function getSize(c, id) {
- return await c.env.GmodExpress.get(`size:${id}`)
+ const size = await c.env.ExpressV1Bucket.get(`size:${id}`)
+
+ if (size === null) {
+ return null
+ }
+
+ return parseInt(await size.text())
}
async function registerRequest(c) {
@@ -100,11 +112,13 @@ async function readRequest(c) {
}
const id = c.req.param("id")
- let data = await getData(c, id)
- if (data === null) {
- console.log("No data found for id", id)
+ const data = await getData(c, id)
+ if (data == null) {
+ console.log("No data found for id in Bucket", id)
return c.notFound()
}
+
+ console.log("Found data in bucket when KV missed", id)
const fullSize = data.byteLength
@@ -141,10 +155,10 @@ async function readSizeRequest(c) {
}
const id = c.req.param("id")
- const size = await getSize(c, id)
+ let size = await getSize(c, id)
if (size === null) {
- console.log("No size found for id", id)
- return c.text("Size not found", 404)
+ console.log("No size found for id in Bucket", id)
+ return c.notFound()
}
return c.json({size: size})
@@ -171,6 +185,7 @@ async function writeRequest(c) {
}
app.get("/", async () => Response.redirect("https://github.com/CFC-Servers/gm_express", 302));
+app.get("/discord", async () => Response.redirect("https://discord.gg/5JUqZjzmYJ", 302));
// V1 Routes
app.get("/v1/register", registerRequest)
diff --git a/wrangler.toml b/wrangler.toml
index e34ec3e..4d590f7 100644
--- a/wrangler.toml
+++ b/wrangler.toml
@@ -2,7 +2,7 @@ name = "gmod-express"
main = "./index.js"
workers_dev = true
-compatibility_date = "2022-11-05"
+compatibility_date = "2023-06-22"
rules = [
{ type = "ESModule", globs = ["setup_app.js"] }
]