Merge branch 'main' into unkey-rails-template

.github/ISSUE_TEMPLATE/ossgg_sidequest.yml

@@ -2,8 +2,9 @@ name: oss.gg hack submission 🕹️
 description: "Submit your contribution for the for the oss.gg hackathon"
 title: "[oss.gg hackathon]"
 labels: 
-  - "🕹️ oss.gg, player submission"
+  - "🕹️ oss.gg"
   - "Needs Approval"
+  - "player submission"
 
 assignees: []
 body:

.github/workflows/pr-alerts-campsite.yml

@@ -0,0 +1,53 @@
+name: Campsite PR Alerts
+
+on:
+  pull_request:
+    types: [opened, closed, merged, ready_for_review, reopened]
+
+jobs:
+  post_to_campsite:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '20'
+
+      - name: Post to Campsite
+        id: post_to_campsite
+        run: |
+          if [[ "${{ github.event.pull_request.draft }}" == "true" ]]; then
+            STATUS_EMOJI="⚪"
+          elif [[ "${{ github.event.action }}" == "opened" || "${{ github.event.action }}" == "reopened" || "${{ github.event.action }}" == "ready_for_review" ]]; then
+            STATUS_EMOJI="🟢"
+          elif [[ "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then
+            STATUS_EMOJI="🟣"
+          elif [[ "${{ github.event.action }}" == "closed" ]]; then
+            STATUS_EMOJI="🔴"
+          fi
+
+          ACTION=${{ github.event.action }}
+          if [[ "${ACTION}" == "ready_for_review" ]]; then
+            ACTION="ready for review"
+          elif [[ "${ACTION}" == "reopened" ]]; then
+            ACTION="reopened"
+          elif [[ "${ACTION}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then
+            ACTION="merged"
+          fi
+
+          ESCAPED_TITLE=$(echo "${{ github.event.pull_request.title }}" | jq -Rr @json)
+
+          CONTENT="${STATUS_EMOJI} Pull request ${ACTION} by ${{ github.event.pull_request.user.login }}: [#${{ github.event.pull_request.number }} ${ESCAPED_TITLE}](${{ github.event.pull_request.html_url }})"
+
+          echo "content=${CONTENT}" >> $GITHUB_OUTPUT
+      - name: Create Campsite message
+        uses: campsite/campsite-github-action@v1
+        with:
+          api_key: ${{ secrets.CAMPSITE_API_KEY }}
+          action_type: create_message
+          thread_id: ${{ secrets.CAMPSITE_PR_ALERTS_THREAD_ID }}
+          content: ${{ steps.post_to_campsite.outputs.content }}

.gitignore

@@ -19,4 +19,7 @@ dist
 .dev.vars
 .wrangler
 .vitest
-.react-email
+.react-email
+
+.secrets.json
+secrets.ts

apps/agent/integration/identities/ratelimits_with_cost_load_test.go

@@ -0,0 +1,146 @@
+package identities_test
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+	unkey "github.com/unkeyed/unkey-go"
+	"github.com/unkeyed/unkey-go/models/components"
+	"github.com/unkeyed/unkey-go/models/operations"
+	attack "github.com/unkeyed/unkey/apps/agent/pkg/testutil"
+	"github.com/unkeyed/unkey/apps/agent/pkg/uid"
+	"github.com/unkeyed/unkey/apps/agent/pkg/util"
+)
+
+func TestIdentityRatelimitsWithCost0Accuracy(t *testing.T) {
+	// Step 1 --------------------------------------------------------------------
+	// Setup the sdk, create an API
+	// ---------------------------------------------------------------------------
+
+	ctx := context.Background()
+	rootKey := os.Getenv("INTEGRATION_TEST_ROOT_KEY")
+	require.NotEmpty(t, rootKey, "INTEGRATION_TEST_ROOT_KEY must be set")
+	baseURL := os.Getenv("UNKEY_BASE_URL")
+	require.NotEmpty(t, baseURL, "UNKEY_BASE_URL must be set")
+
+	options := []unkey.SDKOption{
+		unkey.WithSecurity(rootKey),
+	}
+
+	if baseURL != "" {
+		options = append(options, unkey.WithServerURL(baseURL))
+	}
+	sdk := unkey.New(options...)
+
+	for _, tc := range []struct {
+		rate         attack.Rate
+		testDuration time.Duration
+	}{
+		{
+			rate:         attack.Rate{Freq: 100, Per: time.Second},
+			testDuration: 1 * time.Minute,
+		},
+		{
+			rate:         attack.Rate{Freq: 100, Per: time.Second},
+			testDuration: 5 * time.Minute,
+		},
+		{
+			rate:         attack.Rate{Freq: 100, Per: time.Second},
+			testDuration: 30 * time.Minute,
+		},
+	} {
+		t.Run(fmt.Sprintf("[%s] over %s", tc.rate.String(), tc.testDuration), func(t *testing.T) {
+			api, err := sdk.Apis.CreateAPI(ctx, operations.CreateAPIRequestBody{
+				Name: uid.New("testapi"),
+			})
+			require.NoError(t, err)
+
+			// Step 2 --------------------------------------------------------------------
+			// Create the identity with ratelimits
+			// ---------------------------------------------------------------------------
+
+			ratelimit := operations.Ratelimits{
+				Name:     "ratelimit-a",
+				Limit:    600,
+				Duration: time.Minute.Milliseconds(),
+			}
+
+			externalID := uuid.NewString()
+			_, err = sdk.Identities.CreateIdentity(ctx, operations.CreateIdentityRequestBody{
+				ExternalID: externalID,
+				Ratelimits: []operations.Ratelimits{
+					ratelimit,
+				},
+			})
+			require.NoError(t, err)
+
+			// Step 3 --------------------------------------------------------------------
+			// Create key for this identity
+			// ---------------------------------------------------------------------------
+
+			key, err := sdk.Keys.CreateKey(ctx, operations.CreateKeyRequestBody{
+				APIID:      api.Object.APIID,
+				ExternalID: util.Pointer(externalID),
+			})
+			require.NoError(t, err)
+
+			// Step 5 --------------------------------------------------------------------
+			// Test ratelimits
+			// ---------------------------------------------------------------------------
+
+			total := 0
+			passed := 0
+			withCost := 0
+			errors := 0
+
+			results := attack.Attack(t, tc.rate, tc.testDuration, func() bool {
+
+				cost := int64(0)
+				if rand.Intn(100) == 0 {
+					withCost++
+					cost = 1
+				}
+
+				res, err := sdk.Keys.VerifyKey(context.Background(), components.V1KeysVerifyKeyRequest{
+					APIID: unkey.String(api.Object.APIID),
+					Key:   key.Object.Key,
+					Ratelimits: []components.Ratelimits{
+						{Name: ratelimit.Name,
+							Cost: util.Pointer(cost),
+						},
+					},
+				})
+				if err != nil {
+					errors++
+					return false
+				}
+
+				return res.V1KeysVerifyKeyResponse.Valid
+
+			})
+
+			for valid := range results {
+				total++
+				if valid {
+					passed++
+				}
+
+			}
+
+			// Step 6 --------------------------------------------------------------------
+			// Assert ratelimits worked
+			// ---------------------------------------------------------------------------
+
+			t.Logf("Total: %d, Passed: %d, withCost=1: %d", total, passed, withCost)
+
+			// check requests::api is not exceeded
+		})
+
+	}
+}

apps/api/CHANGELOG.md

@@ -1,5 +1,11 @@
 # api
 
+## 0.1.0
+
+### Minor Changes
+
+- 09d36ad: add /v1/keys.whoami route
+
 ## 0.0.14
 
 ### Patch Changes

apps/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "api",
-  "version": "0.0.14",
+  "version": "0.1.0",
   "private": true,
   "scripts": {
     "build": "tsc",
@@ -18,7 +18,7 @@
     "@vitest/ui": "^1.6.0",
     "typescript": "^5.5.3",
     "vitest": "^1.6.0",
-    "wrangler": "^3.62.0"
+    "wrangler": "^3.80.5"
   },
   "dependencies": {
     "@axiomhq/js": "1.0.0-rc.2",
@@ -27,6 +27,7 @@
     "@hono/zod-validator": "^0.2.1",
     "@planetscale/database": "^1.16.0",
     "@unkey/cache": "workspace:^",
+    "@unkey/clickhouse-zod": "workspace:^",
     "@unkey/db": "workspace:^",
     "@unkey/encryption": "workspace:^",
     "@unkey/error": "workspace:^",
@@ -36,7 +37,6 @@
     "@unkey/logs": "workspace:^",
     "@unkey/metrics": "workspace:^",
     "@unkey/rbac": "workspace:^",
-    "@unkey/clickhouse-zod": "workspace:^",
     "@unkey/schema": "workspace:^",
     "@unkey/worker-logging": "workspace:^",
     "hono": "^4.6.3",

apps/api/src/pkg/env.ts

@@ -1,6 +1,10 @@
 import { z } from "zod";
 import type { MessageBody } from "./key_migration/message";
 
+export const cloudflareRatelimiter = z.custom<{
+  limit: (opts: { key: string }) => Promise<{ success: boolean }>;
+}>((r) => typeof r.limit === "function");
+
 export const zEnv = z.object({
   VERSION: z.string().default("unknown"),
   DATABASE_HOST: z.string(),
@@ -42,6 +46,11 @@ export const zEnv = z.object({
         return 0;
       }
     }),
+  RL_10_60s: cloudflareRatelimiter,
+  RL_30_60s: cloudflareRatelimiter,
+  RL_200_60s: cloudflareRatelimiter,
+  RL_500_10s: cloudflareRatelimiter,
+  RL_200_10s: cloudflareRatelimiter,
 });
 
 export type Env = z.infer<typeof zEnv>;

apps/api/src/pkg/key_migration/handler.ts

@@ -96,6 +96,7 @@ export async function migrateKey(
         expires: message.expires ? new Date(message.expires) : null,
         refillInterval: message.refill?.interval,
         refillAmount: message.refill?.amount,
+        refillDay: message.refill?.refillDay,
         enabled: message.enabled,
         remaining: message.remaining,
         ratelimitAsync: message.ratelimit?.async,

apps/api/src/pkg/key_migration/message.ts

@@ -14,7 +14,7 @@ export type MessageBody = {
   permissions?: string[];
   expires?: number;
   remaining?: number;
-  refill?: { interval: "daily" | "monthly"; amount: number };
+  refill?: { interval: "daily" | "monthly"; amount: number; refillDay?: number };
   ratelimit?: { async: boolean; limit: number; duration: number };
   enabled: boolean;
   environment?: string;