Open-source pg-sharding

.eslintrc.js

@@ -1,5 +1,5 @@
 "use strict";
-const config = require("../../.eslintrc.base.js")(__dirname);
+const config = require("./.eslintrc.base.js")(__dirname);
 config.rules["import/no-extraneous-dependencies"] = "error";
 config.rules["@typescript-eslint/explicit-function-return-type"] = [
   "error",

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: "CI Full Run"
+on:
+  pull_request:
+    branches:
+      - main
+      - grok/*/*
+  push:
+    branches:
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: ["20.x"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: npm install
+      - run: npm run build
+      - run: npm run test
+      - run: npm run lint

.github/workflows/semgrep.yml

@@ -0,0 +1,36 @@
+# Name of this GitHub Actions workflow.
+name: Semgrep
+
+on:
+    # Scan changed files in PRs (diff-aware scanning):
+    pull_request:
+        branches: ['main']
+
+    # Schedule the CI job (this method uses cron syntax):
+    schedule:
+        - cron: '0 0 * * MON-FRI'
+
+jobs:
+    semgrep:
+        # User definable name of this GitHub Actions job.
+        name: Scan
+        # If you are self-hosting, change the following `runs-on` value:
+        runs-on: ubuntu-latest
+
+        container:
+            # A Docker image with Semgrep installed. Do not change this.
+            image: returntocorp/semgrep@sha256:6c7ab81e4d1fd25a09f89f1bd52c984ce107c6ff33affef6ca3bc626a4cc479b
+
+        # Skip any PR created by dependabot to avoid permission issues:
+        if: (github.actor != 'dependabot[bot]')
+
+        steps:
+            # Fetch project source with GitHub Actions Checkout.
+            - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+            # Run the "semgrep ci" command on the command line of the docker image.
+            - run: semgrep ci
+              env:
+                  # Connect to Semgrep Cloud Platform through your SEMGREP_APP_TOKEN.
+                  # Generate a token from Semgrep Cloud Platform > Settings
+                  # and add it to your GitHub secrets.
+                  SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}

.gitignore

@@ -0,0 +1,9 @@
+dist
+
+node_modules
+package-lock.json
+yarn.lock
+.DS_Store
+*.log
+*.tmp
+*.swp

.npmignore

@@ -0,0 +1,12 @@
+__tests__
+.npmrc
+tsconfig.tsbuildinfo
+.github
+
+node_modules
+package-lock.json
+yarn.lock
+.DS_Store
+*.log
+*.tmp
+*.swp

.npmrc

@@ -0,0 +1 @@
+# Published to https://www.npmjs.com

README.md

@@ -1,4 +1,4 @@
-# pg-sharding: micro-shards support for PostgreSQL
+# @clickup/pg-sharding: micro-shards support for PostgreSQL
 
 Each micro-shard is a PG schema with numeric suffix. Micro-shards have the same
 set of tables with same names; it's up to the higher-level tools to keep the

SECURITY.md

@@ -0,0 +1,39 @@
+# Security
+
+Keeping our clients' data secure is an absolute top priority at ClickUp. Our goal is to provide a secure environment, while also being mindful of application performance and the overall user experience.
+
+ClickUp believes effective disclosure of security vulnerabilities requires mutual trust, respect, transparency and common good between ClickUp and Security Researchers. Together, our vigilant expertise promotes the continued security and privacy of ClickUp customers, products, and services.
+
+If you think you've found a security vulnerability in any ClickUp-owned repository, please let us know as outlined below.
+
+ClickUp defines a security vulnerability as an unintended weakness or exposure that could be used to compromise the integrity, availability or confidentiality of our products and services.
+
+## Our Commitment to Reporters
+
+- **Trust**. We maintain trust and confidentiality in our professional exchanges with security researchers.
+- **Respect**. We treat all researchers with respect and recognize your contribution for keeping our customers safe and secure.
+- **Transparency**. We will work with you to validate and remediate reported vulnerabilities in accordance with our commitment to security and privacy.
+- **Common Good**. We investigate and remediate issues in a manner consistent with protecting the safety and security of those potentially affected by a reported vulnerability.
+
+## What We Ask of Reporters
+
+- **Trust**. We request that you communicate about potential vulnerabilities in a responsible manner, providing sufficient time and information for our team to validate and address potential issues.
+- **Respect**. We request that researchers make every effort to avoid privacy violations, degradation of user experience, disruption to production systems, and destruction of data during security testing.
+- **Transparency**. We request that researchers provide the technical details and background necessary for our team to identify and validate reported issues, using the form below.
+- **Common Good**. We request that researchers act for the common good, protecting user privacy and security by refraining from publicly disclosing unverified vulnerabilities until our team has had time to validate and address reported issues.
+
+## Vulnerability Reporting
+
+ClickUp recommends that you share the details of any suspected vulnerabilities across any asset owned, controlled, or operated by ClickUp (or that would reasonably impact the security of ClickUp and our users) using our vulnerability disclosure form at <http://clickup.com/bug-bounty>. The ClickUp Security team will acknowledge receipt of each valid vulnerability report, conduct a thorough investigation, and then take appropriate action for resolution.
+
+## Safe Harbor
+
+When conducting vulnerability research according to this policy, we consider this research to be:
+
+- Authorized in accordance with the Computer Fraud and Abuse Act (CFAA) (and/or similar state laws), and we will not initiate or support legal action against you for accidental, good faith violations of this policy;
+- Exempt from the Digital Millennium Copyright Act (DMCA), and we will not bring a claim against you for circumvention of technology controls;
+- Exempt from restrictions in our Terms & Conditions that would interfere with conducting security research, and we waive those restrictions on a limited basis for work done under this policy; and
+- Lawful, helpful to the overall security of the Internet, and conducted in good faith.
+- You are expected, as always, to comply with all applicable laws.
+
+If at any time you have concerns or are uncertain whether your security research is consistent with this policy, please inquire via <[email protected]> before going any further.

docker-compose.yml

@@ -0,0 +1,15 @@
+version: "3.4"
+services:
+  postgres:
+    image: postgres:16
+    ports:
+      - "54833:5432"
+    environment:
+      POSTGRES_PASSWORD: postgres
+      PGDATA: /tmp/postgresql
+      POSTGRES_INITDB_ARGS: "-c max_connections=1000  -c synchronous_commit=off"
+    healthcheck:
+      test: "pg_isready -U postgres"
+      interval: 1s
+      timeout: 20s
+      retries: 10

internal/docker-compose-up.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+docker-compose up --quiet-pull -d
+
+for i in $(seq 1 20); do
+  docker-compose ps | grep postgres | grep -q healthy && break
+  sleep 1
+done

jest.config.js

@@ -1,5 +1,11 @@
 "use strict";
-
 module.exports = {
-  ...require("../../jest.config.base")(),
+  roots: ["<rootDir>/src"],
+  testMatch: ["**/*.test.ts"],
+  clearMocks: true,
+  restoreMocks: true,
+  ...(process.env.IN_JEST_PROJECT ? {} : { forceExit: true }),
+  transform: {
+    "\\.ts$": "ts-jest",
+  },
 };

package.json

@@ -1,19 +1,32 @@
 {
-  "name": "pg-sharding",
-  "description": "Maintains the desired number of PG schemas (shards) and allows to migrate them.",
+  "name": "@clickup/pg-sharding",
+  "description": "Micro-shards support for PostgreSQL",
   "version": "2.10.291",
   "license": "MIT",
-  "main": "dist/cli.js",
-  "types": "dist/cli.d.ts",
+  "keywords": [
+    "postgresql",
+    "sharding",
+    "rebalance",
+    "cluster"
+  ],
+  "main": "./dist/cli.js",
+  "types": "./dist/cli.d.ts",
+  "exports": "./dist/cli.js",
   "bin": {
     "pg-sharding": "./dist/cli.js"
   },
   "scripts": {
-    "build": "tsc.sh",
-    "dev": "tight-loop.sh tsc.sh --watch",
-    "lint": "lint.sh",
-    "test": "test.sh",
-    "test:db": "set -e; for f in sql/__tests__/test_*.sql; do echo == $f; echo; yarn psql -f $f; echo; echo; done"
+    "build": "tsc",
+    "dev": "tsc --watch --preserveWatchOutput",
+    "lint": "eslint . --ext .ts --cache --cache-location dist/.eslintcache",
+    "test": "set -e; bash internal/docker-compose-up.sh; export PGHOST=127.0.0.1 PGPORT=54833 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=postgres; jest",
+    "test:db": "set -e; bash internal/docker-compose-up.sh; export PGHOST=127.0.0.1 PGPORT=54833 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=postgres; for f in sql/__tests__/test_*.sql; do echo == $f; echo; psql -f $f; echo; echo; done",
+    "psql": "PGHOST=127.0.0.1 PGPORT=54833 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=postgres psql",
+    "docs": "echo No docs step.",
+    "clean": "rm -rf dist node_modules yarn.lock package-lock.json *.log",
+    "copy-package-to-public-dir": "copy-package-to-public-dir.sh",
+    "backport-package-from-public-dir": "backport-package-from-public-dir.sh",
+    "deploy": "npm run build && npm run lint && npm run test && npm publish --access=public"
   },
   "dependencies": {
     "chalk": "^4.1.2",
@@ -25,11 +38,30 @@
     "prompts": "^2.4.2",
     "sprintf-js": "^1.1.2"
   },
+  "devDependencies": {
+    "@types/jest": "^29.5.5",
+    "@types/lodash": "^4.14.175",
+    "@types/minimist": "^1.2.2",
+    "@types/pg": "^8.6.1",
+    "@types/prompts": "^2.4.0",
+    "@types/sprintf-js": "^1.1.2",
+    "@typescript-eslint/eslint-plugin": "^5.59.6",
+    "@typescript-eslint/parser": "^5.59.6",
+    "eslint-import-resolver-typescript": "^3.5.5",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-lodash": "^7.4.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react": "^7.32.2",
+    "eslint-plugin-typescript-enum": "^2.1.0",
+    "eslint-plugin-typescript-sort-keys": "^2.3.0",
+    "eslint-plugin-unused-imports": "^2.0.0",
+    "eslint": "^8.40.0",
+    "ts-jest": "^29.1.1",
+    "typescript": "^5.2.2"
+  },
   "repository": {
     "type": "git",
-    "url": "git://github.com/time-loop/github-packages"
-  },
-  "publishConfig": {
-    "registry": "https://npm.pkg.github.com/"
+    "url": "git://github.com/clickup/pg-sharding"
   }
 }

sql/__tests__/begin.sql

@@ -16,6 +16,9 @@ $$;
 
 BEGIN;
 
+CREATE EXTENSION IF NOT EXISTS postgres_fdw;
+CREATE EXTENSION IF NOT EXISTS dblink;
+
 CREATE SCHEMA test_sharding;
 SET search_path TO test_sharding;
 SET client_min_messages TO NOTICE;

sql/functions/sharding_ensure_absent.sql

@@ -19,6 +19,7 @@ BEGIN
     )
     SELECT shards.shard FROM shards
     WHERE EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = shards.shard)
+    ORDER BY shards.shard
   LOOP
     EXECUTE format('DROP SCHEMA %I', shard);
     RETURN NEXT shard;

sql/functions/sharding_ensure_exist.sql

@@ -19,6 +19,7 @@ BEGIN
     )
     SELECT shards.shard FROM shards
     WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = shards.shard)
+    ORDER BY shards.shard
   LOOP
     EXECUTE format('CREATE SCHEMA %I', shard);
     RETURN NEXT shard::regnamespace;

sql/functions/sharding_list_active_shards.sql

@@ -4,8 +4,7 @@ SET search_path FROM CURRENT
 AS $$
   SELECT COALESCE(array_agg(nspname::regnamespace ORDER BY nspname), '{}')
   FROM pg_namespace
-  WHERE
-    pg_namespace.nspname = ANY(_sharding_active_shards())
+  WHERE pg_namespace.nspname = ANY(_sharding_active_shards())
 $$;
 
 COMMENT ON FUNCTION sharding_list_active_shards()

src/cli.ts

@@ -5,8 +5,9 @@ import cleanup from "./api/cleanup";
 import move from "./api/move";
 import rebalance from "./api/rebalance";
 import { log } from "./helpers/logging";
+import shellQuote from "./helpers/shellQuote";
 
-export { cleanup, move, rebalance };
+export { cleanup, move, rebalance, shellQuote };
 
 const USAGE = [
   "Usage:",
@@ -25,22 +26,22 @@ export async function main(argv: string[]): Promise<boolean> {
 
   if (args._[0] === "move") {
     let shard: number;
-    if ((args.shard ?? "").match(/^(\d+)$/)) {
-      shard = parseInt(args.shard);
+    if ((args["shard"] ?? "").match(/^(\d+)$/)) {
+      shard = parseInt(args["shard"]);
     } else {
       throw "Please provide --shard, a numeric shard number to move";
     }
 
     let fromDsn: string;
-    if ((args.from ?? "").match(/^\w+:\/\//)) {
-      fromDsn = args.from;
+    if ((args["from"] ?? "").match(/^\w+:\/\//)) {
+      fromDsn = args["from"];
     } else {
       throw "Please provide --from, source DB DSN, as postgresql://user:pass@host/db?options";
     }
 
     let toDsn: string;
-    if ((args.to ?? "").match(/^\w+:\/\//)) {
-      toDsn = args.to;
+    if ((args["to"] ?? "").match(/^\w+:\/\//)) {
+      toDsn = args["to"];
     } else {
       throw "Please provide --to, destination DB DSN, as postgresql://user:pass@host/db?options";
     }
@@ -74,8 +75,8 @@ export async function main(argv: string[]): Promise<boolean> {
 
   if (args._[0] === "cleanup") {
     let dsn: string;
-    if ((args.dsn ?? "").match(/^\w+:\/\//)) {
-      dsn = args.dsn;
+    if ((args["dsn"] ?? "").match(/^\w+:\/\//)) {
+      dsn = args["dsn"];
     } else {
       throw "Please provide --dsn, DB DSN to remove old schemas from, as postgresql://user:pass@host/db?options";
     }

src/helpers/runShell.ts

@@ -33,7 +33,7 @@ export default async function runShell(
           ...process.env,
           PGOPTIONS: compact([
             "--client-min-messages=warning",
-            process.env.PGOPTIONS,
+            process.env["PGOPTIONS"],
           ]).join(" "),
         },
       })

tsconfig.json

@@ -1,13 +1,32 @@
 {
-  "extends": "../../tsconfig.base.json",
   "include": ["src/**/*"],
   "compilerOptions": {
-    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo",
-    "emitDeclarationOnly": true, // turns on SWC
-    "rootDir": "src",
+    "allowJs": true,
+    "declaration": true,
+    "declarationMap": true,
+    "disableReferencedProjectLoad": true,
+    "disableSourceOfProjectReferenceRedirect": true,
+    "esModuleInterop": true,
+    "experimentalDecorators": true,
+    "incremental": true,
+    "lib": ["es2019"],
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "noEmitOnError": true,
+    "noErrorTruncation": true,
+    "noImplicitOverride": true,
+    "noImplicitReturns": true,
+    "noPropertyAccessFromIndexSignature": true,
     "outDir": "dist",
-    "lib": ["es2019", "DOM"],
-    "types": ["node", "jest"],
-    "module": "commonjs"
+    "pretty": true,
+    "removeComments": false,
+    "resolveJsonModule": true,
+    "rootDir": "src",
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "strict": true,
+    "target": "es2019",
+    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo",
+    "types": ["node", "jest"]
   }
 }