feat: add automatic header order collector / updater (#149)

.github/workflows/model-updater.yml

@@ -32,6 +32,13 @@ jobs:
       - name: Install Dependencies
         run: npm ci
 
+      - name: Install Playwright browsers and collect HTTP header orders
+        run: |
+          npx playwright install
+          RUN npx playwright install chrome
+          RUN npx playwright install msedge
+          node ./packages/header-order-collector/collector.js > ./packages/header-generator/src/data_files/headers_order.json
+
       - name: Generate model and update versions
         run: |
           npm version patch --git-tag-version=false

package.json

@@ -41,7 +41,7 @@
         "test": "jest --silent",
         "lint": "eslint packages/*/src test",
         "lint:fix": "eslint packages/*/src test --fix",
-        "buildNetwork": "turbo run build && ./scripts/netgen.sh",
+        "buildNetwork": "turbo run build && ./scripts/netgen.sh && turbo run build",
         "benchmark": "ts-node ./test/antibot-services/live-testing/cloudflare.ts"
     },
     "devDependencies": {

packages/header-order-collector/Dockerfile

@@ -0,0 +1,17 @@
+FROM mcr.microsoft.com/playwright:v1.30.0-focal
+
+# Install browsers
+RUN npx playwright install chrome
+RUN npx playwright install msedge
+
+# Install Node.js dependencies
+COPY package.json package-lock.json ./
+RUN npm install
+
+
+# Copy the rest of the files
+COPY ./collector.js ./
+COPY ./server.js ./
+
+# Run the server
+CMD ["node", "collector.js"]

packages/header-order-collector/cert/localhost-cert.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDtTCCAp2gAwIBAgIUETM6CjfHl22Z8zGPhixP1J+JBLAwDQYJKoZIhvcNAQEL
+BQAwajELMAkGA1UEBhMCVVMxDzANBgNVBAgMBk9yZWdvbjERMA8GA1UEBwwIUG9y
+dGxhbmQxFTATBgNVBAoMDENvbXBhbnkgTmFtZTEMMAoGA1UECwwDT3JnMRIwEAYD
+VQQDDAlsb2NhbGhvc3QwHhcNMjMwMzI2MTIyODEyWhcNMjMwNDI1MTIyODEyWjBq
+MQswCQYDVQQGEwJVUzEPMA0GA1UECAwGT3JlZ29uMREwDwYDVQQHDAhQb3J0bGFu
+ZDEVMBMGA1UECgwMQ29tcGFueSBOYW1lMQwwCgYDVQQLDANPcmcxEjAQBgNVBAMM
+CWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL2EKwVI
+rYHIy/Xd/Cm5jXfFtdhQJykuAnxwg9Oww/OO1WYdSYEG/WvCwZRj4e/aj4JzHyp+
+bwemNCfHogq2PW77AnDU640KXEApQd2APHUV12zKICdbsQuG6sZdahQulrUqb7pT
+1tv39xYdHczILDIv0XOfOy2JN+lZhoU/odBsNyBJ9pbjYdGdd4Jur8NWn3bFVDdf
+I5fBg2Zbyp91+fHi4Bmz+eLabkSmdGqtXDUvlcsfoCaZD9MYX33aNI587viFiLj1
+lMIgVyGTJKPVAegfxr4EhgvEZwTbuo6aQX4EeOdr7gQDARxVGGrAJgljPumh93LY
+VU0yrej35IP3Nt0CAwEAAaNTMFEwHQYDVR0OBBYEFBo4Cm4ywZaF2NREh8K3Si4p
+7LfvMB8GA1UdIwQYMBaAFBo4Cm4ywZaF2NREh8K3Si4p7LfvMA8GA1UdEwEB/wQF
+MAMBAf8wDQYJKoZIhvcNAQELBQADggEBAE80jKMCoXp4oGDPqZIzcE2x8Tx63MK6
+oPBgPZkDXDLPNlScU+Uk7/ydl6+t8gewwqF33GlpYuY6YUrcp5N95RCz68YgXsPv
+E/25EjzJI3H1/2PGSTuWASPDocKdwCTKsfmEGKSWPAHlIAEwgkCN8QoxeoBx2kuu
+oNyKt3+OW0H+mw9WW3ZVdYJPqbhKaUx1QshZOKjWWrO2wvDDFwYtfiD9FJb3e0XB
+y06CNnF3MPB9wA/zHYrPPCyo+e5A+ArKsBDH6PWP0hX+MIvxPe64rjdf1vpraYV4
+oRFhzOZ7HihCOjfwStwa76J9KvPTch5kugyx/FvLcCzonwctQ7AnNFY=
+-----END CERTIFICATE-----

packages/header-order-collector/cert/localhost-privkey.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC9hCsFSK2ByMv1
+3fwpuY13xbXYUCcpLgJ8cIPTsMPzjtVmHUmBBv1rwsGUY+Hv2o+Ccx8qfm8HpjQn
+x6IKtj1u+wJw1OuNClxAKUHdgDx1FddsyiAnW7ELhurGXWoULpa1Km+6U9bb9/cW
+HR3MyCwyL9FznzstiTfpWYaFP6HQbDcgSfaW42HRnXeCbq/DVp92xVQ3XyOXwYNm
+W8qfdfnx4uAZs/ni2m5EpnRqrVw1L5XLH6AmmQ/TGF992jSOfO74hYi49ZTCIFch
+kySj1QHoH8a+BIYLxGcE27qOmkF+BHjna+4EAwEcVRhqwCYJYz7pofdy2FVNMq3o
+9+SD9zbdAgMBAAECggEACn9s+6KyO79x7tlgS7Zjy/fncgdEmA8co6R+5OXEgPQP
+RZQHO4610HeODgh3Xjgq3WS/m/S7dT7DYlW4V+QO5JwiXzooHxytDf/LDng5QRoX
+4AaVP+DgHn2XeSv5AYuwHPJwFG/q9a3XYcH0oeCPtCr6bO9r0YKplnM3VJ+4+KjB
+feBjxdppTVa4iY5FpEASsKnvXt9efqAnwR8lgKlWAD37OPMgrZldJlSL98IEp2bj
+vcwabuWiFGAQeMeG5wzBVpsjad6h4eE9XVZi3Gwzup0Rz8OHqf4SsHen//ECT7j2
+8la6RpwNlZjEcJtReaI9yu7USor3SugI2zJxY7X1CQKBgQDVaiQibd/4jS+vXsiC
+yh6A/YAHGQlyvvLL4yzQ80XQvJYT4x9VDK7v8Se1RsTYs3LgDxGZ0EvECiorG3rI
+i0du9UTip2n3OGKgZh2Memfe6j/lkNkvlGBQ3RdRflKUS7SsAWZY1PkCysJZ98N6
+1a9zoIwiSONrKYQSRBVbODYdFQKBgQDjVTp2aAI4FKIdhImViIh2JZdz89kYCJdD
+MK8LvSjDQCA3ADQqXXlzqgZwR27XE7CbGcRsIXGP1a8neQbkAxeSo68UUuCpHq9v
+JoYrfRMS5CZDLHmr3N46wGpgzlgyAH0PtgoNdvHcxskX8jOvYjGsxh+r/XV/yeyT
+RXPDX+70qQKBgQCx4Xtm2xAgmoWRdb5JDXDpcOIiiGLQqomzGZJEQ1EAz/+Daf62
+pBC329iOBwOR39izGrw/6VcNeq91KL1ppHWQ7YxSItp2/h9utE6SxYAIVFWX4ei0
++MZjADUsbs8j3gokiDInfrRi+F+QH6xmmO9ZwAqtlPVBX+Q+WVM3qh2olQKBgEMJ
+mQndtfnPq9bEhMYB0yvz/euFlxlNPz3jF1uA1optWJKfjsKHeg8vrmJhzj/CYKv8
+Nz4jxpdBzjr3p56JguQNPM3jerikgdtnIt2kLoG9ocLhVKfno+jzYOtXLg7gsgB2
+EBTUZJJUVGypUbCAZmGqsyBGx+V9KDRtoKuWmKKxAoGAGQjx7fKum+68X+cbCDjd
+mbvWEPnbdPQIzcfuzb4HCNvoNb8yxkNEvfxbNZgXf+xU4YufPYqNxyvHPdKsBzEw
+vN9+zAtyF7NodU3qyKyLVXO9Q26HXBMOijNkbSpTFYRan8mlWVUMdKzsjt8bcNrZ
+fjT5CpfugV4xmmDhKzITvwE=
+-----END PRIVATE KEY-----

packages/header-order-collector/collector.js

@@ -0,0 +1,55 @@
+const playwright = require('playwright');
+const { runServer: v1 } = require('./server');
+const { runServer: v2 } = require('./serverv2');
+
+const HTTP1port = 3001;
+const HTTP2port = 3002;
+
+async function getHeadersFor(launcher, httpVersion) {
+    const browser = await launcher({
+        headless: false,
+    });
+
+    const context = await browser.newContext({
+        ignoreHTTPSErrors: true,
+    });
+    const page = await context.newPage();
+
+    if ( httpVersion === 1 ) {
+        await page.goto(`http://localhost:${HTTP1port}/`);
+    } else {
+        await page.goto(`https://localhost:${HTTP2port}/`);
+    }
+    await page.click('a');
+
+    const headerNames = await page.evaluate(() => {
+        return JSON.parse(document.body.innerText);
+    });
+    await browser.close();
+    return headerNames;
+}
+
+(async () => {
+        v1(HTTP1port);
+        v2(HTTP2port);
+
+        const browserTypes = {
+            // safari: () => playwright.webkit.launch(),
+            chrome: (p) => playwright.chromium.launch(p),
+            firefox: (p) => playwright.firefox.launch(p),
+            // edge: () => playwright.chromium.launch({ channel: 'msedge' }),
+        };
+
+        const x = await Promise.all(
+            Object.entries(browserTypes)
+                .map(async ([name, launcher]) => {
+                    return [name, [...await getHeadersFor(launcher, 1), ...await getHeadersFor(launcher, 2)]];
+                })
+        );
+
+        console.log(JSON.stringify(Object.fromEntries(x), null, 4));
+
+        process.exit(0);
+    }
+)();
+

packages/header-order-collector/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "header-order-collector",
+  "version": "1.0.0",
+  "description": "",
+  "main": "p.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node server.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "express": "^4.18.2",
+    "playwright": "^1.31.2"
+  }
+}

packages/header-order-collector/server.js

@@ -0,0 +1,38 @@
+const express = require('express');
+const app = express();
+const port = 3001;
+
+function extractHeaderNames(rawHeaders) {
+    return rawHeaders.filter((_, i) => i % 2 === 0);
+}
+
+app.get('/', (req, res) => {
+    res.header('set-cookie', 'A=1; Path=/headers; HttpOnly');
+    res.send(`
+<a href="/headers">Click here</a> to see the headers.
+`)
+})
+
+app.get('/headers', (req, res) => {
+    const rawHeaders = req.rawHeaders;
+    const headerNames = extractHeaderNames(rawHeaders);
+    res.json(headerNames);
+});
+
+
+function runServer(p) {
+    return new Promise(r => {
+        const inst = app.listen(p ?? port, (err) => {
+                if (err) {
+                    throw new Error(err);
+                }
+
+                r(inst);
+            });
+    });
+}
+
+module.exports = {
+    runServer,
+    app,
+};

packages/header-order-collector/serverv2.js

@@ -0,0 +1,38 @@
+const http2 = require('node:http2');
+const fs = require('fs');
+
+const port = 3002;
+
+const server = http2.createSecureServer({
+    key: fs.readFileSync(`${__dirname}/cert/localhost-privkey.pem`),
+    cert: fs.readFileSync(`${__dirname}/cert/localhost-cert.pem`),
+  });
+server.on('error', (err) => console.error(err));
+
+server.on('stream', (stream, headers) => {
+  if(headers[':path'] === '/headers'){
+    stream.respond({
+        'content-type': 'application/json; charset=utf-8',
+        ':status': 200,
+    });
+    stream.end(JSON.stringify(Object.keys(headers)));
+    return;
+  };
+
+  stream.respond({
+    'content-type': 'text/html; charset=utf-8',
+    'set-cookie': 'A=1; Path=/headers; Secure; HttpOnly',
+    ':status': 200,
+  });
+  stream.end(`
+  <a href="/headers">Click here</a> to see the headers.
+  `);
+});
+
+module.exports = {
+    runServer: (p) => server.listen(p ?? port),
+}
+
+if (require.main === module) {
+  server.listen(port);
+}