Refactor/single extension (#1008)

* refactor(Extension): move from Extension/firefox to extension

* refactor(Extension): unify extensions

* refactor(Extension): fix .eslintrs.js and execute npm run fix

* refactor(Extension): apply ESLint and prettier

To allow ESLint to parse the most modern JavaScript a dependecy on babel
has been added.

This change allows us to enforce consistent formatting across the
extension in the future

* chore(conda): unlock node version

This change was enabled by removing npm-run-all

* feat(extension): use content.ts for content.js

* test(custom_function): use new selenium attributes

* fix(webdriver_utils): use new selenium methods

.dockerignore

@@ -18,11 +18,11 @@ firefox-bin
 venv
 
 # npm packages
-Extension/firefox/node_modules
+Extension/node_modules
 Extension/webext-instrumentation/node_modules
 
 # built extension artifacts
-Extension/firefox/dist
-Extension/firefox/openwpm.xpi
-Extension/firefox/src/content.js
-Extension/firefox/src/feature.js
+Extension/dist
+Extension/openwpm.xpi
+Extension/src/content.js
+Extension/src/feature.js

.gitignore

@@ -91,9 +91,9 @@ docs/apidoc/
 node_modules
 
 # built extension artifacts
-Extension/firefox/dist
-Extension/firefox/openwpm.xpi
-Extension/firefox/src/content.js
-Extension/firefox/src/feature.js
+Extension/dist
+Extension/openwpm.xpi
+Extension/bundled/content.js
+Extension/bundled/feature.js
 
 datadir

CONTRIBUTING.md

@@ -35,12 +35,12 @@ the project as well as the one you plan to change fundamentally.
 
 ### Editing instrumentation
 
-The instrumentation extension is included in `/Extension/firefox/`.
+The instrumentation extension is included in `/Extension/`.
 The instrumentation itself (used by the above extension) is included in
 `/Extension/webext-instrumentation/`.
 Any edits within these directories will require the extension to be re-built to produce
 a new `openwpm.xpi` with your updates. You can use `./scripts/build-extension.sh` to do this,
-or you can run `npm run build` from `Extension/firefox/`.
+or you can run `npm run build` from `/Extension/`.
 
 ### Debugging the platform
 

Extension/.eslintrc.js

@@ -0,0 +1,222 @@
+/* eslint-env node */
+/*
+This ESLint config aims to work with both
+the JavaScript and the TypeScript parts of the
+codebase.
+*/
+
+module.exports = {
+  root: true,
+  env: {
+    node: true,
+    es6: true,
+    webextensions: true,
+    browser: true,
+  },
+  parser: "@babel/eslint-parser",
+  extends: ["prettier", "eslint:recommended"],
+  ignorePatterns: [
+    "bundled/feature.js",
+    "bundled/content.js",
+    "bundled/privileged/sockets/bufferpack.js" /** This is a file we just copied in  */,
+    "build/",
+    "dist/",
+    "node_modules/",
+  ],
+  plugins: [
+    "eslint-plugin-import",
+    "eslint-plugin-jsdoc",
+    "eslint-plugin-unicorn",
+    "eslint-plugin-mozilla",
+  ],
+  rules: {
+    "arrow-parens": ["off", "always"],
+    "brace-style": ["off", "off"],
+    "comma-dangle": "off",
+    complexity: "off",
+    "constructor-super": "error",
+    "eol-last": "off",
+    eqeqeq: ["warn", "smart"],
+    "guard-for-in": "warn",
+    "id-blacklist": "warn",
+    "id-match": "warn",
+    "import/no-extraneous-dependencies": ["error"],
+    "import/no-internal-modules": "error",
+    "jsdoc/check-alignment": "error",
+    "jsdoc/check-indentation": "error",
+    "jsdoc/newline-after-description": "error",
+    "linebreak-style": "off",
+    "max-classes-per-file": ["error", 1],
+    "max-len": "off",
+    "new-parens": "off",
+    "newline-per-chained-call": "off",
+    "no-bitwise": "warn",
+    "no-caller": "error",
+    "no-cond-assign": "error",
+    "no-debugger": "error",
+    "no-duplicate-case": "error",
+    "no-duplicate-imports": "error",
+    "no-empty": "warn",
+    "no-eval": "error",
+    "no-extra-bind": "error",
+    "no-extra-semi": "off",
+    "no-fallthrough": "off",
+    "no-invalid-this": "off",
+    "no-irregular-whitespace": "off",
+    "no-multiple-empty-lines": "off",
+    "no-new-func": "error",
+    "no-new-wrappers": "error",
+    "no-redeclare": "error",
+    "no-return-await": "error",
+    "no-sequences": "error",
+    "no-sparse-arrays": "error",
+    "no-template-curly-in-string": "error",
+    "no-throw-literal": "error",
+    "no-trailing-spaces": "off",
+    "no-undef-init": "error",
+    "no-underscore-dangle": "warn",
+    "no-unsafe-finally": "error",
+    "no-unused-labels": "error",
+    "no-unused-vars": ["error", { argsIgnorePattern: "^_" }],
+    "no-var": "error",
+    "object-shorthand": "error",
+    "one-var": ["warn", "never"],
+    "prefer-const": "error",
+    "prefer-object-spread": "error",
+    "quote-props": "off",
+    radix: "error",
+    "space-before-function-paren": "off",
+    "space-in-parens": ["off", "never"],
+    "spaced-comment": [
+      "error",
+      "always",
+      {
+        markers: ["/"],
+      },
+    ],
+    "unicorn/prefer-ternary": "error",
+    "use-isnan": "error",
+    "valid-typeof": "off",
+  },
+  overrides: [
+    {
+      files: ["bundled/privileged/**"],
+      env: {
+        es2022: true,
+        "mozilla/browser-window": true,
+        "mozilla/privileged": true,
+      },
+      globals: {
+        Cu: "readonly",
+        Ci: "readonly",
+        Cc: "readonly",
+        Cr: "readonly",
+        ExtensionAPI: "readonly",
+        Services: "readonly",
+        OS: "readonly",
+      },
+    },
+    {
+      files: ["*.ts"],
+      parser: "@typescript-eslint/parser",
+      parserOptions: {
+        project: "tsconfig.json",
+        sourceType: "module",
+      },
+      plugins: ["@typescript-eslint"],
+      rules: {
+        "@typescript-eslint/adjacent-overload-signatures": "error",
+        "@typescript-eslint/array-type": [
+          "error",
+          {
+            default: "array",
+          },
+        ],
+        "@typescript-eslint/ban-types": [
+          "error",
+          {
+            types: {
+              Object: {
+                message:
+                  "Avoid using the `Object` type. Did you mean `object`?",
+              },
+              Function: {
+                message:
+                  "Avoid using the `Function` type. Prefer a specific function type, like `() => void`.",
+              },
+              Boolean: {
+                message:
+                  "Avoid using the `Boolean` type. Did you mean `boolean`?",
+              },
+              Number: {
+                message:
+                  "Avoid using the `Number` type. Did you mean `number`?",
+              },
+              String: {
+                message:
+                  "Avoid using the `String` type. Did you mean `string`?",
+              },
+              Symbol: {
+                message:
+                  "Avoid using the `Symbol` type. Did you mean `symbol`?",
+              },
+            },
+          },
+        ],
+        "@typescript-eslint/consistent-type-assertions": "off",
+        "@typescript-eslint/dot-notation": "error",
+        "@typescript-eslint/indent": "off",
+        "@typescript-eslint/member-delimiter-style": [
+          "off",
+          {
+            multiline: {
+              delimiter: "none",
+              requireLast: true,
+            },
+            singleline: {
+              delimiter: "semi",
+              requireLast: false,
+            },
+          },
+        ],
+        "@typescript-eslint/no-empty-function": "warn",
+        "@typescript-eslint/no-empty-interface": "error",
+        "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/no-misused-new": "error",
+        "@typescript-eslint/no-namespace": "error",
+        "@typescript-eslint/no-parameter-properties": "off",
+        "@typescript-eslint/no-shadow": [
+          "off",
+          {
+            hoist: "all",
+          },
+        ],
+        "@typescript-eslint/no-this-alias": "error",
+        "@typescript-eslint/no-unused-expressions": "warn",
+        "@typescript-eslint/no-use-before-define": "off",
+        "@typescript-eslint/no-var-requires": "error",
+        "@typescript-eslint/prefer-for-of": "warn",
+        "@typescript-eslint/prefer-function-type": "error",
+        "@typescript-eslint/prefer-namespace-keyword": "error",
+        "@typescript-eslint/quotes": "off",
+        "@typescript-eslint/semi": ["off", null],
+        "@typescript-eslint/triple-slash-reference": [
+          "error",
+          {
+            path: "always",
+            types: "prefer-import",
+            lib: "always",
+          },
+        ],
+        "@typescript-eslint/type-annotation-spacing": "off",
+        "@typescript-eslint/unified-signatures": "error",
+        "no-undef": "off",
+        "no-unused-vars": "off",
+        "@typescript-eslint/no-unused-vars": [
+          "error",
+          { argsIgnorePattern: "^_" },
+        ],
+      },
+    },
+  ],
+};

Extension/.prettierignore

@@ -0,0 +1,6 @@
+bundled/content.js
+bundled/feature.js
+bundled/privileged/sockets/bufferpack.js
+dist
+build
+node_modules

Extension/README.md

@@ -0,0 +1,22 @@
+# OpenWPM Client Extension
+
+Used by the OpenWPM platform.
+This extension implements the OpenWPM instrumentation as a WebExtension.
+It allows users to track and monitor privacy-related browsing behavior.
+
+## Installation
+
+```bash
+npm install
+```
+
+## Instrumentation
+
+The instrumentation leverages the available [JavaScript APIs for WebExtensions](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API) and listens to navigation, web requests, cookie modifications and access to certain javascript API:s, as described in [the main OpenWPM project README](../../README.md#instrumentation-and-data-access) under the following bullet points:
+
+- HTTP Request and Response Headers, redirects, and POST request bodies
+- Javascript Calls
+- Response body content
+- Cookie Access (Experimental)
+
+More specifically, all packets sent by the instrumentation conform to [these interfaces](/src/schema.ts).

Extension/babel.config.json

@@ -0,0 +1,3 @@
+{
+  "presets": ["@babel/preset-env"]
+}

Extension/firefox/src/manifest.json → Extension/bundled/manifest.json

@@ -13,7 +13,7 @@
       "strict_min_version": "60.0"
     }
   },
-  "content_security_policy":"script-src 'self' 'unsafe-eval'; object-src 'self';",
+  "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self';",
   "permissions": [
     "<all_urls>",
     "webRequest",

Extension/firefox/src/privileged/profileDirIO/api.js → Extension/bundled/privileged/profileDirIO/api.js

@@ -1,19 +1,25 @@
-ChromeUtils.defineModuleGetter(this, "ExtensionCommon",
-                               "resource://gre/modules/ExtensionCommon.jsm");
+ChromeUtils.defineModuleGetter(
+  this,
+  "ExtensionCommon",
+  "resource://gre/modules/ExtensionCommon.jsm",
+);
 ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
 Cu.importGlobalProperties(["TextEncoder", "TextDecoder"]);
 
 this.profileDirIO = class extends ExtensionAPI {
-  getAPI(context) {
+  getAPI(_context) {
     return {
       profileDirIO: {
         async writeFile(filename, data) {
-          let encoder = new TextEncoder();
-          let array = encoder.encode(data);
-          let path = OS.Path.join(OS.Constants.Path.profileDir, filename);
-          let tmpPath = OS.Path.join(OS.Constants.Path.profileDir, filename + ".tmp");
+          const encoder = new TextEncoder();
+          const array = encoder.encode(data);
+          const path = OS.Path.join(OS.Constants.Path.profileDir, filename);
+          const tmpPath = OS.Path.join(
+            OS.Constants.Path.profileDir,
+            filename + ".tmp",
+          );
           try {
-            await OS.File.writeAtomic(path, array, {tmpPath});
+            await OS.File.writeAtomic(path, array, { tmpPath });
             console.log(`Wrote to ${path}`);
             return true;
           } catch (e) {
@@ -22,10 +28,10 @@ this.profileDirIO = class extends ExtensionAPI {
           }
         },
         async readFile(filename) {
-          let decoder = new TextDecoder();
-          let path = OS.Path.join(OS.Constants.Path.profileDir, filename);
+          const decoder = new TextDecoder();
+          const path = OS.Path.join(OS.Constants.Path.profileDir, filename);
           try {
-            let array = await OS.File.read(path);
+            const array = await OS.File.read(path);
             return decoder.decode(array);
           } catch (e) {
             Cu.reportError(e);