From a26fb083ab218f28388ddb746356bdafd6ddc847 Mon Sep 17 00:00:00 2001
From: Matt Sutkowski <msutkowski@gmail.com>
Date: Sat, 25 Sep 2021 09:08:35 -0700
Subject: [PATCH 1/2] WIP: Basic plumbing for record/replay modes

---
 example/index.tsx            | 18 +++++-----
 src/component/MSWToolbar.tsx | 65 ++++++++++++++++++++++++++++++++++--
 src/helpers/settings.ts      |  2 +-
 src/types.ts                 | 11 ++++--
 tsconfig.json                |  1 +
 5 files changed, 84 insertions(+), 13 deletions(-)

diff --git a/example/index.tsx b/example/index.tsx
index 22c0d04..803107e 100644
--- a/example/index.tsx
+++ b/example/index.tsx
@@ -25,7 +25,7 @@ const App = () => {
       <ChakraProvider>
         <MSWToolbar
           worker={worker}
-          apiUrl="http://www.example.com"
+          apiUrl="https://pokeapi.co"
           isEnabled={true}
           actions={
             <HStack spacing={2}>
@@ -37,14 +37,16 @@ const App = () => {
         >
           <Button
             onClick={() =>
-              fetch('http://www.example.com').then(async (res) => {
-                if (res.ok) {
-                  const content = await res.json();
-                  alert(
-                    `Here is the mocked response!: ${JSON.stringify(content)}`
-                  );
+              fetch('https://pokeapi.co/api/v2/pokemon/ditto').then(
+                async (res) => {
+                  if (res.ok) {
+                    const content = await res.json();
+                    alert(
+                      `Here is the mocked response!: ${JSON.stringify(content)}`
+                    );
+                  }
                 }
-              })
+              )
             }
             mt={50}
           >
diff --git a/src/component/MSWToolbar.tsx b/src/component/MSWToolbar.tsx
index afd4f70..5c62112 100644
--- a/src/component/MSWToolbar.tsx
+++ b/src/component/MSWToolbar.tsx
@@ -1,5 +1,5 @@
 import * as React from 'react';
-import { rest } from 'msw';
+import { RequestHandler, rest } from 'msw';
 import {
   SetupWorkerApi,
   MockedRequest,
@@ -10,7 +10,35 @@ import { usePrevious } from './hooks';
 import styles from './styles.module.css';
 import { WorkerMode } from '../types';
 import { get, modes, set } from '../helpers';
-import { MSWToolbarProps } from '..';
+import { MSWToolbarProps, TrackedRequest } from '..';
+
+async function buildHandlersFromRecordedRequests(
+  trackedRequests: TrackedRequest
+): Promise<RequestHandler[]> {
+  let handlers = [];
+  for (const [, { request, response }] of trackedRequests.entries()) {
+    if (!response) continue;
+
+    // TODO: this should probably happen before we add it to the map,
+    // and we'd most likely want to set the response type there (text, json)
+    // along with the data
+    const data = await response.json();
+
+    const handler = (rest as any)[request.method.toLocaleLowerCase()](
+      request.url.href,
+      (
+        _req: MockedRequest<any>,
+        res: ResponseComposition<any>,
+        ctx: RestContext
+      ) => {
+        return res(ctx.json(data));
+      }
+    );
+    handlers.push(handler);
+  }
+
+  return handlers;
+}
 
 /**
  * This is a simple toolbar that allows you to toggle MSW handlers in local development as well as easily invalidate all of the caches.
@@ -35,6 +63,7 @@ export const MSWToolbar = ({
   }
 
   const workerRef = React.useRef<SetupWorkerApi>();
+  const trackedRequestsRef = React.useRef<TrackedRequest>(new Map());
 
   const [isReady, setIsReady] = React.useState(isEnabled ? false : true);
 
@@ -83,7 +112,39 @@ export const MSWToolbar = ({
     set(prefix, 'mode', mode);
 
     switch (mode) {
+      case 'record':
+        // Passing empty handlers will cause everything to run as a bypass
+        workerRef.current?.use(...[]);
+        workerRef.current?.events.on('request:start', request => {
+          trackedRequestsRef.current.set(request.id, {
+            request,
+            response: null as any,
+          });
+        });
+
+        workerRef.current?.events.on('response:bypass', async (res, reqId) => {
+          const clone = res.clone();
+          const existingRequestEntry = trackedRequestsRef.current.get(reqId);
+          if (existingRequestEntry) {
+            trackedRequestsRef.current.set(reqId, {
+              ...existingRequestEntry,
+              response: clone,
+            });
+          }
+        });
+
+        return;
+      case 'replay':
+        workerRef.current?.events.removeAllListeners();
+        buildHandlersFromRecordedRequests(trackedRequestsRef.current).then(
+          handlers => {
+            workerRef.current?.resetHandlers(...handlers);
+          }
+        );
+
+        return;
       case 'normal':
+        workerRef.current?.events.removeAllListeners();
         workerRef.current?.resetHandlers();
         return;
       case 'error':
diff --git a/src/helpers/settings.ts b/src/helpers/settings.ts
index dcb3441..ab6831a 100644
--- a/src/helpers/settings.ts
+++ b/src/helpers/settings.ts
@@ -1,7 +1,7 @@
 import { Setting } from '..';
 import { WorkerMode, WorkerStatus } from '../types';
 
-export const modes: WorkerMode[] = ['normal', 'error'];
+export const modes: WorkerMode[] = ['normal', 'error', 'record', 'replay'];
 
 type SettingValueMap = {
   mode: WorkerMode;
diff --git a/src/types.ts b/src/types.ts
index 0583d05..59a41f7 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -1,4 +1,4 @@
-import { SetupWorkerApi } from 'msw';
+import { MockedRequest, SetupWorkerApi } from 'msw';
 
 export interface MSWToolbarProps {
   /**
@@ -32,4 +32,11 @@ export interface MSWToolbarProps {
 
 export type Setting = 'mode' | 'delay' | 'status';
 export type WorkerStatus = 'enabled' | 'disabled';
-export type WorkerMode = 'normal' | 'error';
+export type WorkerMode = 'normal' | 'error' | 'record' | 'replay';
+export type TrackedRequest = Map<
+  string,
+  {
+    request: MockedRequest;
+    response: Response | null;
+  }
+>;
diff --git a/tsconfig.json b/tsconfig.json
index 2c85b2d..fc26cf7 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -31,5 +31,6 @@
     "forceConsistentCasingInFileNames": true,
     // `tsdx build` ignores this option, but it is commonly used when type-checking separately with `tsc`
     "noEmit": true,
+    "target": "es2015"
   }
 }

From dbfa0b94669402ae0d1c47aa0aea9f40bfb275a3 Mon Sep 17 00:00:00 2001
From: Matt Sutkowski <msutkowski@gmail.com>
Date: Sun, 26 Sep 2021 13:23:17 -0700
Subject: [PATCH 2/2] Convert tracked requests to HAR, more infra

---
 example/index.tsx            |   6 +-
 package.json                 |   5 +
 src/component/MSWToolbar.tsx |  65 +++---------
 src/helpers/har.ts           | 162 ++++++++++++++++++++++++++++++
 src/helpers/index.ts         |   1 +
 src/helpers/source.ts        | 187 +++++++++++++++++++++++++++++++++++
 src/types.ts                 |   1 +
 yarn.lock                    |   7 +-
 8 files changed, 379 insertions(+), 55 deletions(-)
 create mode 100644 src/helpers/har.ts
 create mode 100644 src/helpers/source.ts

diff --git a/example/index.tsx b/example/index.tsx
index 803107e..503b5a7 100644
--- a/example/index.tsx
+++ b/example/index.tsx
@@ -36,7 +36,7 @@ const App = () => {
           prefix={APP_NAME}
         >
           <Button
-            onClick={() =>
+            onClick={() => {
               fetch('https://pokeapi.co/api/v2/pokemon/ditto').then(
                 async (res) => {
                   if (res.ok) {
@@ -46,8 +46,8 @@ const App = () => {
                     );
                   }
                 }
-              )
-            }
+              );
+            }}
             mt={50}
           >
             Make a request
diff --git a/package.json b/package.json
index 88e591e..15ea4b3 100644
--- a/package.json
+++ b/package.json
@@ -54,6 +54,7 @@
   ],
   "devDependencies": {
     "@size-limit/preset-small-lib": "^5.0.4",
+    "@types/har-format": "^1.2.7",
     "@types/react": "^17.0.24",
     "@types/react-dom": "^17.0.9",
     "husky": "^7.0.0",
@@ -66,5 +67,9 @@
     "tsdx": "^0.14.1",
     "tslib": "^2.3.1",
     "typescript": "^4.4.3"
+  },
+  "dependencies": {
+    "headers-utils": "^3.0.2",
+    "set-cookie-parser": "^2.4.8"
   }
 }
diff --git a/src/component/MSWToolbar.tsx b/src/component/MSWToolbar.tsx
index 5c62112..2176ad9 100644
--- a/src/component/MSWToolbar.tsx
+++ b/src/component/MSWToolbar.tsx
@@ -1,45 +1,12 @@
 import * as React from 'react';
-import { RequestHandler, rest } from 'msw';
-import {
-  SetupWorkerApi,
-  MockedRequest,
-  ResponseComposition,
-  RestContext,
-} from 'msw';
+import { rest } from 'msw';
+import { SetupWorkerApi } from 'msw';
 import { usePrevious } from './hooks';
 import styles from './styles.module.css';
 import { WorkerMode } from '../types';
-import { get, modes, set } from '../helpers';
+import { buildHandlersForTrackedRequests, get, modes, set } from '../helpers';
 import { MSWToolbarProps, TrackedRequest } from '..';
 
-async function buildHandlersFromRecordedRequests(
-  trackedRequests: TrackedRequest
-): Promise<RequestHandler[]> {
-  let handlers = [];
-  for (const [, { request, response }] of trackedRequests.entries()) {
-    if (!response) continue;
-
-    // TODO: this should probably happen before we add it to the map,
-    // and we'd most likely want to set the response type there (text, json)
-    // along with the data
-    const data = await response.json();
-
-    const handler = (rest as any)[request.method.toLocaleLowerCase()](
-      request.url.href,
-      (
-        _req: MockedRequest<any>,
-        res: ResponseComposition<any>,
-        ctx: RestContext
-      ) => {
-        return res(ctx.json(data));
-      }
-    );
-    handlers.push(handler);
-  }
-
-  return handlers;
-}
-
 /**
  * This is a simple toolbar that allows you to toggle MSW handlers in local development as well as easily invalidate all of the caches.
  *
@@ -117,18 +84,19 @@ export const MSWToolbar = ({
         workerRef.current?.use(...[]);
         workerRef.current?.events.on('request:start', request => {
           trackedRequestsRef.current.set(request.id, {
+            time: Number(new Date()),
             request,
-            response: null as any,
+            response: null,
           });
         });
 
         workerRef.current?.events.on('response:bypass', async (res, reqId) => {
-          const clone = res.clone();
           const existingRequestEntry = trackedRequestsRef.current.get(reqId);
           if (existingRequestEntry) {
             trackedRequestsRef.current.set(reqId, {
               ...existingRequestEntry,
-              response: clone,
+              time: existingRequestEntry.time - Number(new Date()),
+              response: res.clone(),
             });
           }
         });
@@ -136,7 +104,7 @@ export const MSWToolbar = ({
         return;
       case 'replay':
         workerRef.current?.events.removeAllListeners();
-        buildHandlersFromRecordedRequests(trackedRequestsRef.current).then(
+        buildHandlersForTrackedRequests(trackedRequestsRef.current).then(
           handlers => {
             workerRef.current?.resetHandlers(...handlers);
           }
@@ -149,17 +117,12 @@ export const MSWToolbar = ({
         return;
       case 'error':
         workerRef.current?.use(
-          ...['get', 'post', 'put', 'patch', 'delete'].map(method =>
-            (rest as any)[method as any](
-              `${apiUrl}/*`,
-              (
-                _req: MockedRequest<any>,
-                res: ResponseComposition<any>,
-                _ctx: RestContext
-              ) => {
-                return res.networkError('Fake error');
-              }
-            )
+          ...(['get', 'post', 'put', 'patch', 'delete'] as Array<
+            keyof typeof rest
+          >).map(method =>
+            rest[method](`${apiUrl}/*`, (_req, res, _ctx) => {
+              return res.networkError('Fake error');
+            })
           )
         );
         return;
diff --git a/src/helpers/har.ts b/src/helpers/har.ts
new file mode 100644
index 0000000..3efb513
--- /dev/null
+++ b/src/helpers/har.ts
@@ -0,0 +1,162 @@
+import { MockedRequest, RequestHandler } from 'msw';
+import setCookie from 'set-cookie-parser';
+import { Entry, Header as HARHeader, Cookie as HARCookie } from 'har-format';
+import { HeadersObject, headersToObject } from 'headers-utils/lib';
+import { TrackedRequest } from '..';
+import { fromTraffic } from './source';
+
+const normalizeValueAsString = (value: string | string[]): string => {
+  return Array.isArray(value) ? value.join(', ') : value;
+};
+
+function buildHARHeaders(headers: HeadersObject): HARHeader[] {
+  const result = Object.keys(headers).map(name => ({
+    name,
+    value: normalizeValueAsString(headers[name]),
+  }));
+
+  return result;
+}
+
+function buildHARCookies(cookies: Record<string, string>): HARCookie[] {
+  return Object.keys(cookies).map(name => ({
+    name,
+    value: cookies[name],
+  }));
+}
+
+function buildResponseCookies(headers: HeadersObject) {
+  const cookies: HARCookie[] = [];
+
+  const setCookies = headers['set-cookie'];
+
+  if (setCookies) {
+    for (const headerValue of setCookies) {
+      let parsed: setCookie.Cookie[];
+      try {
+        parsed = setCookie.parse(headerValue);
+      } catch (err) {
+        return;
+      }
+      for (const cookie of parsed) {
+        const { name, value, path, domain, expires, httpOnly, secure } = cookie;
+        const harCookie: HARCookie = {
+          name,
+          value,
+          httpOnly: httpOnly ?? false,
+          secure: secure ?? false,
+          path,
+          domain,
+          expires: expires?.toISOString(),
+        };
+
+        cookies.push(harCookie);
+      }
+    }
+  }
+
+  return cookies;
+}
+
+const convertCapturedLifecycleToHAR = async (
+  time: number,
+  request: MockedRequest,
+  res: Response
+) => {
+  const response = res.clone();
+  let entry: Entry = {
+    time,
+    startedDateTime: new Date().toISOString(),
+    cache: {
+      beforeRequest: null,
+      afterRequest: null,
+    },
+    timings: {
+      blocked: -1,
+      dns: -1,
+      connect: -1,
+      send: 0,
+      wait: 0,
+      receive: 0,
+      ssl: -1,
+    },
+    request: {
+      httpVersion: 'HTTP/1.1', // TODO: Can we capture this?
+      method: request.method,
+      url: request.url.href,
+      cookies: buildHARCookies(request.cookies),
+      headers: request.headers as any,
+      queryString: [...(request.url.searchParams as any)].map(
+        ([name, value]) => ({
+          name,
+          value,
+        })
+      ),
+      headersSize: -1,
+      bodySize: -1,
+    },
+  } as any;
+
+  const text = await response.text();
+  const bodySize = Buffer.byteLength(text);
+
+  const headers = headersToObject(response.headers);
+
+  entry.response = {
+    status: response.status,
+    statusText: response.statusText,
+    // httpVersion,
+    cookies: buildResponseCookies(headers),
+    headers: buildHARHeaders(headers),
+    content: {
+      mimeType: headers['content-type'],
+      text,
+      size: bodySize,
+    },
+    redirectURL: headers['location'] || '',
+    headersSize: -1,
+    bodySize: -1,
+  } as any;
+
+  entry.response.content.text = text;
+  entry.response.content.size = bodySize;
+
+  return entry;
+};
+
+function createHarLog(entries: Entry[] = []) {
+  return {
+    log: {
+      version: '1.1',
+      creator: {
+        version: '1',
+        name: 'msw-toolbar',
+      },
+      entries,
+    },
+  };
+}
+
+async function buildHandlersForTrackedRequests(
+  trackedRequests: TrackedRequest
+): Promise<RequestHandler[]> {
+  let entries = [];
+  for (const [, { request, response, time }] of trackedRequests.entries()) {
+    if (!response) continue;
+    const harEntry = await convertCapturedLifecycleToHAR(
+      time,
+      request,
+      response
+    );
+    entries.push(harEntry);
+  }
+  const harLog = createHarLog(entries);
+
+  return fromTraffic(harLog);
+}
+
+export {
+  convertCapturedLifecycleToHAR,
+  createHarLog,
+  buildHandlersForTrackedRequests,
+};
diff --git a/src/helpers/index.ts b/src/helpers/index.ts
index fa8ac58..f51bb74 100644
--- a/src/helpers/index.ts
+++ b/src/helpers/index.ts
@@ -1 +1,2 @@
 export * from './settings';
+export * from './har';
diff --git a/src/helpers/source.ts b/src/helpers/source.ts
new file mode 100644
index 0000000..08331ec
--- /dev/null
+++ b/src/helpers/source.ts
@@ -0,0 +1,187 @@
+import { invariant } from 'outvariant';
+import { Har, Entry, Request, Response } from 'har-format';
+import { Cookie, parse } from 'set-cookie-parser';
+import { Headers, headersToObject } from 'headers-utils';
+import {
+  RestHandler,
+  rest,
+  context,
+  ResponseFunction,
+  ResponseTransformer,
+  ResponseComposition,
+} from 'msw';
+import { Buffer } from 'buffer';
+
+export function decodeBase64String(text: string): Uint8Array {
+  return Uint8Array.from(Buffer.from(text, 'base64').toString('binary'), char =>
+    char.charCodeAt(0)
+  );
+}
+
+export type MapEntryFn = (entry: Entry) => Entry | undefined;
+
+type ResponseProducer = (
+  res: ResponseComposition<unknown>,
+  transformers: ResponseTransformer<unknown>[]
+) => ReturnType<ResponseFunction>;
+
+const defaultResponseProducer: ResponseProducer = (res, transformers) => {
+  return res(...transformers);
+};
+
+/**
+ * Create a request handler from the given Network Archive entry.
+ */
+function toRequestHandler(
+  entry: Entry,
+  produceResponse: ResponseProducer = defaultResponseProducer
+): RestHandler {
+  const { request } = entry;
+  const method = request.method.toLowerCase() as keyof typeof rest;
+  const transformers = toResponseTransformers(entry);
+
+  return rest[method](request.url, (_req, res) => {
+    return produceResponse(res, transformers);
+  });
+}
+
+/**
+ * Map a traffic entry to the array of response transformers.
+ */
+export function toResponseTransformers(entry: Entry): ResponseTransformer[] {
+  const { response, time } = entry;
+
+  const transformers: ResponseTransformer[] = [];
+  const responseHeaders = new Headers();
+  const responseCookies: Cookie[] = [];
+
+  // Response status and status text.
+  transformers.push(context.status(response.status, response.statusText));
+
+  // Response headers.
+  for (const header of response.headers) {
+    const headerName = header.name.toLowerCase();
+
+    // Skip response cookie headers because a mocked response cookies
+    // are not implemented through headers (security consideration).
+    // Store the list of cookie headers to apply them via `ctx.cookie` later.
+    if (['set-cookie', 'set-cookie2'].includes(headerName)) {
+      responseCookies.push(...parse(header.value));
+      continue;
+    }
+
+    // Skip the "Content-Encoding" header to prevent "incorrect header check" errors.
+    // MSW must not attempt to compress the response body, even if it was originally
+    // compressed. All response bodies are sent uncompressed.
+    if (headerName === 'content-encoding') {
+      continue;
+    }
+
+    responseHeaders.set(header.name, header.value);
+  }
+
+  transformers.push(context.set(headersToObject(responseHeaders)));
+
+  // Response cookies.
+  for (const cookie of responseCookies) {
+    const { name, value, ...options } = cookie;
+
+    transformers.push(
+      context.cookie(name, value, {
+        ...options,
+        sameSite: options.sameSite === '',
+      })
+    );
+  }
+
+  // Response delay.
+  if (time) {
+    transformers.push(context.delay(time));
+  }
+
+  // Response body.
+  const responseBody = toResponseBody(response);
+
+  if (responseBody) {
+    transformers.push(context.body(responseBody));
+  }
+
+  return transformers;
+}
+
+/**
+ * Extract a response body from the given HAR response entry.
+ * Decodes any base64-encoded text response bodies.
+ */
+export function toResponseBody(
+  response: Response
+): Uint8Array | string | undefined {
+  const { text, encoding, mimeType } = response.content;
+
+  if (!text) {
+    return;
+  }
+
+  if (encoding === 'base64' && mimeType.includes('text')) {
+    const responseBody = decodeBase64String(text);
+    return responseBody;
+  }
+
+  return text;
+}
+
+/**
+ * Generate request handlers from the given HAR file.
+ */
+export function fromTraffic(har: Har, mapEntry?: MapEntryFn): RestHandler[] {
+  invariant(
+    har,
+    'Failed to generate request handlers from traffic: expected an HAR object but got %s.',
+    typeof har
+  );
+
+  invariant(
+    har.log.entries.length > 0,
+    'Failed to generate request handlers from traffic: given HAR object has no entries.'
+  );
+
+  const requestPaths = new Set<string>();
+
+  const handlers = har.log.entries.reduceRight<RestHandler[]>(
+    (handlers, entry) => {
+      const resolvedEntry = mapEntry ? mapEntry(entry) : entry;
+
+      if (!resolvedEntry) {
+        return handlers;
+      }
+
+      const requestPath = createRequestPath(resolvedEntry.request);
+      const isUniqueHandler = !requestPaths.has(requestPath);
+
+      const handler = toRequestHandler(resolvedEntry, (res, transformers) => {
+        // Reducing the entries from right to left implies that the first
+        // entry we meet is, in fact, the last entry recorded.
+        // Always create a regular handler for the last entry.
+        // If there are any consecutive entries for the same URL,
+        // create a one-time handler instead to preserve response order.
+        const responseFn = isUniqueHandler ? res : res.once;
+        return responseFn(...transformers);
+      });
+
+      // Prepend the handler to the list of handler because we're reducing
+      // from right to left, but the order of handlers must correspond
+      // to the chronological order of requests.
+      handlers.unshift(handler);
+      requestPaths.add(requestPath);
+
+      return handlers;
+    },
+    []
+  );
+
+  return handlers;
+}
+
+function createRequestPath(request: Request): string {
+  return `${request.method}+${request.url}`;
+}
diff --git a/src/types.ts b/src/types.ts
index 59a41f7..52213ca 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -38,5 +38,6 @@ export type TrackedRequest = Map<
   {
     request: MockedRequest;
     response: Response | null;
+    time: number;
   }
 >;
diff --git a/yarn.lock b/yarn.lock
index cb3b77b..081ff6f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1312,6 +1312,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/har-format@^1.2.7":
+  version "1.2.7"
+  resolved "https://registry.yarnpkg.com/@types/har-format/-/har-format-1.2.7.tgz#debfe36378f26c4fc2abca1df99f00a8ff94fd29"
+  integrity sha512-/TPzUG0tJn5x1TUcVLlDx2LqbE58hyOzDVAc9kf8SpOEmguHjU6bKUyfqb211AdqLOmU/SNyXvLKPNP5qTlfRw==
+
 "@types/inquirer@^7.3.3":
   version "7.3.3"
   resolved "https://registry.yarnpkg.com/@types/inquirer/-/inquirer-7.3.3.tgz#92e6676efb67fa6925c69a2ee638f67a822952ac"
@@ -7486,7 +7491,7 @@ set-blocking@^2.0.0:
   resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
   integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
 
-set-cookie-parser@^2.4.6:
+set-cookie-parser@^2.4.6, set-cookie-parser@^2.4.8:
   version "2.4.8"
   resolved "https://registry.yarnpkg.com/set-cookie-parser/-/set-cookie-parser-2.4.8.tgz#d0da0ed388bc8f24e706a391f9c9e252a13c58b2"
   integrity sha512-edRH8mBKEWNVIVMKejNnuJxleqYE/ZSdcT8/Nem9/mmosx12pctd80s2Oy00KNZzrogMZS5mauK2/ymL1bvlvg==