Editor fixes: preserve whitespace, exclude as a printf conversion…

… flag, match brackets better (mozilla#2998)

* Add initial EditField tests

* In editor, preserve whitespace & fix bracket matching and closing

* Exclude ` ` as a printf conversion flag

translate/src/modules/translationform/components/EditField.test.tsx

@@ -0,0 +1,148 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+
+import * as Fluent from '@fluent/react';
+import { act, render, screen } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+import React from 'react';
+import sinon from 'sinon';
+
+import { EditFieldHandle, EditorActions } from '~/context/Editor';
+import { EntityView } from '~/context/EntityView';
+import { Locale } from '~/context/Locale';
+// @ts-expect-error
+import { createReduxStore, MockStore } from '~/test/store';
+
+import { EditField } from './EditField';
+
+function MockEditField({
+  defaultValue,
+  singleField,
+  format,
+  fieldRef,
+  isAuthenticated = true,
+  setResultFromInput = sinon.spy(),
+}: {
+  defaultValue: string;
+  singleField?: boolean;
+  format: 'ftl' | 'plain';
+  fieldRef?: React.RefObject<EditFieldHandle>;
+  isAuthenticated?: boolean;
+  setResultFromInput?: EditorActions['setResultFromInput'];
+}) {
+  const store = createReduxStore({ user: { isAuthenticated, settings: {} } });
+  return (
+    <Fluent.LocalizationProvider
+      l10n={{ getString: (id) => id } as Fluent.ReactLocalization}
+    >
+      <MockStore store={store}>
+        <Locale.Provider value={{ code: 'kg' } as Locale}>
+          <EntityView.Provider
+            value={{ entity: { format, translation: {} } } as EntityView}
+          >
+            <EditorActions.Provider
+              value={{ setResultFromInput } as EditorActions}
+            >
+              <EditField
+                ref={fieldRef}
+                defaultValue={defaultValue}
+                index={0}
+                singleField={singleField}
+              />
+            </EditorActions.Provider>
+          </EntityView.Provider>
+        </Locale.Provider>
+      </MockStore>
+    </Fluent.LocalizationProvider>
+  );
+}
+
+describe('<EditField>', () => {
+  let createRangeBackup: () => Range;
+
+  beforeAll(() => {
+    createRangeBackup = document.createRange;
+    // Hack adopted from https://discuss.codemirror.net/t/working-in-jsdom-or-node-js-natively/138/5
+    document.createRange = () =>
+      ({
+        setEnd() {},
+        setStart() {},
+        getBoundingClientRect: () => ({ right: 0 }),
+        getClientRects: () => ({ length: 0, left: 0, right: 0 }),
+      } as unknown as Range);
+  });
+
+  afterAll(() => {
+    document.createRange = createRangeBackup;
+  });
+
+  it('renders field correctly', () => {
+    const { container } = render(
+      <MockEditField defaultValue='foo' format='ftl' />,
+    );
+
+    const lines = container.querySelectorAll('.cm-line');
+    expect(lines).toHaveLength(1);
+    expect(lines[0].textContent).toEqual('foo');
+  });
+
+  it('sets the result on user input', async () => {
+    const spy = sinon.spy();
+    const { container } = render(
+      <MockEditField
+        defaultValue='foo'
+        format='ftl'
+        setResultFromInput={spy}
+      />,
+    );
+    await userEvent.click(container.querySelector('.cm-line')!);
+    await userEvent.keyboard('x{ArrowRight}{ArrowRight}{ArrowRight}  y');
+    expect(spy.getCalls()).toMatchObject([
+      { args: [0, 'xfoo'] },
+      { args: [0, 'xfoo '] },
+      { args: [0, 'xfoo  '] },
+      { args: [0, 'xfoo  y'] },
+    ]);
+  });
+
+  it('ignores user input when readonly', async () => {
+    const spy = sinon.spy();
+    const { container } = render(
+      <MockEditField
+        defaultValue='foo'
+        format='ftl'
+        isAuthenticated={false}
+        setResultFromInput={spy}
+      />,
+    );
+    await userEvent.click(container.querySelector('.cm-line')!);
+    await userEvent.keyboard('x');
+    expect(spy.getCalls()).toMatchObject([]);
+  });
+
+  it('sets the result via ref', async () => {
+    const spy = sinon.spy();
+    const ref = React.createRef<EditFieldHandle>();
+    render(
+      <MockEditField
+        defaultValue='foo'
+        format='ftl'
+        fieldRef={ref}
+        setResultFromInput={spy}
+      />,
+    );
+    act(() => {
+      ref.current!.focus();
+      ref.current!.setSelection('bar');
+    });
+    expect(spy.getCalls()).toMatchObject([{ args: [0, 'foobar'] }]);
+  });
+
+  it('does not highlight `% d` as code (#2988)', () => {
+    render(<MockEditField defaultValue='{0}% done' format='plain' />);
+    const placeholder = screen.getByText(/0/);
+    const notPrintf = screen.getByText(/% d/);
+    const certainlyText = screen.getByText(/one/);
+    expect(notPrintf).not.toBe(placeholder);
+    expect(notPrintf).toBe(certainlyText);
+  });
+});

translate/src/modules/translationform/utils/editFieldExtensions.ts

@@ -8,7 +8,6 @@ import {
 import {
   HighlightStyle,
   StreamLanguage,
-  bracketMatching,
   syntaxHighlighting,
 } from '@codemirror/language';
 import { Extension } from '@codemirror/state';
@@ -73,16 +72,15 @@ const style = HighlightStyle.define([
   }, // <...>
   { tag: tags.brace, color: '#872bff', fontWeight: 'bold', whiteSpace: 'pre' }, // { }
   { tag: tags.name, color: '#872bff', whiteSpace: 'pre' }, // {...}
-  { tag: [tags.quote, tags.literal], whiteSpace: 'pre' }, // "..."
-  { tag: tags.string, whiteSpace: 'pre-line' },
+  { tag: [tags.quote, tags.literal], whiteSpace: 'pre-wrap' }, // "..."
+  { tag: tags.string, whiteSpace: 'pre-wrap' },
 ]);
 
 export const getExtensions = (
   format: string,
   ref: ReturnType<typeof useKeyHandlers>,
 ): Extension[] => [
   history(),
-  bracketMatching(),
   closeBrackets(),
   EditorView.lineWrapping,
   StreamLanguage.define<any>(format === 'ftl' ? fluentMode : commonMode),

translate/src/modules/translationform/utils/editFieldModes.ts

@@ -3,6 +3,7 @@ import { StreamParser } from '@codemirror/language';
 export const fluentMode: StreamParser<Array<'expression' | 'literal' | 'tag'>> =
   {
     name: 'fluent',
+    languageData: { closeBrackets: { brackets: ['(', '[', '{', '"', '<'] } },
     startState: () => [],
     token(stream, state) {
       const ch = stream.next();
@@ -31,8 +32,19 @@ export const fluentMode: StreamParser<Array<'expression' | 'literal' | 'tag'>> =
             case '{':
               state.push('expression');
               return 'brace';
+            // These will mis-highlight actual } or > in literals,
+            // but that's a rare enough occurrence when balanced
+            // with the improved editing experience.
+            case '}':
+              state.pop();
+              state.pop();
+              return 'brace';
+            case '>':
+              state.pop();
+              state.pop();
+              return 'bracket';
             default:
-              stream.eatWhile(/[^"{]+/);
+              stream.eatWhile(/[^"{}>]+/);
               return 'literal';
           }
 
@@ -68,13 +80,17 @@ export const fluentMode: StreamParser<Array<'expression' | 'literal' | 'tag'>> =
     },
   };
 
+// Excludes ` ` even if it's a valid Python conversion flag,
+// due to false positives.
+// https://github.com/mozilla/pontoon/issues/2988
 const printf =
-  /^%(\d\$|\(.*?\))?[-+ 0'#]*[\d*]*(\.[\d*])?(hh?|ll?|[jLtz])?[%@AacdEeFfGginopSsuXx]/;
+  /^%(\d\$|\(.*?\))?[-+0'#]*[\d*]*(\.[\d*])?(hh?|ll?|[jLtz])?[%@AacdEeFfGginopSsuXx]/;
 
 const pythonFormat = /^{[\w.[\]]*(![rsa])?(:.*?)?}/;
 
 export const commonMode: StreamParser<Array<'literal' | 'tag'>> = {
   name: 'common',
+  languageData: { closeBrackets: { brackets: ['(', '[', '{', '"', '<'] } },
   startState: () => [],
   token(stream, state) {
     if (stream.match(printf) || stream.match(pythonFormat)) {